wip: rerun panel

app/packages/app/package.json

@@ -58,7 +58,9 @@
         "vite": "^5.2.14",
         "vite-plugin-relay": "^1.0.7",
         "vite-plugin-rewrite-all": "^1.0.2",
-        "vite-plugin-svgr": "^4.2.0"
+        "vite-plugin-svgr": "^4.2.0",
+        "vite-plugin-top-level-await": "^1.4.4",
+        "vite-plugin-wasm": "^3.3.0"
     },
     "peerDependencies": {
         "@mui/icons-material": "*",

app/packages/app/vite.config.ts

@@ -3,6 +3,8 @@ import nodePolyfills from "rollup-plugin-polyfill-node";
 import { defineConfig } from "vite";
 import relay from "vite-plugin-relay";
 import svgr from "vite-plugin-svgr";
+import wasm from "vite-plugin-wasm";
+import topLevelAwait from "vite-plugin-top-level-await";
 import { basePlugins } from "../../vite.base.config";
 
 async function loadConfig() {
@@ -12,6 +14,8 @@ async function loadConfig() {
     base: "",
     plugins: [
       ...basePlugins,
+      wasm(),
+      topLevelAwait(),
       svgr(),
       reactRefresh({
         parserPlugins: ["classProperties", "classPrivateProperties"],
@@ -57,6 +61,10 @@ async function loadConfig() {
         },
       },
     },
+    optimizeDeps: {
+      exclude:
+        process.env.NODE_ENV === "production" ? [] : ["@rerun-io/web-viewer"],
+    },
   });
 }
 

app/packages/core/src/components/Grid/Grid.tsx

@@ -1,6 +1,13 @@
 import styles from "./Grid.module.css";
 
 import { freeVideos } from "@fiftyone/looker";
+import { RerunViewer } from "@fiftyone/playback/src/views/RerunMvp";
+import {
+  TimelineCreator,
+  TimelineSubscriber1,
+  TimelineSubscriber2,
+} from "@fiftyone/playback/src/views/TimelineExample";
+import { PluginComponentType, registerComponent } from "@fiftyone/plugins";
 import type { ID } from "@fiftyone/spotlight";
 import Spotlight from "@fiftyone/spotlight";
 import type { Lookers } from "@fiftyone/state";
@@ -24,6 +31,11 @@ import useSelect from "./useSelect";
 import useSelectSample from "./useSelectSample";
 import useSpotlightPager from "./useSpotlightPager";
 import useThreshold from "./useThreshold";
+import useSelect from "./useSelect";
+import useSelectSample from "./useSelectSample";
+import useSpotlightPager from "./useSpotlightPager";
+import useThreshold from "./useThreshold";
+import { doesSchemaContainEmbeddedDocType } from "@fiftyone/utilities";
 
 function Grid() {
   const id = useMemo(() => uuid(), []);
@@ -177,4 +189,28 @@ function Grid() {
   return <div id={id} className={styles.spotlightGrid} data-cy="fo-grid" />;
 }
 
+// THIS IS JUST FOR TESTING: WILL BE MOVED TO A DEDICATED PLUGIN
+
+const RerunFileDescriptor = {
+  EMBEDDED_DOC_TYPE: "fiftyone.utils.rerun.RrdFile",
+};
+
+registerComponent({
+  name: "Rerun",
+  label: "Rerun",
+  component: RerunViewer,
+  activator: (ctx) => {
+    // only activate if schema has rrd file
+    return doesSchemaContainEmbeddedDocType(
+      ctx.schema,
+      RerunFileDescriptor.EMBEDDED_DOC_TYPE
+    );
+  },
+  type: PluginComponentType.Panel,
+  panelOptions: {
+    surfaces: "modal",
+    helpMarkdown: `Rereun viewer for FiftyOne`,
+  },
+});
+
 export default React.memo(Grid);

app/packages/playback/package.json

@@ -13,6 +13,8 @@
         "vite": "^5.4.6"
     },
     "dependencies": {
+        "@rerun-io/web-viewer": "^0.18.2",
+        "@rerun-io/web-viewer-react": "^0.18.2",
         "jotai": "^2.9.3",
         "jotai-optics": "^0.4.0",
         "vite-plugin-svgr": "^4.2.0"

app/packages/playback/src/views/RerunMvp.tsx

@@ -0,0 +1,100 @@
+import * as fos from "@fiftyone/state";
+import { getFieldsWithEmbeddedDocType } from "@fiftyone/utilities";
+import { WebViewer } from "@rerun-io/web-viewer";
+import React, { useLayoutEffect, useMemo, useRef } from "react";
+import { useRecoilValue } from "recoil";
+
+const RerunFileDescriptor = {
+  EMBEDDED_DOC_TYPE: "fiftyone.utils.rerun.RrdFile",
+};
+
+type RerunFieldDescriptor = {
+  _cls: "RrdFile";
+  filepath: string;
+  version: string;
+};
+
+const Rrd_V_0_18_2 = React.memo(({ url }: { url: string }) => {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const viewerRef = useRef<WebViewer>();
+
+  useLayoutEffect(() => {
+    if (!containerRef.current) {
+      return;
+    }
+
+    (async () => {
+      viewerRef.current = new WebViewer();
+      await viewerRef.current.start(url, containerRef.current, {
+        render_backend: "webgpu",
+        allow_fullscreen: false,
+        hide_welcome_screen: true,
+        width: "100%",
+        height: "100%",
+      });
+    })();
+
+    return () => {
+      viewerRef.current?.stop();
+    };
+  }, [url]);
+
+  return <div ref={containerRef} style={{ height: "100%", width: "100%" }} />;
+});
+
+const RrdRenderer = React.memo(
+  ({ url, version }: { url: string; version: string }) => {
+    switch (version) {
+      case "0.18.2":
+        return <Rrd_V_0_18_2 url={url} />;
+      default:
+        return (
+          <div>
+            We only support version 0.18.2 of the rrd file format, whereas the
+            current version is {version}.
+          </div>
+        );
+    }
+  }
+);
+
+export const RerunViewer = () => {
+  const currentSample = useRecoilValue(fos.modalSample);
+
+  const schema = useRecoilValue(
+    fos.fieldSchema({ space: fos.State.SPACE.SAMPLE })
+  );
+
+  const rerunFieldPath = useMemo(() => {
+    return getFieldsWithEmbeddedDocType(
+      schema,
+      RerunFileDescriptor.EMBEDDED_DOC_TYPE
+    ).at(0)?.path;
+  }, [schema]);
+
+  const rrdParams = useMemo(() => {
+    if (!rerunFieldPath || !currentSample.urls) {
+      return undefined;
+    }
+
+    const filePathAndVersion = currentSample?.sample?.[
+      rerunFieldPath
+    ] as unknown as RerunFieldDescriptor;
+
+    const urlsStandardized = fos.getStandardizedUrls(currentSample.urls);
+
+    const rrdFilePath = urlsStandardized[`${rerunFieldPath}.filepath`];
+
+    const url = fos.getSampleSrc(rrdFilePath);
+    return {
+      url,
+      version: filePathAndVersion.version,
+    };
+  }, [currentSample, rerunFieldPath]);
+
+  if (!rrdParams) {
+    return null;
+  }
+
+  return <RrdRenderer url={rrdParams.url} version={rrdParams.version} />;
+};

app/packages/state/src/utils.ts

@@ -18,7 +18,7 @@ import {
   RecordSource,
   Store,
 } from "relay-runtime";
-import { State } from "./recoil";
+import { ModalSample, State } from "./recoil";
 
 export const deferrer =
   (initialized: MutableRefObject<boolean>) =>
@@ -105,6 +105,7 @@ export const getStandardizedUrls = (
   urls:
     | readonly { readonly field: string; readonly url: string }[]
     | { [field: string]: string }
+    | ModalSample["urls"]
 ) => {
   if (!Array.isArray(urls)) {
     return urls;

app/packages/utilities/src/schema.test.ts

@@ -19,7 +19,7 @@ const SCHEMA: schema.Schema = {
       "fiftyone.core.odm.embedded_document.DynamicEmbeddedDocument",
     ftype: "fiftyone.core.fields.EmbeddedDocumentField",
     info: {},
-    name: "top",
+    name: "embedded",
     path: "embedded",
     subfield: null,
     fields: {
@@ -30,7 +30,7 @@ const SCHEMA: schema.Schema = {
         ftype: "fiftyone.core.fields.EmbeddedDocumentField",
         info: {},
         name: "field",
-        path: "field",
+        path: "embedded.field",
         subfield: null,
       },
     },
@@ -42,7 +42,7 @@ const SCHEMA: schema.Schema = {
       "fiftyone.core.odm.embedded_document.DynamicEmbeddedDocument",
     ftype: "fiftyone.core.fields.EmbeddedDocumentField",
     info: {},
-    name: "top",
+    name: "embeddedWithDbFields",
     path: "embeddedWithDbFields",
     subfield: null,
     fields: {
@@ -54,15 +54,15 @@ const SCHEMA: schema.Schema = {
         ftype: "fiftyone.core.fields.EmbeddedDocumentField",
         info: {},
         name: "sample_id",
-        path: "sample_id",
+        path: "embeddedWithDbFields.sample_id",
         subfield: null,
       },
     },
   },
 };
 
 describe("schema", () => {
-  describe("getCls ", () => {
+  describe("getCls", () => {
     it("should get top level cls", () => {
       expect(schema.getCls("top", SCHEMA)).toBe("TopLabel");
     });
@@ -79,7 +79,7 @@ describe("schema", () => {
     });
   });
 
-  describe("getFieldInfo ", () => {
+  describe("getFieldInfo", () => {
     it("should get top level field info", () => {
       expect(schema.getFieldInfo("top", SCHEMA)).toEqual({
         ...SCHEMA.top,
@@ -89,7 +89,7 @@ describe("schema", () => {
 
     it("should get embedded field info", () => {
       expect(schema.getFieldInfo("embedded.field", SCHEMA)).toEqual({
-        ...SCHEMA.embedded.fields.field,
+        ...SCHEMA.embedded.fields!.field,
         pathWithDbField: "",
       });
     });
@@ -109,4 +109,69 @@ describe("schema", () => {
       expect(field?.pathWithDbField).toBe("embeddedWithDbFields._sample_id");
     });
   });
+
+  describe("getFieldsWithEmbeddedDocType", () => {
+    it("should get all fields with embeddedDocType at top level", () => {
+      expect(
+        schema.getFieldsWithEmbeddedDocType(
+          SCHEMA,
+          "fiftyone.core.labels.TopLabel"
+        )
+      ).toEqual([SCHEMA.top]);
+    });
+
+    it("should get all fields with embeddedDocType in nested fields", () => {
+      expect(
+        schema.getFieldsWithEmbeddedDocType(
+          SCHEMA,
+          "fiftyone.core.labels.EmbeddedLabel"
+        )
+      ).toEqual([
+        SCHEMA.embedded.fields!.field,
+        SCHEMA.embeddedWithDbFields.fields!.sample_id,
+      ]);
+    });
+
+    it("should return empty array if embeddedDocType does not exist", () => {
+      expect(
+        schema.getFieldsWithEmbeddedDocType(SCHEMA, "nonexistentDocType")
+      ).toEqual([]);
+    });
+
+    it("should return empty array for empty schema", () => {
+      expect(schema.getFieldsWithEmbeddedDocType({}, "anyDocType")).toEqual([]);
+    });
+  });
+
+  describe("doesSchemaContainEmbeddedDocType", () => {
+    it("should return true if embeddedDocType exists at top level", () => {
+      expect(
+        schema.doesSchemaContainEmbeddedDocType(
+          SCHEMA,
+          "fiftyone.core.labels.TopLabel"
+        )
+      ).toBe(true);
+    });
+
+    it("should return true if embeddedDocType exists in nested fields", () => {
+      expect(
+        schema.doesSchemaContainEmbeddedDocType(
+          SCHEMA,
+          "fiftyone.core.labels.EmbeddedLabel"
+        )
+      ).toBe(true);
+    });
+
+    it("should return false if embeddedDocType does not exist", () => {
+      expect(
+        schema.doesSchemaContainEmbeddedDocType(SCHEMA, "nonexistentDocType")
+      ).toBe(false);
+    });
+
+    it("should return false for empty schema", () => {
+      expect(schema.doesSchemaContainEmbeddedDocType({}, "anyDocType")).toBe(
+        false
+      );
+    });
+  });
 });

app/packages/utilities/src/schema.ts

@@ -51,3 +51,43 @@ export function getCls(fieldPath: string, schema: Schema): string | undefined {
 
   return field.embeddedDocType.split(".").slice(-1)[0];
 }
+
+export function getFieldsWithEmbeddedDocType(
+  schema: Schema,
+  embeddedDocType: string
+): Field[] {
+  const result: Field[] = [];
+
+  function recurse(schema: Schema) {
+    for (const field of Object.values(schema ?? {})) {
+      if (field.embeddedDocType === embeddedDocType) {
+        result.push(field);
+      }
+      if (field.fields) {
+        recurse(field.fields);
+      }
+    }
+  }
+
+  recurse(schema);
+  return result;
+}
+
+export function doesSchemaContainEmbeddedDocType(
+  schema: Schema,
+  embeddedDocType: string
+): boolean {
+  function recurse(schema: Schema): boolean {
+    return Object.values(schema ?? {}).some((field) => {
+      if (field.embeddedDocType === embeddedDocType) {
+        return true;
+      }
+      if (field.fields) {
+        return recurse(field.fields);
+      }
+      return false;
+    });
+  }
+
+  return recurse(schema);
+}