diff --git a/app/packages/core/src/components/Modal/ImaVidLooker.tsx b/app/packages/core/src/components/Modal/ImaVidLooker.tsx index 1f1c179d4d..7af9d2b911 100644 --- a/app/packages/core/src/components/Modal/ImaVidLooker.tsx +++ b/app/packages/core/src/components/Modal/ImaVidLooker.tsx @@ -1,6 +1,5 @@ import { useTheme } from "@fiftyone/components"; import { AbstractLooker, ImaVidLooker } from "@fiftyone/looker"; -import { BUFFERING_PAUSE_TIMEOUT } from "@fiftyone/looker/src/lookers/imavid/constants"; import { BaseState } from "@fiftyone/looker/src/state"; import { FoTimelineConfig, useCreateTimeline } from "@fiftyone/playback"; import { useDefaultTimelineNameImperative } from "@fiftyone/playback/src/lib/use-default-timeline-name"; @@ -54,6 +53,8 @@ export const ImaVidLookerReact = React.memo( }); const { activeLookerRef, setActiveLookerRef } = useModalContext(); + const imaVidLookerRef = + activeLookerRef as unknown as React.MutableRefObject; const looker = React.useMemo( () => createLooker.current(sampleDataWithExtraParams), @@ -152,19 +153,59 @@ export const ImaVidLookerReact = React.memo( ); }, [ref]); - const loadRange = React.useCallback(async (range: BufferRange) => { - // todo: implement - return new Promise((resolve) => { - setTimeout(() => { - resolve(); - }, BUFFERING_PAUSE_TIMEOUT); - }); - }, []); + const loadRange = React.useCallback( + async (range: Readonly) => { + const storeBufferManager = + imaVidLookerRef.current.frameStoreController.storeBufferManager; + const fetchBufferManager = + imaVidLookerRef.current.frameStoreController.fetchBufferManager; + + if (storeBufferManager.containsRange(range)) { + return; + } + + const unprocessedStoreBufferRange = + storeBufferManager.getUnprocessedBufferRange(range); + const unprocessedBufferRange = + fetchBufferManager.getUnprocessedBufferRange( + unprocessedStoreBufferRange + ); + + if (!unprocessedBufferRange) { + return; + } + + imaVidLookerRef.current.frameStoreController.enqueueFetch( + unprocessedBufferRange + ); + + return new Promise((resolve) => { + const fetchMoreListener = (e: CustomEvent) => { + if ( + e.detail.id === imaVidLookerRef.current.frameStoreController.key + ) { + if (storeBufferManager.containsRange(unprocessedBufferRange)) { + resolve(); + window.removeEventListener( + "fetchMore", + fetchMoreListener as EventListener + ); + } + } + }; + + window.addEventListener( + "fetchMore", + fetchMoreListener as EventListener, + { once: true } + ); + }); + }, + [] + ); const renderFrame = React.useCallback((frameNumber: number) => { - ( - activeLookerRef.current as unknown as ImaVidLooker - )?.element.drawFrameNoAnimation(frameNumber); + imaVidLookerRef.current?.element.drawFrameNoAnimation(frameNumber); }, []); const { getName } = useDefaultTimelineNameImperative(); @@ -189,7 +230,7 @@ export const ImaVidLookerReact = React.memo( const readyWhen = useCallback(async () => { return new Promise((resolve) => { - // wait for total frame count to be resolved + // hack: wait for total frame count to be resolved let intervalId; intervalId = setInterval(() => { if (totalFrameCountRef.current) { @@ -200,6 +241,10 @@ export const ImaVidLookerReact = React.memo( }); }, []); + const onAnimationStutter = useCallback(() => { + imaVidLookerRef.current?.element.checkFetchBufferManager(); + }, []); + const { isTimelineInitialized, registerOnPauseCallback, @@ -210,6 +255,10 @@ export const ImaVidLookerReact = React.memo( name: timelineName, config: timelineCreationConfig, waitUntilInitialized: readyWhen, + // using this mechanism to resume fetch if it was paused + // ideally we have control of fetch in this component but can't do that yet + // since imavid is part of the grid too + onAnimationStutter, }); /** @@ -224,35 +273,27 @@ export const ImaVidLookerReact = React.memo( }); registerOnPlayCallback(() => { - (activeLookerRef.current as unknown as ImaVidLooker).element.update( - () => ({ - playing: true, - }) - ); + imaVidLookerRef.current?.element?.update(() => ({ + playing: true, + })); }); registerOnPauseCallback(() => { - (activeLookerRef.current as unknown as ImaVidLooker).element.update( - () => ({ - playing: false, - }) - ); + imaVidLookerRef.current?.element?.update(() => ({ + playing: false, + })); }); registerOnSeekCallbacks({ start: () => { - (activeLookerRef.current as unknown as ImaVidLooker).element.update( - () => ({ - seeking: true, - }) - ); + imaVidLookerRef.current?.element?.update(() => ({ + seeking: true, + })); }, end: () => { - (activeLookerRef.current as unknown as ImaVidLooker).element.update( - () => ({ - seeking: false, - }) - ); + imaVidLookerRef.current?.element?.update(() => ({ + seeking: false, + })); }, }); } @@ -265,9 +306,8 @@ export const ImaVidLookerReact = React.memo( // hack: poll every 10ms for total frame count // replace with event listener or callback let intervalId = setInterval(() => { - const totalFrameCount = ( - activeLookerRef.current as unknown as ImaVidLooker - ).frameStoreController.totalFrameCount; + const totalFrameCount = + imaVidLookerRef.current.frameStoreController.totalFrameCount; if (totalFrameCount) { setTotalFrameCount(totalFrameCount); clearInterval(intervalId); diff --git a/app/packages/looker/src/elements/imavid/index.ts b/app/packages/looker/src/elements/imavid/index.ts index 2545f48d92..b6038dcaf3 100644 --- a/app/packages/looker/src/elements/imavid/index.ts +++ b/app/packages/looker/src/elements/imavid/index.ts @@ -362,6 +362,8 @@ export class ImaVidElement extends BaseElement { /** * Queue up frames to be fetched if necessary. * This method is not blocking, it merely enqueues a fetch job. + * + * This is for legacy imavid, which is used for thumbnail imavid. */ private ensureBuffers(state: Readonly) { if (!this.framesController.totalFrameCount) { @@ -407,6 +409,19 @@ export class ImaVidElement extends BaseElement { } } + /** + * Starts fetch if there are buffers in the fetch buffer manager + */ + public checkFetchBufferManager() { + if (!this.framesController.totalFrameCount) { + return; + } + + if (this.framesController.fetchBufferManager.buffers.length > 0) { + this.framesController.resumeFetch(); + } + } + renderSelf(state: Readonly) { const { options: { playbackRate, loop }, @@ -443,7 +458,11 @@ export class ImaVidElement extends BaseElement { this.framesController.destroy(); } - this.ensureBuffers(state); + if (this.isThumbnail) { + this.ensureBuffers(state); + } else { + this.checkFetchBufferManager(); + } if (!playing && this.isAnimationActive) { // this flag will be picked up in `drawFrame`, that in turn will call `pause` diff --git a/app/packages/looker/src/lookers/imavid/controller.ts b/app/packages/looker/src/lookers/imavid/controller.ts index 816a4ce278..7ea1b8040e 100644 --- a/app/packages/looker/src/lookers/imavid/controller.ts +++ b/app/packages/looker/src/lookers/imavid/controller.ts @@ -123,8 +123,8 @@ export class ImaVidFramesController { BUFFER_METADATA_FETCHING ); - // subtract by two because 1) cursor is one based and 2) cursor here translates to "after" the cursor - return this.fetchMore(range[0] - 2, range[1] - range[0] || 2).finally( + // subtract/add by two because 1) cursor is one based and 2) cursor here translates to "after" the cursor + return this.fetchMore(range[0] - 2, range[1] - range[0] + 2).finally( () => { this.fetchBufferManager.removeMetadataFromBufferRange(index); } @@ -165,7 +165,7 @@ export class ImaVidFramesController { return this.config.page; } - private get key() { + public get key() { return this.config.key; } @@ -257,23 +257,35 @@ export class ImaVidFramesController { const frameIndices = imageFetchPromisesMap.keys(); const imageFetchPromises = imageFetchPromisesMap.values(); - Promise.all(imageFetchPromises).then((sampleIds) => { - for (let i = 0; i < sampleIds.length; i++) { - const frameIndex = frameIndices.next().value; - const sampleId = sampleIds[i]; - this.store.frameIndex.set(frameIndex, sampleId); - this.store.reverseFrameIndex.set(sampleId, frameIndex); - - this.storeBufferManager.addNewRange([ + Promise.all(imageFetchPromises) + .then((sampleIds) => { + for (let i = 0; i < sampleIds.length; i++) { + const frameIndex = frameIndices.next().value; + const sampleId = sampleIds[i]; + this.store.frameIndex.set(frameIndex, sampleId); + this.store.reverseFrameIndex.set(sampleId, frameIndex); + } + resolve(); + }) + .then(() => { + const newRange = [ Number(data.samples.edges[0].cursor) + 1, Number( data.samples.edges[data.samples.edges.length - 1].cursor ) + 1, - ]); - - resolve(); - } - }); + ] as BufferRange; + + this.storeBufferManager.addNewRange(newRange); + + window.dispatchEvent( + new CustomEvent("fetchMore", { + detail: { + id: this.key, + }, + bubbles: false, + }) + ); + }); } }, }); diff --git a/app/packages/playback/src/lib/constants.ts b/app/packages/playback/src/lib/constants.ts index a48ad2fd2e..f711f22dd4 100644 --- a/app/packages/playback/src/lib/constants.ts +++ b/app/packages/playback/src/lib/constants.ts @@ -1,7 +1,7 @@ export const DEFAULT_FRAME_NUMBER = 1; export const DEFAULT_LOOP = false; export const DEFAULT_SPEED = 1; -export const DEFAULT_TARGET_FRAME_RATE = 29.97; +export const DEFAULT_TARGET_FRAME_RATE = 30; export const DEFAULT_USE_TIME_INDICATOR = false; export const GLOBAL_TIMELINE_ID = "fo-timeline-global"; export const LOAD_RANGE_SIZE = 250; diff --git a/app/packages/playback/src/lib/state.ts b/app/packages/playback/src/lib/state.ts index 6674f2d932..66702627dc 100644 --- a/app/packages/playback/src/lib/state.ts +++ b/app/packages/playback/src/lib/state.ts @@ -133,6 +133,11 @@ export type CreateFoTimeline = { * If true, the creator will be responsible for managing the animation loop. */ optOutOfAnimation?: boolean; + + /** + * Callback to be called when the animation stutters. + */ + onAnimationStutter?: () => void; }; const _frameNumbers = atomFamily((_timelineName: TimelineName) => @@ -339,7 +344,7 @@ export const setFrameNumberAtom = atom( set(_currentBufferingRange(name), newLoadRange); try { - await Promise.all(rangeLoadPromises); + await Promise.allSettled(rangeLoadPromises); bufferManager.addNewRange(newLoadRange); } catch (e) { // todo: handle error better, maybe retry @@ -358,9 +363,8 @@ export const setFrameNumberAtom = atom( renderPromises.push(subscriber.renderFrame(newFrameNumber)); }); - Promise.all(renderPromises).then(() => { - set(_frameNumbers(name), newFrameNumber); - }); + await Promise.allSettled(renderPromises); + set(_frameNumbers(name), newFrameNumber); } ); diff --git a/app/packages/playback/src/lib/use-create-timeline.ts b/app/packages/playback/src/lib/use-create-timeline.ts index 2610682eee..b70719b009 100644 --- a/app/packages/playback/src/lib/use-create-timeline.ts +++ b/app/packages/playback/src/lib/use-create-timeline.ts @@ -52,6 +52,13 @@ export const useCreateTimeline = ( const setFrameNumber = useSetAtom(setFrameNumberAtom); const setPlayHeadState = useSetAtom(updatePlayheadStateAtom); + /** + * this effect syncs onAnimationStutter ref from props + */ + useEffect(() => { + onAnimationStutterRef.current = newTimelineProps.onAnimationStutter; + }, [newTimelineProps.onAnimationStutter]); + /** * this effect creates the timeline */ @@ -137,6 +144,7 @@ export const useCreateTimeline = ( const isAnimationActiveRef = useRef(false); const isLastDrawFinishedRef = useRef(true); const frameNumberRef = useRef(frameNumber); + const onAnimationStutterRef = useRef(newTimelineProps.onAnimationStutter); const onPlayListenerRef = useRef<() => void>(); const onPauseListenerRef = useRef<() => void>(); const onSeekCallbackRefs = useRef<{ start: () => void; end: () => void }>(); @@ -145,11 +153,15 @@ export const useCreateTimeline = ( const updateFreqRef = useRef(updateFreq); const play = useCallback(() => { + if (!isTimelineInitialized) { + return; + } + setPlayHeadState({ name: timelineName, state: "playing" }); if (onPlayListenerRef.current) { onPlayListenerRef.current(); } - }, [timelineName]); + }, [timelineName, isTimelineInitialized]); const pause = useCallback(() => { setPlayHeadState({ name: timelineName, state: "paused" }); @@ -167,7 +179,7 @@ export const useCreateTimeline = ( play(); e.stopPropagation(); }, - [timelineName] + [timelineName, play] ); const onPauseEvent = useCallback( @@ -179,7 +191,7 @@ export const useCreateTimeline = ( pause(); e.stopPropagation(); }, - [timelineName] + [timelineName, pause] ); const onSeek = useCallback( @@ -258,13 +270,18 @@ export const useCreateTimeline = ( // queue next animation before draw animationId.current = requestAnimationFrame(animate); + // usually happens when we're out of frames in store if (!isLastDrawFinishedRef.current) { + queueMicrotask(() => { + onAnimationStutterRef.current?.(); + }); return; } // drawing logic is owned by subscribers and invoked by setFrameNumber // we don't increase frame number until the draw is complete isLastDrawFinishedRef.current = false; + setFrameNumber({ name: timelineName, newFrameNumber: targetFrameNumber, @@ -272,6 +289,9 @@ export const useCreateTimeline = ( .then(() => { frameNumberRef.current = targetFrameNumber; }) + .catch((e) => { + console.error("error setting frame number", e); + }) .finally(() => { isLastDrawFinishedRef.current = true; }); @@ -292,6 +312,7 @@ export const useCreateTimeline = ( const cancelAnimation = useCallback(() => { cancelAnimationFrame(animationId.current); isAnimationActiveRef.current = false; + lastDrawTime.current = -1; }, []); useEventHandler(window, "play", onPlayEvent); diff --git a/app/packages/playback/src/views/PlaybackElements.tsx b/app/packages/playback/src/views/PlaybackElements.tsx index 28a1fe02df..0b3ffeece8 100644 --- a/app/packages/playback/src/views/PlaybackElements.tsx +++ b/app/packages/playback/src/views/PlaybackElements.tsx @@ -97,7 +97,7 @@ export const Seekbar = React.forwardRef< unBuffered: "var(--fo-palette-neutral-softBorder)", currentProgress: "var(--fo-palette-primary-plainColor)", buffered: "var(--fo-palette-secondary-main)", - loading: "red", + loading: "#a86738", }), [loadedScaled, loadingScaled, value] ); @@ -161,9 +161,6 @@ export const Speed = React.forwardRef< >(({ speed, setSpeed, ...props }, ref) => { const { style, className, ...otherProps } = props; - const [isPlaybackConfigurerOpen, setIsPlaybackConfigurerOpen] = - React.useState(false); - const onChangeSpeed = React.useCallback( (e: React.ChangeEvent) => { setSpeed(parseFloat(e.target.value)); @@ -173,6 +170,10 @@ export const Speed = React.forwardRef< const rangeValue = React.useMemo(() => (speed / 2) * 100, [speed]); + const resetSpeed = React.useCallback(() => { + setSpeed(1); + }, []); + return ( { - setIsPlaybackConfigurerOpen(false); - }} + title={`${speed}x (click to reset)`} > { - setIsPlaybackConfigurerOpen(true); - }} + onClick={resetSpeed} /> { expect(mergedBuffers[0][1]).toBe(20); }); + test("addBufferRangeToBuffer method - same ranges", async () => { + bufferManager.addNewRange([1, 10]); + bufferManager.addNewRange([1, 10]); + + const mergedBuffers = bufferManager.buffers; + + // we expect [1, 10] + expect(mergedBuffers.length).toBe(1); + expect(mergedBuffers[0][0]).toBe(1); + expect(mergedBuffers[0][1]).toBe(10); + }); + test("addBufferRangeToBuffer method - multiple merges", async () => { bufferManager.addNewRange([1, 4]); bufferManager.addNewRange([5, 7]); diff --git a/app/packages/utilities/src/buffer-manager/index.ts b/app/packages/utilities/src/buffer-manager/index.ts index bdfbbeda2c..278162c034 100644 --- a/app/packages/utilities/src/buffer-manager/index.ts +++ b/app/packages/utilities/src/buffer-manager/index.ts @@ -11,8 +11,8 @@ export class BufferManager { [rangeIndex: number]: string; }; - constructor(buffers: Buffers = []) { - this.buffers = buffers; + constructor(buffers: Readonly = []) { + this.buffers = [...buffers]; this.bufferMetadata = {}; } @@ -53,21 +53,21 @@ export class BufferManager { * Time complexity: O(nlogn) */ public addNewRange( - range: Readonly, + newRange: Readonly, ignoreRangesWithMetadata = true ): void { - if (!range) { + if (!newRange) { return; } - if (range[1] < range[0]) { + if (newRange[1] < newRange[0]) { throw new Error( - `invalid range: range[1] (value = ${range[1]}) must be >= range[0] (value = ${range[0]})` + `invalid range: range[1] (value = ${newRange[1]}) must be >= range[0] (value = ${newRange[0]})` ); } // add the new range to the buffer - this.buffers.push(range); + this.buffers.push(newRange); // sort the buffers based on their start value this.buffers.sort((a, b) => a[0] - b[0]); @@ -106,7 +106,7 @@ export class BufferManager { // if current interval is not overlapping with stack top, // push it to the stack if (!areTwoRangesConsecutive && top[1] < rangesWithoutMetadata[i][0]) { - stack.push(rangesWithoutMetadata[i]); + stack.push([...rangesWithoutMetadata[i]]); } // else if end of current interval is more than the // end of stack top interval, update the stack top @@ -207,7 +207,11 @@ export class BufferManager { * input range: [5, 105] * output: [101-105] */ - public getUnprocessedBufferRange(range: Readonly) { + public getUnprocessedBufferRange(range: Readonly | null) { + if (!range) { + return null; + } + const startContainedInRangeIndex = this.getRangeIndexForFrame(range[0]); if (startContainedInRangeIndex === -1) {