FPS Clamping DIVE (#722)

* FPS Clamping DIVE

* Implement on worker

* Fix parseint

* Update clamping

* WIP again

* Fix tests

* Finish clamping, parse FPS from CSV

* Fix video stream discovery

Author: subdavis

client/dive-common/components/Viewer.vue

@@ -30,7 +30,6 @@ import HeadTail from 'dive-common/recipes/headtail';
 import EditorMenu from 'dive-common/components/EditorMenu.vue';
 import ConfidenceFilter from 'dive-common/components/ConfidenceFilter.vue';
 import UserGuideButton from 'dive-common/components/UserGuideButton.vue';
-import RunPipelineMenu from 'dive-common/components/RunPipelineMenu.vue';
 import DeleteControls from 'dive-common/components/DeleteControls.vue';
 import ControlsContainer from 'dive-common/components/ControlsContainer.vue';
 import Sidebar from 'dive-common/components/Sidebar.vue';
@@ -51,7 +50,6 @@ export default defineComponent({
     VideoAnnotator,
     ImageAnnotator,
     ConfidenceFilter,
-    RunPipelineMenu,
     UserGuideButton,
     EditorMenu,
   },

client/dive-common/constants.ts

@@ -8,6 +8,7 @@ const MediaTypes = {
 };
 
 const DefaultVideoFPS = 10;
+const FPSOptions = [1, 5, 10, 15, 24, 25, 30, 50, 60];
 
 const websafeVideoTypes = [
   'video/mp4',
@@ -79,6 +80,7 @@ export {
   ImageSequenceType,
   VideoType,
   MediaTypes,
+  FPSOptions,
   calibrationFileTypes,
   fileVideoTypes,
   otherImageTypes,

client/platform/desktop/backend/cli.ts

@@ -47,8 +47,8 @@ function updater(update: DesktopJobUpdate) {
 }
 
 async function parseViameFile(file: string) {
-  const tracks = await parseFile(file);
-  stdout.write(JSON.stringify(tracks));
+  const data = await parseFile(file);
+  stdout.write(JSON.stringify(data));
 }
 
 async function parseJsonFile(filepath: string, metapath: string) {

client/platform/desktop/backend/native/common.spec.ts

@@ -77,7 +77,11 @@ const urlMapper = (a: string) => `http://localhost:8888/api/media?path=${a}`;
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const updater = (update: DesktopJobUpdate) => undefined;
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-const checkMedia = async (settingsVal: Settings, file: string) => file.includes('mp4');
+const checkMedia = async (settingsVal: Settings, file: string) => ({
+  websafe: file.includes('mp4'),
+  originalFps: 30,
+  originalFpsString: '30/1',
+});
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const convertMedia = async (settingsVal: Settings, args: ConversionArgs,
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -120,6 +124,11 @@ mockfs({
       notanimage: '',
       'notanimage.txt': '',
     },
+    imageSuccessWithAnnotations: {
+      'foo.png': '',
+      'bar.png': '',
+      'file1.csv': '# comment line\n# metadata,fps: 32,"whateever"\n#comment line',
+    },
     videoSuccess: {
       'video1.avi': '',
       'video1.mp4': '',
@@ -298,6 +307,7 @@ describe('native.common', () => {
     );
     expect(data.videoUrl).toBe(`http://localhost:8888/api/media?path=${videoPath}`);
   });
+
   it('loadJsonMetadata type multi without multiCam', async () => {
     await expect(common.loadMetadata(settings, 'projectid5missingMultiCam', urlMapper))
       .rejects.toThrow('Dataset: missingMulti is of type multiCam or stereo but contains no multiCam data');
@@ -310,12 +320,15 @@ describe('native.common', () => {
       version: 1,
       type: 'image-sequence',
       fps: 100,
+      originalFps: 0,
       name: 'myproject1_name',
       createdAt: (new Date()).toString(),
       originalBasePath: '/foo/bar/baz',
       id: 'myproject1',
       originalImageFiles: [],
+      transcodedImageFiles: [],
       originalVideoFile: '',
+      transcodedVideoFile: '',
     };
     const result = await common.createKwiverRunWorkingDir(settings, [jsonMeta], 'mypipeline.pipe');
     const stat = fs.statSync(result);
@@ -333,12 +346,34 @@ describe('native.common', () => {
     expect(payload.jsonMeta.originalBasePath).toBe('/home/user/data/imageSuccess');
   });
 
+  it('import with CSV annotations', async () => {
+    const payload = await common.beginMediaImport(settings, '/home/user/data/imageSuccessWithAnnotations', checkMedia);
+    payload.jsonMeta.fps = 12; // simulate user specify FPS action
+    await common.finalizeMediaImport(settings, payload, updater, convertMedia);
+    const meta = await common.loadMetadata(settings, payload.jsonMeta.id, urlMapper);
+    expect(meta.fps).toBe(32);
+  });
+
+  it('import with user selected FPS > originalFPS', async () => {
+    const payload = await common.beginMediaImport(settings, '/home/user/data/videoSuccess/video1.mp4', checkMedia);
+    payload.jsonMeta.fps = 50; // above 30
+    await common.finalizeMediaImport(settings, payload, updater, convertMedia);
+    const meta1 = await common.loadMetadata(settings, payload.jsonMeta.id, urlMapper);
+    expect(meta1.fps).toBe(30);
+
+    payload.jsonMeta.fps = -1; // above 30
+    await common.finalizeMediaImport(settings, payload, updater, convertMedia);
+    const meta2 = await common.loadMetadata(settings, payload.jsonMeta.id, urlMapper);
+    expect(meta2.fps).toBe(1);
+  });
+
   it('importMedia video success', async () => {
     const payload = await common.beginMediaImport(settings, '/home/user/data/videoSuccess/video1.mp4', checkMedia);
     expect(payload.jsonMeta.name).toBe('video1');
     expect(payload.jsonMeta.originalImageFiles.length).toBe(0);
     expect(payload.jsonMeta.originalVideoFile).toBe('video1.mp4');
     expect(payload.jsonMeta.originalBasePath).toBe('/home/user/data/videoSuccess');
+    expect(payload.jsonMeta.fps).toBe(5); // 5 is still the default
   });
 
   it('importMedia empty json file success', async () => {

client/platform/desktop/backend/native/common.ts

@@ -18,13 +18,14 @@ import {
 } from 'dive-common/constants';
 import {
   JsonMeta, Settings, JsonMetaCurrentVersion, DesktopMetadata, DesktopJobUpdater,
-  ConvertMedia, RunTraining, ExportDatasetArgs, MediaImportPayload,
+  ConvertMedia, RunTraining, ExportDatasetArgs, MediaImportPayload, CheckMediaResults,
 } from 'platform/desktop/constants';
 import {
   cleanString, filterByGlob, makeid, strNumericCompare,
 } from 'platform/desktop/sharedUtils';
 import { Attribute, Attributes } from 'vue-media-annotator/use/useAttributes';
 import processTrackAttributes from './attributeProcessor';
+import { upgrade } from './migrations';
 import { getMultiCamUrls, transcodeMultiCam } from './multiCamUtils';
 
 const ProjectsFolderName = 'DIVE_Projects';
@@ -148,13 +149,7 @@ async function loadJsonMetadata(metaAbsPath: string): Promise<JsonMeta> {
     throw new Error(`Unable to parse ${metaAbsPath}: ${err}`);
   }
   /* check if this file meets the current schema version */
-  if ('version' in metaJson) {
-    const { version } = metaJson;
-    if (version !== JsonMetaCurrentVersion) {
-      // TODO: schema migration for older schema versions
-      throw new Error('outdated meta schema version found, migration not implemented');
-    }
-  }
+  upgrade(metaJson);
   return metaJson as JsonMeta;
 }
 
@@ -406,7 +401,7 @@ async function _saveAsJson(absPath: string, data: unknown) {
 }
 
 async function saveMetadata(settings: Settings, datasetId: string,
-  args: DatasetMetaMutable & { attributes?: Record<string, Attribute>}) {
+  args: DatasetMetaMutable & { attributes?: Record<string, Attribute> }) {
   const projectDirInfo = await getValidatedProjectDir(settings, datasetId);
   const release = await _acquireLock(projectDirInfo.basePath, projectDirInfo.metaFileAbsPath, 'meta');
   const existing = await loadJsonMetadata(projectDirInfo.metaFileAbsPath);
@@ -460,7 +455,7 @@ async function processOtherAnnotationFiles(
   datasetId: string,
   absPaths: string[],
 ): Promise<{ fps?: number; processedFiles: string[]; attributes?: Attributes }> {
-  const fps = undefined;
+  let fps: number | undefined;
   const processedFiles = []; // which files were processed to generate the detections
   let attributes: Attributes = {};
 
@@ -474,8 +469,9 @@ async function processOtherAnnotationFiles(
       // Attempt to process the file
       try {
         // eslint-disable-next-line no-await-in-loop
-        const tracks = await viameSerializers.parseFile(path);
-        const processed = processTrackAttributes(tracks);
+        const data = await viameSerializers.parseFile(path);
+        const processed = processTrackAttributes(data.tracks);
+        fps = fps || data.fps;
         attributes = processed.attributes;
         // eslint-disable-next-line no-await-in-loop
         await _saveSerialized(settings, datasetId, processed.data, true);
@@ -547,7 +543,7 @@ async function _initializeProjectDir(settings: Settings, jsonMeta: JsonMeta): Pr
 async function beginMediaImport(
   settings: Settings,
   path: string,
-  checkMedia: (settings: Settings, path: string) => Promise<boolean>,
+  checkMedia: (settings: Settings, path: string) => Promise<CheckMediaResults>,
 ): Promise<MediaImportPayload> {
   let datasetType: DatasetType;
 
@@ -568,11 +564,13 @@ async function beginMediaImport(
   const dsName = npath.parse(path).name;
   const dsId = `${cleanString(dsName).substr(0, 20)}_${makeid(10)}`;
 
+  const _defaultFps = datasetType === 'video' ? 5 : 1;
   const jsonMeta: JsonMeta = {
     version: JsonMetaCurrentVersion,
     type: datasetType,
     id: dsId,
-    fps: 5, // TODO
+    fps: _defaultFps, // adjusted below
+    originalFps: _defaultFps, // adjusted below
     originalBasePath: path,
     originalVideoFile: '',
     createdAt: (new Date()).toString(),
@@ -599,10 +597,16 @@ async function beginMediaImport(
       if (websafeImageTypes.includes(mimetype) || otherImageTypes.includes(mimetype)) {
         throw new Error('User chose image file for video import option');
       } else if (websafeVideoTypes.includes(mimetype) || otherVideoTypes.includes(mimetype)) {
-        const webSafeVideo = await checkMedia(settings, path);
-        if (!webSafeVideo || otherVideoTypes.includes(mimetype)) {
+        const checkMediaResult = await checkMedia(settings, path);
+        if (!checkMediaResult.websafe || otherVideoTypes.includes(mimetype)) {
           mediaConvertList.push(path);
         }
+        const newAnnotationFps = Math.floor(
+          // Prevent FPS smaller than 1
+          Math.max(1, Math.min(jsonMeta.fps, checkMediaResult.originalFps)),
+        );
+        jsonMeta.originalFps = checkMediaResult.originalFps;
+        jsonMeta.fps = newAnnotationFps;
       } else {
         throw new Error(`unsupported MIME type for video ${mimetype}`);
       }
@@ -652,6 +656,12 @@ async function finalizeMediaImport(
     mediaConvertList = found.mediaConvertList;
   }
 
+  // Verify that the user didn't choose an FPS value higher than originalFPS
+  // This shouldn't be possible in the UI, but we should still prevent it here.
+  jsonMeta.fps = Math.floor(
+    Math.max(1, Math.min(jsonMeta.fps, jsonMeta.originalFps)),
+  );
+
   //Now we will kick off any conversions that are necessary
   let jobBase = null;
   if (mediaConvertList.length) {

client/platform/desktop/backend/native/linux.ts

@@ -14,6 +14,7 @@ import {
   RunTraining,
   DesktopJobUpdater,
   ConversionArgs,
+  CheckMediaResults,
 } from 'platform/desktop/constants';
 import { observeChild } from 'platform/desktop/backend/native/processManager';
 import * as viame from './viame';
@@ -208,7 +209,7 @@ async function ffmpegCommand(settings: Settings) {
  * Checs the video file for the codec type and
  * returns true if it is x264, if not will return false for media conversion
  */
-async function checkMedia(settings: Settings, file: string): Promise<boolean> {
+async function checkMedia(settings: Settings, file: string): Promise<CheckMediaResults> {
   await ffmpegCommand(settings);
   return viame.checkMedia({
     ...ViameLinuxConstants,

client/platform/desktop/backend/native/migrations.ts

@@ -0,0 +1,22 @@
+/* eslint-disable no-param-reassign */
+import { JsonMeta, JsonMetaCurrentVersion } from 'platform/desktop/constants';
+
+function upgrade(meta: JsonMeta) {
+  if (meta.version === JsonMetaCurrentVersion) {
+    /* Perform soft upgrades of backward-compatible properties */
+    if (meta.originalFps === undefined) {
+      // This will be an incorrect value.
+      meta.originalFps = meta.fps;
+    }
+  } else if (meta.version < JsonMetaCurrentVersion) {
+    /* Perform major version upgrade */
+    console.error('Impossible schema', meta);
+    throw new Error('Impossible schema revision detected.  This is not a valid DIVE project metadata file.  Check the console for details.');
+  } else if (meta.version > JsonMetaCurrentVersion) {
+    throw new Error('You are trying to open a newer version of the project schema.  Please upgrade DIVE to the latest version');
+  }
+  meta.version = JsonMetaCurrentVersion;
+}
+
+// eslint-disable-next-line import/prefer-default-export
+export { upgrade };

client/platform/desktop/backend/native/multiCam.spec.ts

@@ -26,7 +26,11 @@ const settings: Settings = {
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const updater = (update: DesktopJobUpdate) => undefined;
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-const checkMedia = async (settingsVal: Settings, file: string) => file.includes('mp4');
+const checkMedia = async (settingsVal: Settings, file: string) => ({
+  websafe: file.includes('mp4'),
+  originalFpsString: '30/1',
+  originalFps: 30,
+});
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const convertMedia = async (settingsVal: Settings, args: ConversionArgs,
   // eslint-disable-next-line @typescript-eslint/no-unused-vars

client/platform/desktop/backend/native/multiCamImport.ts

@@ -1,15 +1,16 @@
 import npath from 'path';
 import fs from 'fs-extra';
 import mime from 'mime-types';
-import {
-  DatasetType, MultiCamImportFolderArgs,
-  MultiCamImportKeywordArgs, MultiCamImportArgs,
-} from 'dive-common/apispec';
+import { DatasetType } from 'dive-common/apispec';
 import {
   websafeImageTypes, websafeVideoTypes, otherImageTypes, otherVideoTypes,
 } from 'dive-common/constants';
 import {
   JsonMeta, Settings, JsonMetaCurrentVersion,
+  MultiCamImportFolderArgs,
+  MultiCamImportKeywordArgs,
+  MultiCamImportArgs,
+  CheckMediaResults,
   MediaImportPayload,
 } from 'platform/desktop/constants';
 import { cleanString, makeid } from 'platform/desktop/sharedUtils';
@@ -43,7 +44,7 @@ async function asyncForEach(array: any[], callback: Function) {
 async function beginMultiCamImport(
   settings: Settings,
   args: MultiCamImportArgs,
-  checkMedia: (settings: Settings, path: string) => Promise<boolean>,
+  checkMedia: (settings: Settings, path: string) => Promise<CheckMediaResults>,
 ): Promise<MediaImportPayload> {
   const datasetType: DatasetType | 'multi' = 'multi';
 
@@ -99,6 +100,7 @@ async function beginMultiCamImport(
     type: datasetType,
     id: dsId,
     fps: 5, // TODO
+    originalFps: 5,
     originalBasePath: mainFolder,
     originalVideoFile: '',
     createdAt: (new Date()).toString(),
@@ -128,13 +130,21 @@ async function beginMultiCamImport(
             if (websafeImageTypes.includes(mimetype) || otherImageTypes.includes(mimetype)) {
               throw new Error('User chose image file for video import option');
             } else if (websafeVideoTypes.includes(mimetype) || otherVideoTypes.includes(mimetype)) {
-              const webSafeVideo = await checkMedia(settings, video);
-              if (!webSafeVideo || otherVideoTypes.includes(mimetype)) {
+              const checkMediaResult = await checkMedia(settings, video);
+              if (!checkMediaResult.websafe || otherVideoTypes.includes(mimetype)) {
                 mediaConvertList.push(video);
               }
               if (jsonMeta.multiCam && jsonMeta.multiCam.cameras[key] !== undefined) {
                 jsonMeta.multiCam.cameras[key].originalVideoFile = npath.basename(video);
               }
+              const newAnnotationFps = Math.floor(
+                Math.min(jsonMeta.fps, checkMediaResult.originalFps),
+              );
+              if (newAnnotationFps <= 0) {
+                throw new Error('fps < 1 unsupported');
+              }
+              jsonMeta.originalFps = checkMediaResult.originalFps;
+              jsonMeta.fps = newAnnotationFps;
             } else {
               throw new Error(`unsupported MIME type for video ${mimetype}`);
             }