feat:Change plugin metadata to be exposed as object (#51)

Co-authored-by: Brendan Gibson <[email protected]>

Author: rsanteri

docs/plugin-system.md

@@ -44,7 +44,7 @@ Example of basic plugin HTML:
           });
 
           Metaflow.subscribeToMetadata((message) => {
-            console.log(`Metadata for ${resource.run_number} got updated! Metadata array for the task is ${JSON.stringify(message.data)}`);
+            console.log(`Metadata for ${resource.run_number} got updated! Metadata object for the task is ${JSON.stringify(message.data)}`);
           });
         });
 

plugin-api/Examples/react-plugin/app.jsx

@@ -19,7 +19,7 @@ function App() {
       <div>
         {metadata ? (
           <div>
-            <div>{metadata.length}</div>
+            <div>{JSON.stringify(metadata)}</div>
           </div>
         ) : 'Waiting for data'}
       </div>

plugin-api/MetaflowPluginAPI.js

@@ -1,5 +1,5 @@
 const VERSION_INFO = {
-  api: "1.0.0",
+  api: '1.1.0',
 };
 
 const Listeners = [];
@@ -15,7 +15,7 @@ const PluginInfo = {
 function messageHandler(event) {
   if (event.data && event.data.type) {
     switch (event.data.type) {
-      case "ReadyToRender": {
+      case 'ReadyToRender': {
         if (!initialised) {
           Metaflow.parameters = event.data.config;
           Metaflow.resource = event.data.resource;
@@ -27,13 +27,13 @@ function messageHandler(event) {
         }
         return;
       }
-      case "DataUpdate": {
+      case 'DataUpdate': {
         for (const listener of Listeners) {
           listener(event.data);
         }
         return;
       }
-      case "EventUpdate": {
+      case 'EventUpdate': {
         for (const listener of EventListeners) {
           listener(event.data);
         }
@@ -43,29 +43,29 @@ function messageHandler(event) {
   }
 }
 
-window.addEventListener("message", messageHandler);
+window.addEventListener('message', messageHandler);
 
 const Metaflow = {
   parameters: {},
   resource: {},
   /**
    * onReady function will be called with basic info like resource ids and custom server parameters.
-   * @param {*} onready 
+   * @param {*} onready
    */
   onReady(onready) {
     onReadyFn = onready;
     window.parent.postMessage(
       {
         name: window.name,
-        type: "PluginRegisterEvent",
+        type: 'PluginRegisterEvent',
         version: VERSION_INFO,
       },
-      "*"
+      '*',
     );
   },
   /**
    * Alias for onReady
-   * @param {*} _settings Deprecated settings for register function 
+   * @param {*} _settings Deprecated settings for register function
    * @param {*} onready   Callback to startup plugin.
    */
   register(_settings, onready) {
@@ -79,21 +79,11 @@ const Metaflow = {
    */
   setHeight(fixedHeight) {
     if (fixedHeight) {
-      window.parent.postMessage(
-        { name: window.name, type: "PluginHeightCheck", height: fixedHeight },
-        "*"
-      );
+      window.parent.postMessage({ name: window.name, type: 'PluginHeightCheck', height: fixedHeight }, '*');
     } else {
       const body = document.body;
-      const height = Math.max(
-        body.scrollHeight,
-        body.offsetHeight,
-        body.clientHeight
-      );
-      window.parent.postMessage(
-        { name: window.name, type: "PluginHeightCheck", height: height },
-        "*"
-      );
+      const height = Math.max(body.scrollHeight, body.offsetHeight, body.clientHeight);
+      window.parent.postMessage({ name: window.name, type: 'PluginHeightCheck', height: height }, '*');
     }
   },
   /**
@@ -103,10 +93,7 @@ const Metaflow = {
    */
   subscribe(paths, fn) {
     Listeners.push(fn);
-    window.parent.postMessage(
-      { name: window.name, type: "PluginSubscribeToData", paths: paths },
-      "*"
-    );
+    window.parent.postMessage({ name: window.name, type: 'PluginSubscribeToData', paths: paths }, '*');
   },
   /**
    * Subscribe to events
@@ -115,21 +102,15 @@ const Metaflow = {
    */
   on(events, fn) {
     EventListeners.push(fn);
-    window.parent.postMessage(
-      { name: window.name, type: "PluginSubscribeToEvent", events: events },
-      "*"
-    );
+    window.parent.postMessage({ name: window.name, type: 'PluginSubscribeToEvent', events: events }, '*');
   },
   /**
    * Call event with any name and payload. Other plugins or systems in app might subscribe to these events.
    * @param {string} event
    * @param {*} data
    */
   call(event, data) {
-    window.parent.postMessage(
-      { name: window.name, type: "PluginCallEvent", event: event, data: data },
-      "*"
-    );
+    window.parent.postMessage({ name: window.name, type: 'PluginCallEvent', event: event, data: data }, '*');
   },
   /**
    * Send notification on main application
@@ -139,11 +120,11 @@ const Metaflow = {
     window.parent.postMessage(
       {
         name: window.name,
-        type: "PluginCallEvent",
-        event: "SEND_NOTIFICATION",
+        type: 'PluginCallEvent',
+        event: 'SEND_NOTIFICATION',
         data: message,
       },
-      "*"
+      '*',
     );
   },
   /**
@@ -154,40 +135,37 @@ const Metaflow = {
     window.parent.postMessage(
       {
         name: window.name,
-        type: "PluginCallEvent",
-        event: "UPDATE_PLUGIN",
+        type: 'PluginCallEvent',
+        event: 'UPDATE_PLUGIN',
         data: {
           slot: PluginInfo.slot,
           name: PluginInfo.manifest.name,
           visible: visible,
         },
       },
-      "*"
+      '*',
     );
   },
   //
   // Request to be removed?
   //
   remove(fn) {
-    window.parent.postMessage(
-      { name: window.name, type: "PluginRemoveRequest" },
-      "*"
-    );
+    window.parent.postMessage({ name: window.name, type: 'PluginRemoveRequest' }, '*');
   },
 
   subscribeToMetadata(fn) {
-    this.subscribe(["metadata"], fn);
+    this.subscribe(['metadata'], (event) => fn(event.data));
   },
 
   subscribeToRunMetadata(fn) {
-    this.subscribe(["run-metadata"], fn);
+    this.subscribe(['run-metadata'], (event) => fn(event.data));
   },
 };
 
-if (typeof exports !== "undefined") {
+if (typeof exports !== 'undefined') {
   exports.Metaflow = Metaflow;
 }
 
-if (typeof window !== "undefined") {
+if (typeof window !== 'undefined') {
   window.Metaflow = Metaflow;
 }

plugin-api/README.md

@@ -10,33 +10,14 @@ Plugins will use JS API to communicate with the host application. Plugins must c
 | -- | -- | -- |
 | onReady                | (callback: (configuration, resource) => void) => void                         | Register callback function to be run when host and plugin iframe are ready. |
 | subscribe              | (paths: string[], cb: (message: { path: string, data: any }) => void) => void  | Subscribe to contextual data updates from application. Possible paths: 'metadata' or 'run-metadata' |
-| subscribeToMetadata    | (callback: (message: { type: string, data: Metadata[] }) => void) => void     | Subscribe to task metadata |
-| subscribeToRunMetadata | (callback: (message: { type: string, data: Metadata[] }) => void) => void     | Subscribe to run metadata |
+| subscribeToMetadata    | (callback: (message: Record<string,string>) => void) => void     | Subscribe to task metadata |
+| subscribeToRunMetadata | (callback: (message: Record<string,string>) => void) => void     | Subscribe to run metadata |
 | on                     | (events: string[], cb: (message: { type: string, data: any }) => void) => void | Subscribe to any event by event string. |
 | call                   | (event: string, data: any) => void                                             | Call any custom event with string and any data. |
 | sendNotification       | (message: string \| { type: string, message: string }) => void                 | Call notification API from host application. |
 | setHeight              | (height: number \| undefined) => void                                          | Update height of iframe container for plugin. |
 | setVisibility          | (visibility: boolean) => void                                                  | Update visibility of the plugin. Note that if will stay in iframe even if visibility is set false. |
 
-Type of metadata is 
-
-```typescript
-type Metadata = {
-  flow_id: string;
-  run_number: string;
-  step_name: string;
-  task_id: string;
-  user_name: string;
-  ts_epoch: number;
-  tags?: string[];
-  system_tags: string[];
-  id: number;
-  field_name: string;
-  value: string;
-  type: string;
-}
-```
-
 ## How to
 
 Here are three different ways to use the plugin API file.

plugin-api/dev/index.plugin-dev.tsx

@@ -28,22 +28,10 @@ const PLUGIN_DEFINITIONS = [
 
 // Mock data as you need for your plugin. This data will be available in plugin with Metaflow.subscribe(['key-here'], callback)
 const DATA = {
-  metadata: [
-    {
-      flow_id: 'PluginDevFlow',
-      user_name: 'dev-user',
-      ts_epoch: 1634036924920,
-      tags: [],
-      system_tags: ['metaflow-version:1'],
-      id: 1,
-      run_number: 1,
-      step_name: 'start',
-      task_id: '123',
-      field_name: 'datafield',
-      value: 'value',
-      type: 'aws-batch-job-id',
-    },
-  ],
+  metadata: {
+    field: 'value',
+    is_plugin_test: 'true',
+  },
 };
 
 describe('Testing', () => {

plugin-api/dev/plugins/dev-plugin/index.html

@@ -37,8 +37,8 @@
 
     <p>
       Plugin contains at least one html file and manifest.json. HTML file contains everything plugin needs to render and
-      manifest file contains configurations for the plugin. In development tooling basic information is mocked in
-      test file instead.
+      manifest file contains configurations for the plugin. In development tooling basic information is mocked in test
+      file instead.
     </p>
 
     <p>
@@ -54,7 +54,10 @@
     <p><b>Recommended workflow</b></p>
 
     <ol>
-      <li>Create new folder to <code>plugin-api/dev/plugins</code> and initialise it as repository to your version control.</li>
+      <li>
+        Create new folder to <code>plugin-api/dev/plugins</code> and initialise it as repository to your version
+        control.
+      </li>
       <li>Add <code>index.html</code> and <code>manifest.json</code></li>
       <li>
         Change <code>PLUGIN_DEFINITIONS[0].name from plugin-api/dev/index.plugin-dev.tsx</code> to be your folder name.
@@ -82,9 +85,8 @@
           document.getElementById('init').innerHTML = 'Plugin is up and ready!';
           document.getElementById('resource').innerHTML = 'Resource Info: ' + JSON.stringify(resource);
 
-
           Metaflow.subscribeToMetadata((message) => {
-            document.getElementById('metadata').innerHTML = 'Metadata: ' + JSON.stringify(message.data, 0, 2);
+            document.getElementById('metadata').innerHTML = 'Metadata: ' + JSON.stringify(message, 0, 2);
             Metaflow.setHeight();
           });
         });

src/components/Plugins/PluginManager.tsx

@@ -47,7 +47,7 @@ type PluginVersionInfo = {
 export type AllowedSlot = 'task-details' | 'run-header';
 
 const SUPPORTED_PLUGIN_API_VERSION = '0.13.0';
-const RECOMMENDED_PLUGIN_API_VERSION = '1.0.0';
+const RECOMMENDED_PLUGIN_API_VERSION = '1.1.0';
 const ALLOWED_SLOTS: AllowedSlot[] = ['task-details', 'run-header'];
 
 //

src/pages/Run/RunPage.tsx

@@ -22,6 +22,7 @@ import { FixedContent } from '../../components/Structure';
 import useTaskListSettings from '../../components/Timeline/useTaskListSettings';
 import useResource from '../../hooks/useResource';
 import { PluginsContext } from '../../components/Plugins/PluginManager';
+import { metadataToRecord } from '../../utils/metadata';
 import { GraphModel } from '../../components/DAG/DAGUtils';
 
 //
@@ -82,7 +83,7 @@ const RunPage: React.FC<RunPageProps> = ({ run, params }) => {
       step_name: '_parameters',
     },
     onUpdate(items) {
-      plContext.addDataToStore('run-metadata', items);
+      plContext.addDataToStore('run-metadata', metadataToRecord(items));
     },
   });
 
@@ -250,7 +251,7 @@ const RunPage: React.FC<RunPageProps> = ({ run, params }) => {
                   )}
                 </ErrorBoundary>
               ),
-            }
+            },
           ]}
         />
       </RunPageContainer>

src/pages/Task/useTaskMetadata.ts

@@ -2,6 +2,7 @@ import { useContext, useEffect } from 'react';
 import { PluginsContext } from '../../components/Plugins/PluginManager';
 import useResource, { Resource } from '../../hooks/useResource';
 import { Metadata } from '../../types';
+import { metadataToRecord } from '../../utils/metadata';
 
 type TaskMetadataConfig = {
   url: string;
@@ -55,11 +56,11 @@ function useTaskMetadata({ url, attemptId, paused }: TaskMetadataConfig): TaskMe
 
   // Update metadata to plugin store
   useEffect(() => {
-    addDataToStore('metadata', data);
+    addDataToStore('metadata', metadataToRecord(data));
   }, [dataCount]); //eslint-disable-line
 
   useEffect(() => {
-    return () => addDataToStore('metadata', []);
+    return () => addDataToStore('metadata', {});
   }, [url, attemptId]); //eslint-disable-line
 
   return { data, taskMetadataResource, attemptMetadataResource };

src/utils/__tests__/metadata.test.cypress.ts

@@ -0,0 +1,16 @@
+import { metadataToRecord } from '../metadata';
+import { createMetadata } from '../testhelper';
+
+describe('Metadata utils', () => {
+  it('metadataToRecord()', () => {
+    expect(metadataToRecord([])).to.eql({});
+    expect(metadataToRecord([createMetadata({ field_name: 'test', value: '123' })])).to.eql({ test: '123' });
+    expect(
+      metadataToRecord([
+        createMetadata({ field_name: 'test', value: '123' }),
+        createMetadata({ field_name: 'second', value: 'row' }),
+        createMetadata({ field_name: 'third', value: 'row' }),
+      ]),
+    ).to.eql({ test: '123', second: 'row', third: 'row' });
+  });
+});

src/utils/metadata.ts

@@ -0,0 +1,7 @@
+import { Metadata } from '../types';
+
+export function metadataToRecord(data: Metadata[]): Record<string, string> {
+  return data.reduce((obj, metadata) => {
+    return { ...obj, [metadata.field_name]: metadata.value };
+  }, {});
+}