Merge branch 'master' into feature/historical-conditions

Author: jvigliotta

e2e/README.md

@@ -105,11 +105,13 @@ For those interested in the mechanics of snapshot testing with Playwright, you c
 
 - Our Snapshot tests receive a `@snapshot` tag.
 - Snapshots need to be executed within the official Playwright container to ensure we're using the exact rendering platform in CI and locally. To do a valid comparison locally:
+- Note, microsoft might have retired older images like `-jammy`. Run the following command to find the available tags:
+`curl -s https://mcr.microsoft.com/v2/playwright/tags/list | jq -r '.tags[]' | grep "v{X.X.X}`
 
 ```sh
 // Replace {X.X.X} with the current Playwright version 
 // from our package.json configuration file
-docker run --rm --network host -v $(pwd):/work/ -w /work/ -it mcr.microsoft.com/playwright:v{X.X.X}-focal /bin/bash
+docker run --rm --network host -v $(pwd):/work/ -w /work/ -it mcr.microsoft.com/playwright:v{X.X.X}-jammy /bin/bash
 npm install
 npm run test:e2e:checksnapshots
 ```
@@ -123,7 +125,7 @@ To compare a snapshot, run a test and open the html report with the 'Expected' v
 ```sh
 // Replace {X.X.X} with the current Playwright version 
 // from our package.json configuration file
-docker run --rm --network host -v $(pwd):/work/ -w /work/ -it mcr.microsoft.com/playwright:v{X.X.X}-focal /bin/bash
+docker run --rm --network host -v $(pwd):/work/ -w /work/ -it mcr.microsoft.com/playwright:v{X.X.X}-jammy /bin/bash
 npm install
 npm run test:e2e:updatesnapshots
 ```

e2e/appActions.js

@@ -285,6 +285,36 @@ async function createStableStateTelemetry(page, parent = 'mine') {
   };
 }
 
+/**
+ * Create a Out of order State Telemetry Object (State Generator) for use in visual tests
+ * and tests against plotting telemetry (e.g. logPlot tests). This will change state every 2 seconds.
+ * @param {import('@playwright/test').Page} page
+ * @param {string | import('../src/api/objects/ObjectAPI').Identifier} [parent] the uuid or identifier of the parent object. Defaults to 'mine'
+ * @returns {Promise<CreatedObjectInfo>} An object containing information about the telemetry object.
+ */
+async function createOutOfOrderStateTelemetry(page, parent = 'mine', duration = 0.25) {
+  const parentUrl = await getHashUrlToDomainObject(page, parent);
+
+  await page.goto(`${parentUrl}`);
+  const createdObject = await createDomainObjectWithDefaults(
+    page,
+    {
+      type: 'State Generator',
+      name: 'Stable State Generator'
+    },
+    { outOfOrder: true, duration: duration.toString() }
+  );
+  // Wait until the URL is updated
+  const uuid = await getFocusedObjectUuid(page);
+  const url = await getHashUrlToDomainObject(page, uuid);
+
+  return {
+    name: createdObject.name,
+    uuid,
+    url
+  };
+}
+
 /**
  * Navigates directly to a given object url, in fixed time mode, with the given start and end bounds. Note: does not set
  * default view type.
@@ -782,6 +812,7 @@ export {
   createDomainObjectWithDefaults,
   createExampleTelemetryObject,
   createNotification,
+  createOutOfOrderStateTelemetry,
   createPlanFromJSON,
   createStableStateTelemetry,
   expandEntireTree,

e2e/tests/functional/plugins/plot/plotRendering.e2e.spec.js

@@ -27,9 +27,11 @@
 
 import {
   createDomainObjectWithDefaults,
+  createOutOfOrderStateTelemetry,
   getCanvasPixels,
   setRealTimeMode
 } from '../../../../appActions.js';
+import { VISUAL_REALTIME_URL } from '../../../../constants.js';
 import { expect, test } from '../../../../pluginFixtures.js';
 
 test.describe('Plot Rendering', () => {
@@ -95,6 +97,28 @@ test.describe('Plot Rendering', () => {
   });
 });
 
+test.describe.skip('Plot rendering with out of order data', () => {
+  let telemetry;
+
+  test.beforeEach(async ({ page }) => {
+    await page.goto(VISUAL_REALTIME_URL, { waitUntil: 'domcontentloaded' });
+
+    telemetry = await createOutOfOrderStateTelemetry(page);
+  });
+
+  test('Out of Order Plot Paused', async ({ page, theme }) => {
+    await page.goto(telemetry.url, { waitUntil: 'domcontentloaded' });
+
+    // hover over plot for plot controls
+    await page.getByLabel('Plot Canvas').hover();
+    // click on pause control
+    await page.getByTitle('Pause incoming real-time data').click();
+
+    // there should be no out of order data in the plot. This is verified by checking that the out of order y-axis label is not present in the plot. If the out of order data is present, the y-axis label will be present in the plot.
+    await expect(page.getByText('OUT OF ORDER', { exact: true })).toHaveCount(0);
+  });
+});
+
 /**
  * This function edits a sine wave generator with the default options and enables the infinity values option.
  *

example/generator/GeneratorMetadataProvider.js

@@ -106,6 +106,10 @@ const METADATA_BY_TYPE = {
           {
             value: 1,
             string: 'ON'
+          },
+          {
+            value: 99,
+            string: 'OUT OF ORDER'
           }
         ],
         filters: [

example/generator/StateGeneratorProvider.js

@@ -35,10 +35,17 @@ export default class StateGeneratorProvider {
 
   subscribe(domainObject, callback, options) {
     const duration = domainObject.telemetry.duration * 1000;
-
+    let tick = 0;
     const interval = setInterval(() => {
-      const now = this.openmct.time.now() || Date.now();
-      const datum = this.#pointForTimestamp(now, duration, domainObject.name);
+      tick += 1;
+      let now = this.openmct.time.now() || Date.now();
+      let flip = false;
+      if (domainObject.telemetry.outOfOrder && tick % 3 === 0) {
+        // 2 steps forward, 1 step back by duration * 2 to simulate out of order data
+        now -= duration * 1.5;
+        flip = true;
+      }
+      const datum = this.#pointForTimestamp(now, duration, domainObject.name, flip);
 
       if (!this.#shouldBeFiltered(datum, options)) {
         datum.value = String(datum.value);
@@ -73,12 +80,15 @@ export default class StateGeneratorProvider {
     return Promise.resolve(data);
   }
 
-  #pointForTimestamp(timestamp, duration, name) {
+  #pointForTimestamp(timestamp, duration, name, flip = false) {
     const key = this.openmct.time.getTimeSystem()?.key || 'utc';
     const point = {
       name: name,
       value: Math.floor(timestamp / duration) % 2
     };
+    if (flip) {
+      point.value = 99;
+    }
     point[key] = Math.floor(timestamp / duration) * duration;
     return point;
   }

example/generator/plugin.js

@@ -41,11 +41,20 @@ export default function (openmct) {
         key: 'duration',
         required: true,
         property: ['telemetry', 'duration']
+      },
+      {
+        name: 'Out of order data',
+        control: 'toggleSwitch',
+        cssClass: 'l-input',
+        key: 'outOfOrder',
+        required: true,
+        property: ['telemetry', 'outOfOrder']
       }
     ],
     initialize: function (object) {
       object.telemetry = {
-        duration: 5
+        duration: 5,
+        outOfOrder: false
       };
     }
   });

src/plugins/plot/configuration/PlotSeries.js

@@ -214,7 +214,8 @@ export default class PlotSeries extends Model {
       this.unsubscribe = this.openmct.telemetry.subscribe(
         this.domainObject,
         (data) => {
-          this.addAll(data, true);
+          // We cannot assume that the incoming data is chronologically sound, so sorted = false
+          this.addAll(_(data).sortBy(this.getXVal).value(), false);
         },
         {
           filters: this.filters,
@@ -373,6 +374,7 @@ export default class PlotSeries extends Model {
    * @private
    */
   sortedIndex(point) {
+    // lodash's sortedIndexBy uses binary search, so it should be quite efficient for most cases.
     return _.sortedIndexBy(this.getSeriesData(), point, this.getXVal);
   }
   /**
@@ -436,26 +438,34 @@ export default class PlotSeries extends Model {
     let insertIndex = data.length;
     const currentYVal = this.getYVal(newData);
     const lastYVal = this.getYVal(data[insertIndex - 1]);
+    const currentXVal = this.getXVal(newData);
 
     if (this.isValueInvalid(currentYVal) && this.isValueInvalid(lastYVal)) {
       console.warn(`[Plot] Invalid Y Values detected: ${currentYVal} ${lastYVal}`);
 
       return;
     }
 
-    if (!sorted) {
+    // if the first new data point has an X value > the last data point we already have,  stick it at the end.
+    const isDataInThePast = currentXVal <= this.getXVal(data[insertIndex - 1]);
+    if (!sorted && isDataInThePast) {
       insertIndex = this.sortedIndex(newData);
-      if (this.getXVal(data[insertIndex]) === this.getXVal(newData)) {
+      if (this.getXVal(data[insertIndex]) === currentXVal) {
         return;
       }
 
-      if (this.getXVal(data[insertIndex - 1]) === this.getXVal(newData)) {
+      if (this.getXVal(data[insertIndex - 1]) === currentXVal) {
         return;
       }
     }
 
     this.updateStats(newData);
     newData.mctLimitState = this.evaluate(newData);
+    // Note: Splicing is a performance bottleneck for large data sets when the insert index
+    // is NOT at the end of the array, this is because inserting into the middle of an array requires
+    // shifting all subsequent elements. For now, leave it as is since for the
+    // majority of cases real-time data will be appended to the end of the array,
+    // and historical data will be added in sorted order.
     data.splice(insertIndex, 0, newData);
     this.updateSeriesData(data);
     this.emit('add', newData, insertIndex, this);