Merge branch 'fix-event-sync' of https://github.com/ngageoint/mage-server into fix-observation-sync

ryanslatten · ryanslatten · commit 25416d81d080 · 2025-01-15T01:00:28.000-05:00
diff --git a/plugins/arcgis/service/src/ArcGISConfig.ts b/plugins/arcgis/service/src/ArcGISConfig.ts
@@ -1,3 +1,5 @@
+import { MageEventId } from "@ngageoint/mage.service/lib/entities/events/entities.events"
+
 /**
  * Contains an arc feature service url and layers.
  */
@@ -9,8 +11,8 @@ export interface FeatureServiceConfig {
     url: string
 
     /**
-     * Serialized ArcGISIdentityManager
-     */
+    * Serialized ArcGISIdentityManager
+    */
     identityManager: string
 
     /**
@@ -35,9 +37,9 @@ export interface FeatureLayerConfig {
     geometryType?: string
 
     /**
-     * The event ids or names that sync to this arc feature layer.
+     * The event ids that sync to this arc feature layer.
      */
-    events?: (number|string)[]
+    eventIds?: MageEventId[]
 }
 
 
diff --git a/plugins/arcgis/service/src/EventDeletionHandler.ts b/plugins/arcgis/service/src/EventDeletionHandler.ts
@@ -34,6 +34,10 @@ export class EventDeletionHandler {
         this._config = config;
     }
 
+    public updateConfig(newConfig: ArcGISPluginConfig): void {
+        this._config = newConfig;
+    }
+
     /**
      * 
      * @param activeEvents The current set of active events.
@@ -83,7 +87,7 @@ export class EventDeletionHandler {
     }
 
     /**
-     * Called when the query is finished.  It goes through the results and gathers all even Ids currently stored
+     * Called when the query is finished.  It goes through the results and gathers all event Ids currently stored
      * in the arc layer.  It then will remove any events from the arc layer that do not exist.
      * @param layerProcessor The feature layer processor.
      * @param result The returned results.
diff --git a/plugins/arcgis/service/src/EventLayerProcessorOrganizer.ts b/plugins/arcgis/service/src/EventLayerProcessorOrganizer.ts
@@ -19,7 +19,7 @@ export class EventLayerProcessorOrganizer {
         for (const event of events) {
             let syncProcessors = new Array<FeatureLayerProcessor>();
             for (const layerProcessor of layerProcessors) {
-                if (layerProcessor.layerInfo.hasEvent(event.name)) {
+                if (layerProcessor.layerInfo.hasEvent(event.id)) {
                     syncProcessors.push(layerProcessor);
                 }
             }
diff --git a/plugins/arcgis/service/src/FeatureServiceAdmin.ts b/plugins/arcgis/service/src/FeatureServiceAdmin.ts
@@ -1,6 +1,6 @@
 import { ArcGISPluginConfig } from "./ArcGISPluginConfig"
 import { FeatureServiceConfig, FeatureLayerConfig } from "./ArcGISConfig"
-import { MageEvent, MageEventRepository } from '@ngageoint/mage.service/lib/entities/events/entities.events'
+import { MageEvent, MageEventId, MageEventRepository } from '@ngageoint/mage.service/lib/entities/events/entities.events'
 import { Layer, Field } from "./AddLayersRequest"
 import { Form, FormField, FormFieldType, FormId } from '@ngageoint/mage.service/lib/entities/events/entities.events.forms'
 import { ObservationsTransformer } from "./ObservationsTransformer"
@@ -120,29 +120,29 @@ export class FeatureServiceAdmin {
 	}
 
 	/**
-	 * Get the layer events
+	 * Get the Mage layer events
 	 * @param layer feature layer
 	 * @param eventRepo event repository
-	 * @returns layer events
+	 * @returns Mage layer events
 	 */
 	private async layerEvents(layer: FeatureLayerConfig, eventRepo: MageEventRepository): Promise<MageEvent[]> {
-		const layerEvents: Set<number|string> = new Set()
-		if (layer.events != null) {
-			for (const layerEvent of layer.events) {
-				layerEvents.add(layerEvent)
+		const layerEventIds: Set<MageEventId> = new Set()
+		 if (layer.eventIds != null) {
+			for (const layerEventId of layer.eventIds) {
+				layerEventIds.add(layerEventId)
 			}
 		}
 
 		let mageEvents
-		if (layerEvents.size > 0) {
+		if (layerEventIds.size > 0) {
 			mageEvents = await eventRepo.findAll()
 		} else {
 			mageEvents = await eventRepo.findActiveEvents()
 		}
 
 		const events: MageEvent[] = []
 		for (const mageEvent of mageEvents) {
-			if (layerEvents.size == 0 || layerEvents.has(mageEvent.name) || layerEvents.has(mageEvent.id)) {
+			if (layerEventIds.size == 0 || layerEventIds.has(mageEvent.id)) {
 				const event = await eventRepo.findById(mageEvent.id)
 				if (event != null) {
 					events.push(event)
diff --git a/plugins/arcgis/service/src/LayerInfo.ts b/plugins/arcgis/service/src/LayerInfo.ts
@@ -1,3 +1,4 @@
+import { MageEventId } from "@ngageoint/mage.service/lib/entities/events/entities.events";
 import { LayerInfoResult, LayerField } from "./LayerInfoResult";
 
 /**
@@ -28,7 +29,7 @@ export class LayerInfo {
     /**
      * The events that are synching to this layer.
      */
-    events: Set<string> = new Set<string>()
+    events: Set<MageEventId> = new Set<MageEventId>()
 
     /**
      * Constructor.
@@ -37,12 +38,10 @@ export class LayerInfo {
      * @param layerInfo The layer info.
      * @param token The access token.
      */
-    constructor(url: string, events: string[], layerInfo: LayerInfoResult) {
+    constructor(url: string, events: MageEventId[], layerInfo: LayerInfoResult) {
         this.url = url
-        if (events != undefined && events != null && events.length == 0) {
-            this.events.add('nothing to sync')
-        }
-        if (events != undefined || events != null) {
+    
+        if (events && events.length > 0) {
             for (const event of events) {
                 this.events.add(event);
             }
@@ -69,11 +68,11 @@ export class LayerInfo {
 
     /**
      * Determine if the layer is enabled for the event.
-     * @param event The event.
+     * @param eventId The event.
      * @return true if enabled
      */
-    hasEvent(event: string) {
-        return this.events.size == 0 || this.events.has(event)
+    hasEvent(eventId: MageEventId) {
+        return this.events.size == 0 || this.events.has(eventId)
     }
 
 }
diff --git a/plugins/arcgis/service/src/ObservationProcessor.ts b/plugins/arcgis/service/src/ObservationProcessor.ts
@@ -1,4 +1,5 @@
 import { PagingParameters } from '@ngageoint/mage.service/lib/entities/entities.global';
+import { MageEventId } from "@ngageoint/mage.service/lib/entities/events/entities.events";
 import { MageEventRepository } from '@ngageoint/mage.service/lib/entities/events/entities.events';
 import { EventScopedObservationRepository, ObservationRepositoryForEvent } from '@ngageoint/mage.service/lib/entities/observations/entities.observations';
 import { UserRepository } from '@ngageoint/mage.service/lib/entities/users/entities.users';
@@ -164,19 +165,23 @@ export class ObservationProcessor {
 	private async updateConfig(): Promise<ArcGISPluginConfig> {
 		const config = await this.safeGetConfig()
 		
-		// Include form definitions while detecting changes in config
-		const eventForms = await this._eventRepo.findAll();
+		// Include configured eventform definitions while detecting changes in config
+		const eventIds = config.featureServices
+    .flatMap(service => service.layers)
+    .flatMap(layer => layer.eventIds)
+    .filter((eventId): eventId is MageEventId => typeof eventId === 'number');
+
+		const eventForms = await this._eventRepo.findAllByIds(eventIds);
 		const fullConfig = { ...config, eventForms };
 
 		const configJson = JSON.stringify(fullConfig)
 		if (this._previousConfig == null || this._previousConfig != configJson) {
 			this._transformer = new ObservationsTransformer(config, console);
 			this._geometryChangeHandler = new GeometryChangedHandler(this._transformer);
-			this._eventDeletionHandler = new EventDeletionHandler(this._console, config);
+			this._eventDeletionHandler.updateConfig(config);			
 			this._layerProcessors = [];
 			await this.getFeatureServiceLayers(config);
 			this._previousConfig = configJson
-			this._firstRun = true;
 		}
 		return config
 	}
@@ -186,7 +191,6 @@ export class ObservationProcessor {
 	 */
 	async start() {
 		this._isRunning = true;
-		this._firstRun = true;
 		this.processAndScheduleNext();
 	}
 
@@ -207,7 +211,7 @@ export class ObservationProcessor {
 			try {
 				const identityManager = await this._identityService.signin(service)
 				const response = await request(service.url, { authentication: identityManager })
-				this.handleFeatureService(response, service, config)
+				await this.handleFeatureService(response, service, config)
 			} catch (err) {
 				console.error(err)
 			}
@@ -235,25 +239,7 @@ export class ObservationProcessor {
 			}
 
 			for (const featureLayer of featureServiceConfig.layers) {
-				const eventNames: string[] = []
-				const events = featureLayer.events
-				if (events != null) {
-					for (const event of events) {
-						const eventId = Number(event);
-						if (isNaN(eventId)) {
-							eventNames.push(String(event));
-						} else {
-							const mageEvent = await this._eventRepo.findById(eventId)
-							if (mageEvent != null) {
-								eventNames.push(mageEvent.name);
-							}
-						}
-					}
-				}
-				if (eventNames.length > 0) {
-					featureLayer.events = eventNames
-				}
-
+				// TODO - this used to convert event ids to names and set back on featureLayer.events. What is impact of not doing? 			
 				const layer = serviceLayers.get(featureLayer.layer)
 
 				let layerId = undefined
@@ -270,7 +256,7 @@ export class ObservationProcessor {
 					const featureService = new FeatureService(console, featureServiceConfig, identityManager)
 					const layerInfo = await featureService.queryLayerInfo(layerId);
 					const url = `${featureServiceConfig.url}/${layerId}`;
-					this.handleLayerInfo(url, featureServiceConfig, featureLayer, layerInfo, config);
+					await this.handleLayerInfo(url, featureServiceConfig, featureLayer, layerInfo, config);
 				}
 			}
 		}
@@ -286,10 +272,10 @@ export class ObservationProcessor {
 	 */
 	private async handleLayerInfo(url: string, featureServiceConfig: FeatureServiceConfig, featureLayer: FeatureLayerConfig, layerInfo: LayerInfoResult, config: ArcGISPluginConfig) {
 		if (layerInfo.geometryType != null) {
-			const events = featureLayer.events as string[]
 			const admin = new FeatureServiceAdmin(config, this._identityService, this._console)
+			const eventIds = featureLayer.eventIds || []
 			await admin.updateLayer(featureServiceConfig, featureLayer, layerInfo, this._eventRepo)
-			const info = new LayerInfo(url, events, layerInfo)
+			const info = new LayerInfo(url, eventIds, layerInfo)
 			const identityManager = await this._identityService.signin(featureServiceConfig)
 			const layerProcessor = new FeatureLayerProcessor(info, config, identityManager, this._console);
 			this._layerProcessors.push(layerProcessor);
@@ -310,9 +296,13 @@ export class ObservationProcessor {
 					layerProcessor.processPendingUpdates();
 				}
 				this._console.info('ArcGIS plugin processing new observations...');
-				const activeEvents = await this._eventRepo.findActiveEvents();
-				this._eventDeletionHandler.checkForEventDeletion(activeEvents, this._layerProcessors, this._firstRun);
-				const eventsToProcessors = this._organizer.organize(activeEvents, this._layerProcessors);
+				const enabledEvents = (await this._eventRepo.findActiveEvents()).filter(event =>
+					this._layerProcessors.some(layerProcessor =>
+						layerProcessor.layerInfo.hasEvent(event.id)
+					)
+				);
+				this._eventDeletionHandler.checkForEventDeletion(enabledEvents, this._layerProcessors, this._firstRun);
+				const eventsToProcessors = this._organizer.organize(enabledEvents, this._layerProcessors);
 				const nextQueryTime = Date.now();
 				for (const pair of eventsToProcessors) {
 					this._console.info('ArcGIS getting newest observations for event ' + pair.event.name);
diff --git a/plugins/arcgis/service/src/index.ts b/plugins/arcgis/service/src/index.ts
@@ -4,9 +4,10 @@ import { ObservationRepositoryToken } from '@ngageoint/mage.service/lib/plugins.
 import { MageEventRepositoryToken } from '@ngageoint/mage.service/lib/plugins.api/plugins.api.events'
 import { UserRepositoryToken } from '@ngageoint/mage.service/lib/plugins.api/plugins.api.users'
 import { SettingPermission } from '@ngageoint/mage.service/lib/entities/authorization/entities.permissions'
+import { MageEventId } from '@ngageoint/mage.service/lib/entities/events/entities.events'
 import { ObservationProcessor } from './ObservationProcessor'
 import { ArcGISIdentityManager, request } from "@esri/arcgis-rest-request"
-import { FeatureServiceConfig } from './ArcGISConfig'
+import { FeatureServiceConfig, FeatureLayerConfig } from './ArcGISConfig'
 import { URL } from "node:url"
 import express from 'express'
 import { ArcGISIdentityService, createArcGISIdentityService, getPortalUrl } from './ArcGISService'
@@ -35,12 +36,12 @@ const InjectedServices = {
 
 const pluginWebRoute = "plugins/@ngageoint/mage.arcgis.service"
 
-const sanitizeFeatureService = async (config: FeatureServiceConfig, identityService: ArcGISIdentityService): Promise<Omit<FeatureServiceConfig & { authenticated: boolean }, 'identityManager'>>  => {
+const sanitizeFeatureService = async (config: FeatureServiceConfig, identityService: ArcGISIdentityService): Promise<Omit<FeatureServiceConfig & { authenticated: boolean }, 'identityManager'>> => {
   let authenticated = false
   try {
     await identityService.signin(config)
     authenticated = true
-  } catch(ignore) {}
+  } catch (ignore) { }
 
   const { identityManager, ...sanitized } = config;
   return { ...sanitized, authenticated }
@@ -132,7 +133,7 @@ const arcgisPluginHooks: InitPluginHook<typeof InjectedServices> = {
                   </head>
                 </html>
               `);
-              }).catch((error) => res.status(400).json(error))
+            }).catch((error) => res.status(400).json(error))
           })
 
           return routes
@@ -166,18 +167,51 @@ const arcgisPluginHooks: InitPluginHook<typeof InjectedServices> = {
               const config = await stateRepo.get()
               const { featureServices: updatedServices, ...updateConfig } = req.body
 
-              // Map exisiting identityManager, client does not send this
-              const featureServices: FeatureServiceConfig[] = updatedServices.map((updateService: FeatureServiceConfig) => {
-                const existingService = config.featureServices.find((featureService: FeatureServiceConfig) => featureService.url === updateService.url)
+
+              // Convert event names to event IDs
+              // Fetch all events and create a mapping of event names to event IDs
+              const allEvents = await eventRepo.findAll();
+              const eventNameToIdMap = new Map<string, MageEventId>();
+              allEvents.forEach(event => {
+                eventNameToIdMap.set(event.name, event.id);
+              });
+
+              // Process the incoming feature services with eventIds instead of event names
+              const featureServices: FeatureServiceConfig[] = updatedServices.map((updateService: any) => {
+                const existingService = config.featureServices.find(
+                  (featureService: FeatureServiceConfig) => featureService.url === updateService.url
+                );
+
+                // Process layers
+                const layers: FeatureLayerConfig[] = updateService.layers.map((layer: any) => {
+                  // Extract event names from the incoming layer data
+                  const eventNames: string[] = layer.events || [];
+
+                  // Convert event names to event IDs using the mapping
+                  const eventIds = eventNames
+                    .map(eventName => eventNameToIdMap.get(eventName))
+                    .filter((id): id is MageEventId => id !== undefined);
+
+                  // Construct the FeatureLayerConfig with eventIds
+                  const featureLayerConfig: FeatureLayerConfig = {
+                    layer: layer.layer,
+                    geometryType: layer.geometryType,
+                    eventIds: eventIds,
+                  };
+
+                  return featureLayerConfig;
+                });
+
                 return {
                   url: updateService.url,
-                  layers: updateService.layers,
-                  identityManager: existingService?.identityManager || ''
-                }
-              })
+                  layers: layers,
+                  // Map exisiting identityManager, client does not send this
+                  identityManager: existingService?.identityManager,
+                };
+              });
 
               await stateRepo.patch({ ...updateConfig, featureServices })
-              
+
               // Sync configuration with feature servers by restarting observation processor
               processor.stop()
               processor.start()
@@ -195,7 +229,7 @@ const arcgisPluginHooks: InitPluginHook<typeof InjectedServices> = {
             let service: FeatureServiceConfig
             let identityManager: ArcGISIdentityManager
             if (token) {
-              identityManager = await ArcGISIdentityManager.fromToken({ token }) 
+              identityManager = await ArcGISIdentityManager.fromToken({ token })
               service = { url, layers: [], identityManager: identityManager.serialize() }
             } else if (username && password) {
               identityManager = await ArcGISIdentityManager.signIn({
@@ -221,7 +255,7 @@ const arcgisPluginHooks: InitPluginHook<typeof InjectedServices> = {
               return res.send('Invalid credentials provided to communicate with feature service').status(400)
             }
           })
-            
+
           routes.get('/featureService/layers', async (req, res, next) => {
             const url = req.query.featureServiceUrl as string
             const config = await processor.safeGetConfig()

Original file line number	Diff line number	Diff line change
`@@ -34,6 +34,10 @@ export class EventDeletionHandler {`
`34`	`34`	`this._config = config;`
`35`	`35`	`}`
`36`	`36`
	`37`	`+ public updateConfig(newConfig: ArcGISPluginConfig): void {`
	`38`	`+ this._config = newConfig;`
	`39`	`+ }`
	`40`	`+`
`37`	`41`	`/**`
`38`	`42`	`*`
`39`	`43`	`* @param activeEvents The current set of active events.`
`@@ -83,7 +87,7 @@ export class EventDeletionHandler {`
`83`	`87`	`}`
`84`	`88`
`85`	`89`	`/**`
`86`		`- * Called when the query is finished. It goes through the results and gathers all even Ids currently stored`
	`90`	`+ * Called when the query is finished. It goes through the results and gathers all event Ids currently stored`
`87`	`91`	`* in the arc layer. It then will remove any events from the arc layer that do not exist.`
`88`	`92`	`* @param layerProcessor The feature layer processor.`
`89`	`93`	`* @param result The returned results.`
Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ export class EventLayerProcessorOrganizer {`
`19`	`19`	`for (const event of events) {`
`20`	`20`	`let syncProcessors = new Array<FeatureLayerProcessor>();`
`21`	`21`	`for (const layerProcessor of layerProcessors) {`
`22`		`- if (layerProcessor.layerInfo.hasEvent(event.name)) {`
	`22`	`+ if (layerProcessor.layerInfo.hasEvent(event.id)) {`
`23`	`23`	`syncProcessors.push(layerProcessor);`
`24`	`24`	`}`
`25`	`25`	`}`