diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/deployments/import.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/deployments/import.ts new file mode 100644 index 0000000000..e6113046f2 --- /dev/null +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/deployments/import.ts @@ -0,0 +1,142 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles'; +import { getDBConnection } from '../../../../../../database/db'; +import { HTTP422CSVValidationError } from '../../../../../../errors/http-error'; +import { CSVValidationErrorResponse } from '../../../../../../openapi/schemas/csv'; +import { csvFileSchema } from '../../../../../../openapi/schemas/file'; +import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization'; +import { ImportDeploymentService } from '../../../../../../services/import-services/deployments/import-deployment-service'; +import { CSV_ERROR_MESSAGE } from '../../../../../../utils/csv-utils/csv-config-validation.interface'; +import { getLogger } from '../../../../../../utils/logger'; +import { parseMulterFile } from '../../../../../../utils/media/media-utils'; +import { getFileFromRequest } from '../../../../../../utils/request'; +import { constructXLSXWorkbook, getDefaultWorksheet } from '../../../../../../utils/xlsx-utils/worksheet-utils'; + +const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/deployments/import'); + +export const POST: Operation = [ + authorizeRequestHandler((req) => { + return { + or: [ + { + validProjectPermissions: [PROJECT_PERMISSION.COORDINATOR, PROJECT_PERMISSION.COLLABORATOR], + surveyId: Number(req.params.surveyId), + discriminator: 'ProjectPermission' + }, + { + validSystemRoles: [SYSTEM_ROLE.DATA_ADMINISTRATOR], + discriminator: 'SystemRole' + } + ] + }; + }), + importDeploymentCSV() +]; + +POST.apiDoc = { + description: 'Upload survey deployment submission file.', + tags: ['deployment'], + security: [ + { + Bearer: [] + } + ], + parameters: [ + { + in: 'path', + name: 'projectId', + required: true + }, + { + in: 'path', + name: 'surveyId', + required: true + } + ], + requestBody: { + description: 'Survey deployment submission file to upload', + required: true, + content: { + 'multipart/form-data': { + schema: { + type: 'object', + additionalProperties: false, + required: ['media'], + properties: { + media: { + description: 'A survey deployment submission file.', + type: 'array', + minItems: 1, + maxItems: 1, + items: csvFileSchema + } + } + } + } + } + }, + responses: { + 200: { + description: 'Import OK' + }, + 400: { + $ref: '#/components/responses/400' + }, + 401: { + $ref: '#/components/responses/401' + }, + 403: { + $ref: '#/components/responses/403' + }, + 422: CSVValidationErrorResponse, + 500: { + $ref: '#/components/responses/500' + }, + default: { + $ref: '#/components/responses/default' + } + } +}; + +/** + * Imports deployments from a CSV file. + * + * @return {*} {RequestHandler} + */ +export function importDeploymentCSV(): RequestHandler { + return async (req, res) => { + const surveyId = Number(req.params.surveyId); + const rawFile = getFileFromRequest(req); + + const connection = getDBConnection(req.keycloak_token); + + const mediaFile = parseMulterFile(rawFile); + const worksheet = getDefaultWorksheet(constructXLSXWorkbook(mediaFile)); + + try { + await connection.open(); + + const deploymentService = new ImportDeploymentService(connection, worksheet, surveyId); + + const errors = await deploymentService.importCSVWorksheet(); + + if (errors.length) { + throw new HTTP422CSVValidationError(CSV_ERROR_MESSAGE, errors); + } + + await connection.commit(); + + return res.status(200).send(); + } catch (error) { + if (error instanceof HTTP422CSVValidationError === false) { + defaultLog.error({ label: 'importDeployment', message: 'error', error }); + } + + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/services/import-services/deployments/deployment-header-configs.test.ts b/api/src/services/import-services/deployments/deployment-header-configs.test.ts new file mode 100644 index 0000000000..389d035d18 --- /dev/null +++ b/api/src/services/import-services/deployments/deployment-header-configs.test.ts @@ -0,0 +1,265 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { ExtendedDeploymentRecord } from '../../../repositories/telemetry-repositories/telemetry-deployment-repository.interface'; +import { CSVParams } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { getTelemetrySerialCellValidator, getTelemetryVendorCellValidator } from './deployment-header-configs'; + +describe('TelemetryHeaderConfigs', () => { + describe('getTelemetryVendorCellValidator', () => { + it('should return no errors if the vendor is supported', () => { + const vendors = new Set(['lotek']); + const cellValidator = getTelemetryVendorCellValidator(vendors); + + const result = cellValidator({ cell: 'lotek' } as CSVParams); + expect(result).to.deep.equal([]); + }); + + it('should return an error if the vendor is not supported', () => { + const vendors = new Set(['lotek']); + const cellValidator = getTelemetryVendorCellValidator(vendors); + + const result = cellValidator({ cell: 'not-supported' } as CSVParams); + expect(result).to.deep.equal([ + { + error: 'Telemetry vendor not supported', + solution: 'Use a valid telemetry vendor', + values: ['lotek'] + } + ]); + }); + }); + + describe('getTelemetrySerialCellValidator', () => { + it('should return an error if the device is not found in the survey deployments', () => { + const deployments = [ + { + device_key: 'lotek:1234' + } + ]; + + const surveyCritterAliasMap = new Map(); + + const utils = { + getCellValue: () => 'lotek' + }; + + const cellValidator = getTelemetrySerialCellValidator( + deployments as ExtendedDeploymentRecord[], + surveyCritterAliasMap, + utils as any + ); + + const result = cellValidator({ cell: 5555 } as CSVParams); + expect(result).to.deep.equal([ + { + error: 'Device not found in deployments', + solution: 'Check that the serial number and vendor match a deployment in the Survey' + } + ]); + }); + + it('should return an error if alias provided and does not match a deployment', () => { + const deployments = [ + { + device_key: 'lotek:1234', + critterbase_critter_id: 'critter_id' + } + ]; + + const surveyCritterAliasMap = new Map([['alias', { critter_id: 'bad_critter_id' }]]); + + const getCellValueStub = sinon.stub(); + + getCellValueStub.onCall(0).returns('lotek'); + getCellValueStub.onCall(1).returns('alias'); + + const utils = { + getCellValue: getCellValueStub + }; + + const cellValidator = getTelemetrySerialCellValidator( + deployments as ExtendedDeploymentRecord[], + surveyCritterAliasMap, + utils as any + ); + + const result = cellValidator({ cell: 1234, row: { ALIAS: 'alias' } } as any); + expect(result).to.deep.equal([ + { + error: 'Device and alias does not match any deployments for the critter', + solution: 'Check that the serial number, vendor and critter alias match a deployment in the Survey' + } + ]); + }); + + describe('matching deployments with acquisition timestamps', () => { + it('should fail when two overlapping ambiguous timestamps (same device)', () => { + const deployments = [ + { + device_key: 'lotek:1234', + critterbase_critter_id: 'critter_id', + attachment_start_timestamp: '2021-01-01 12:00:00' + }, + { + device_key: 'lotek:1234', + critterbase_critter_id: 'critter_id2', + attachment_start_timestamp: '2021-01-01 12:00:00', + attachment_end_timestamp: '2021-01-03 12:00:00' + } + ]; + + const surveyCritterAliasMap = new Map(); + + const getCellValueStub = sinon.stub(); + + getCellValueStub.onCall(0).returns('lotek'); + getCellValueStub.onCall(1).returns(undefined); + + const utils = { + getCellValue: getCellValueStub + }; + + const cellValidator = getTelemetrySerialCellValidator( + deployments as ExtendedDeploymentRecord[], + surveyCritterAliasMap, + utils as any + ); + + const result = cellValidator({ cell: 1234 } as any); + expect(result[0].error).to.contain('uniquely identify'); + }); + + it('should pass when two non-overlapping timestamps (same device)', () => { + const deployments = [ + { + device_key: 'lotek:1234', + critterbase_critter_id: 'critter_id', + attachment_start_timestamp: '2021-01-01 12:00:00', + attachment_end_timestamp: '2021-01-02 12:00:00' + }, + { + device_key: 'lotek:1234', + critterbase_critter_id: 'critter_id2', + attachment_start_timestamp: '2021-01-03 12:00:00', + attachment_end_timestamp: '2021-01-04 12:00:00' + } + ]; + + const surveyCritterAliasMap = new Map(); + + const getCellValueStub = sinon.stub(); + + getCellValueStub.onCall(0).returns('lotek'); + getCellValueStub.onCall(1).returns(undefined); + getCellValueStub.onCall(2).returns('2021-01-03'); + getCellValueStub.onCall(3).returns('12:00:00'); + + const utils = { + getCellValue: getCellValueStub + }; + + const cellValidator = getTelemetrySerialCellValidator( + deployments as ExtendedDeploymentRecord[], + surveyCritterAliasMap, + utils as any + ); + + const result = cellValidator({ cell: 1234 } as any); + expect(result).to.deep.equal([]); + }); + + it('should pass when two overlapping ambiguous timestamps (same device) and alias pre-filters deployments', () => { + const deployments = [ + { + device_key: 'lotek:1234', + critterbase_critter_id: 'critter_id', + attachment_start_timestamp: '2021-01-01 12:00:00', + attachment_end_timestamp: '2021-01-02 12:00:00' + }, + { + device_key: 'lotek:1234', + critterbase_critter_id: 'critter_id2', + attachment_start_timestamp: '2021-01-03 12:00:00', + attachment_end_timestamp: '2021-01-04 12:00:00' + }, + { + device_key: 'lotek:1234', + critterbase_critter_id: 'critter_id', + attachment_start_timestamp: '2021-01-03 12:00:00', + attachment_end_timestamp: '2021-01-04 12:00:00' + } + ]; + + const surveyCritterAliasMap = new Map([['alias', { critter_id: 'critter_id' }]]); + + const getCellValueStub = sinon.stub(); + + getCellValueStub.onCall(0).returns('lotek'); + getCellValueStub.onCall(1).returns('alias'); + getCellValueStub.onCall(2).returns('2021-01-03'); + getCellValueStub.onCall(3).returns('12:00:00'); + + const utils = { + getCellValue: getCellValueStub + }; + + const cellValidator = getTelemetrySerialCellValidator( + deployments as ExtendedDeploymentRecord[], + surveyCritterAliasMap, + utils as any + ); + + const result = cellValidator({ cell: 1234 } as any); + expect(result).to.deep.equal([]); + }); + }); + + it('should return no errors if the device is found in the survey deployments', () => { + const deployments = [ + { + device_key: 'lotek:1234' + } + ]; + + const surveyCritterAliasMap = new Map(); + + const utils = { + getCellValue: () => 'lotek' + }; + + const cellValidator = getTelemetrySerialCellValidator( + deployments as ExtendedDeploymentRecord[], + surveyCritterAliasMap, + utils as any + ); + + const result = cellValidator({ cell: 1234 } as CSVParams); + expect(result).to.deep.equal([]); + }); + + it('should mutate the mutateCell property to the deployment ID', () => { + const deployments = [ + { + device_key: 'lotek:1234', + deployment_id: 1 + } + ]; + + const surveyCritterAliasMap = new Map(); + + const utils = { + getCellValue: () => 'lotek' + }; + + const cellValidator = getTelemetrySerialCellValidator( + deployments as ExtendedDeploymentRecord[], + surveyCritterAliasMap, + utils as any + ); + + const params = { cell: 1234, row: {} } as CSVParams; + cellValidator(params); + expect(params.mutateCell).to.equal(1); + }); + }); +}); diff --git a/api/src/services/import-services/deployments/deployment-header-configs.ts b/api/src/services/import-services/deployments/deployment-header-configs.ts new file mode 100644 index 0000000000..1e525ddb48 --- /dev/null +++ b/api/src/services/import-services/deployments/deployment-header-configs.ts @@ -0,0 +1,137 @@ +import { DeviceRecord } from '../../../database-models/device'; +import { CaseInsensitiveMap } from '../../../utils/case-insensitive-map'; +import { CSVConfigUtils } from '../../../utils/csv-utils/csv-config-utils'; +import { CSVCellValidator } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { setToLowercase } from '../../../utils/string-utils'; +import { ICaptureDetailed } from '../../critterbase-service'; +import { getTelemetryDeviceKey } from '../../telemetry-services/telemetry-utils'; +import { updateCSVRowState } from '../utils/row-state'; +import { DeploymentCSVStaticHeader } from './import-deployment-service'; + +/** + * Get the deployment critter alias cell validator. + * This validator maps aliases directly to SIMS internal critter_id (integer). + * + * Rules: + * 1. The alias must exist in the deployment alias map + * 2. Updates row state with SIMS internal critter_id + * + * @param {Map} deploymentAliasMap Map of alias (lowercase) → SIMS critter_id (integer) + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getDeploymentCritterAliasCellValidator = (deploymentAliasMap: Map): CSVCellValidator => { + return (params) => { + if (params.cell === undefined) { + return [ + { + error: 'Cell is required', + solution: 'Use a valid critter alias that exists in the Survey' + } + ]; + } + + const alias = String(params.cell).toLowerCase(); + const simscritterId = deploymentAliasMap.get(alias); + + if (simscritterId === undefined) { + return [ + { + error: `Unable to find a matching survey critter`, + solution: `Use a valid critter alias that exists in the Survey` + } + ]; + } + + // Update the row state with the SIMS internal critter ID (integer) + updateCSVRowState(params.row, { critterId: simscritterId }); + + return []; + }; +}; + +/** + * Get a cell validator for the frequency column in a Deployment CSV. + * + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getFrequencyUnitCellValidator = (frequency_units: Set): CSVCellValidator => { + const frequency_unitsLowerCased = setToLowercase(frequency_units); + + return (params) => { + if (frequency_unitsLowerCased.has(String(params.cell).toLowerCase())) { + return []; + } + + return [ + { + error: `Frequency unit not supported`, + solution: `Use a valid frequency unit`, + values: Array.from(frequency_units) + } + ]; + }; +}; + +export const getCritterCaptureCellValidator = (captures: ICaptureDetailed[]): CSVCellValidator => { + return (params) => { + const capture = captures.find((capture) => params.cell === capture.capture_date); + + if (!capture) { + return [ + { + error: 'Capture date not found', + message: `There is no capture on ${params.cell}.`, + solution: 'Create a capture for this date or correct the date in your csv.' + } + ]; + } + + return []; + }; +}; + +/** + * Get a cell validator for the serial number column in a Deployment CSV. + * + * Rules: + * 1. The serial and vendor must generate a valid device key + * 2. The device key must exist in the device dictionary + * 3. Updates row state with device_id + * + * @param {DeviceRecord[]} devices The list of devices for the survey + * @param {CSVConfigUtils} utils The CSV config utils + * @returns {*} {CSVCellValidator} The validate cell callback + */ +export const getDeviceSerialCellValidator = ( + devices: DeviceRecord[], + utils: CSVConfigUtils +): CSVCellValidator => { + const deviceMap = new CaseInsensitiveMap(); + + // Populate the dictionary: device_key -> device + for (const device of devices) { + deviceMap.set(device.device_key, device); + } + + return (params) => { + const serial = Number(params.cell); + const vendor = String(utils.getCellValue('VENDOR', params.row)).toLowerCase(); + const deviceKey = getTelemetryDeviceKey({ vendor, serial }); + + // Rule: Validate device key exists in the device map + const matchingDevice = deviceMap.get(deviceKey); + if (!matchingDevice) { + return [ + { + error: `Device not found`, + solution: `Check that the serial number '${serial}' and vendor '${vendor}' match a device in the Survey` + } + ]; + } + + // Update the row state with the device ID + updateCSVRowState(params.row, { deviceId: matchingDevice.device_id }); + + return []; + }; +}; diff --git a/api/src/services/import-services/deployments/import-deployment-service.test.ts b/api/src/services/import-services/deployments/import-deployment-service.test.ts new file mode 100644 index 0000000000..3672868481 --- /dev/null +++ b/api/src/services/import-services/deployments/import-deployment-service.test.ts @@ -0,0 +1,113 @@ +import chai, { expect } from 'chai'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { WorkSheet } from 'xlsx'; +import { ExtendedDeploymentRecord } from '../../../repositories/telemetry-repositories/telemetry-deployment-repository.interface'; +import * as csv from '../../../utils/csv-utils/csv-config-validation'; +import { CSVConfig, CSVRowState } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { getMockDBConnection } from '../../../__mocks__/db'; +import { ImportTelemetryService } from './import-deployment-service'; + +chai.use(sinonChai); + +describe('ImportTelemetryService', () => { + beforeEach(() => { + sinon.restore(); + }); + + describe('getCSVConfig', () => { + it('should return the CSVConfig for Telemetry', async () => { + const connection = getMockDBConnection(); + const service = new ImportTelemetryService(connection, {}, 1); + + const getSurveyDeploymentsStub = sinon.stub(service.deploymentService, 'getDeploymentsForSurvey'); + const getVendorsStub = sinon.stub(service.codeRepository, 'getActiveTelemetryDeviceMakes'); + const getSurveyCritterAliasMapStub = sinon.stub(service.surveyCritterService, 'getSurveyCritterAliasMap'); + + getSurveyDeploymentsStub.resolves([{ device_key: 'lotek:1234' } as ExtendedDeploymentRecord]); + getVendorsStub.resolves([{ name: 'Lotek' } as any]); + getSurveyCritterAliasMapStub.resolves(new Map()); + + const config = await service.getCSVConfig(); + + expect(getSurveyDeploymentsStub).to.have.been.calledOnceWithExactly(1); + expect(getVendorsStub).to.have.been.calledOnceWithExactly(); + expect(config.staticHeadersConfig).to.have.keys( + 'SERIAL', + 'VENDOR', + 'ALIAS', + 'LATITUDE', + 'LONGITUDE', + 'DATE', + 'TIME' + ); + }); + }); + + describe('importCSVWorksheet', () => { + it('should import the CSV worksheet', async () => { + const mockConnection = getMockDBConnection(); + const worksheet = {} as WorkSheet; + const surveyId = 1; + + const service = new ImportTelemetryService(mockConnection, worksheet, surveyId); + + const mockCSVConfig = {} as CSVConfig; + const mockGetConfig = sinon.stub(service, 'getCSVConfig').resolves(mockCSVConfig); + const bulkCreateStub = sinon.stub(service.telemetryVendorService, 'bulkCreateTelemetryInBatches').resolves(); + + const mockValidate = sinon.stub(csv, 'validateCSVWorksheet').returns({ + errors: [], + rows: [ + { + SERIAL: 'uuid', + VENDOR: 'lotek', + LATITUDE: 1.234, + LONGITUDE: 2.345, + DATE: '2021-01-01', + TIME: '12:00:00', + [CSVRowState]: {} + } + ] + }); + + await service.importCSVWorksheet(); + + expect(mockGetConfig).to.have.been.called; + expect(mockValidate).to.have.been.calledOnceWithExactly(worksheet, mockCSVConfig); + expect(bulkCreateStub).to.have.been.calledOnceWithExactly(1, [ + { + deployment_id: 'uuid', + latitude: 1.234, + longitude: 2.345, + acquisition_date: '2021-01-01 12:00:00', + transmission_date: null + } + ]); + }); + + it('should return CSV Validation error if rows fail validation', async () => { + const mockConnection = getMockDBConnection(); + const worksheet = {} as WorkSheet; + const surveyId = 1; + + const service = new ImportTelemetryService(mockConnection, worksheet, surveyId); + + const mockCSVConfig = {} as CSVConfig; + const mockGetConfig = sinon.stub(service, 'getCSVConfig').resolves(mockCSVConfig); + + const mockValidate = sinon.stub(csv, 'validateCSVWorksheet').returns({ + errors: [{ error: 'error', solution: 'solution', values: [], cell: 'A1', row: 1, header: 'SERIAL' }], + rows: [] + }); + + const errors = await service.importCSVWorksheet(); + + expect(mockGetConfig).to.have.been.called; + expect(mockValidate).to.have.been.calledOnceWithExactly(worksheet, mockCSVConfig); + expect(errors).to.deep.equal([ + { error: 'error', solution: 'solution', values: [], cell: 'A1', row: 1, header: 'SERIAL' } + ]); + }); + }); +}); diff --git a/api/src/services/import-services/deployments/import-deployment-service.ts b/api/src/services/import-services/deployments/import-deployment-service.ts new file mode 100644 index 0000000000..3492ad9768 --- /dev/null +++ b/api/src/services/import-services/deployments/import-deployment-service.ts @@ -0,0 +1,323 @@ +import { WorkSheet } from 'xlsx'; +import { IDBConnection } from '../../../database/db'; +import { CodeRepository } from '../../../repositories/code-repository'; +import { CreateDeployment } from '../../../repositories/telemetry-repositories/telemetry-deployment-repository.interface'; +import { CSVConfigUtils } from '../../../utils/csv-utils/csv-config-utils'; +import { validateCSVWorksheet } from '../../../utils/csv-utils/csv-config-validation'; +import { CSVConfig, CSVError, CSVRowState } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { + getDateCellValidator, + getPositiveNumberCellValidator, + getTimeCellSetter, + getTimeCellValidator +} from '../../../utils/csv-utils/csv-header-configs'; +import { getAllAliases } from '../../../utils/csv-utils/csv-helpers'; +import { getLogger } from '../../../utils/logger'; +import { DBService } from '../../db-service'; +import { SurveyCritterService } from '../../survey-critter-service'; +import { TelemetryDeploymentService } from '../../telemetry-services/telemetry-deployment-service'; +import { TelemetryDeviceService } from '../../telemetry-services/telemetry-device-service'; +import { TelemetryVendorService } from '../../telemetry-services/telemetry-vendor-service'; +import { getTelemetryVendorCellValidator } from '../telemetry/telemetry-header-configs'; +import { + getDeploymentCritterAliasCellValidator, + getDeviceSerialCellValidator, + getFrequencyUnitCellValidator +} from './deployment-header-configs'; + +const defaultLog = getLogger('services/import-services/import-telemetry-service'); + +// Deployment CSV static headers +export type DeploymentCSVStaticHeader = + | 'VENDOR' + | 'SERIAL' + | 'ALIAS' + | 'CAPTURE_DATE' + | 'CAPTURE_TIME' + | 'END_DATE' + | 'END_TIME' + | 'FREQUENCY' + | 'FREQUENCY_UNIT' + | 'END_CAPTURE_DATE' + | 'MORTALITY_DATE'; + +/** + * ImportDeploymentService - A service for importing Deployments from a CSV into SIMS. + * + * @class ImportDeploymentService + * @extends DBService + */ +export class ImportDeploymentService extends DBService { + worksheet: WorkSheet; + surveyId: number; + + // Services + deploymentService: TelemetryDeploymentService; + telemetryVendorService: TelemetryVendorService; + surveyCritterService: SurveyCritterService; + codeRepository: CodeRepository; + utils: CSVConfigUtils; + deviceService: TelemetryDeviceService; + + /** + * Construct an instance of ImportDeploymentService. + * + * @param {IDBConnection} connection - DB connection + * @param {string} surveyId + */ + constructor(connection: IDBConnection, worksheet: WorkSheet, surveyId: number) { + super(connection); + + const initialConfig: CSVConfig = { + staticHeadersConfig: { + SERIAL: { aliases: getAllAliases(['DEVICE_ID', 'DEVICE', 'COLLAR', 'COLLAR_ID']) }, + VENDOR: { aliases: ['MAKE', 'MANUFACTURER'] }, + ALIAS: { aliases: ['NICKNAME', 'ANIMAL'] }, + CAPTURE_DATE: { + aliases: ['START_DATE', 'START DATE', 'CAPTURE DATE', 'DATE DEPLOYED', 'ATTACHMENT_START_DATE'] + }, + CAPTURE_TIME: { + aliases: ['START_TIME', 'START TIME', 'CAPTURE TIME', 'TIME DEPLOYED', 'ATTACHMENT_START_TIME'], + optional: true + }, + END_DATE: { + aliases: ['END DATE', 'DATE REMOVED', 'DATE RECOVERED', 'ATTACHMENT_END_DATE'], + optional: true + }, + END_TIME: { + aliases: ['END TIME', 'TIME REMOVED', 'TIME RECOVERED', 'ATTACHMENT_END_TIME'], + optional: true + }, + FREQUENCY: { aliases: ['FREQ'], optional: true }, + FREQUENCY_UNIT: { aliases: ['FREQ_UNIT', 'FREQUENCY UNIT'], optional: true }, + END_CAPTURE_DATE: { + aliases: ['END CAPTURE DATE', 'CAPTURE END DATE'], + optional: true + }, + MORTALITY_DATE: { aliases: ['MORTALITY DATE', 'DATE OF MORTALITY'], optional: true } + }, + ignoreDynamicHeaders: false + }; + + this.worksheet = worksheet; + this.surveyId = surveyId; + + // Initialize services + this.deploymentService = new TelemetryDeploymentService(connection); + this.telemetryVendorService = new TelemetryVendorService(connection); + this.surveyCritterService = new SurveyCritterService(connection); + this.codeRepository = new CodeRepository(connection); + this.utils = new CSVConfigUtils(this.worksheet, initialConfig); + this.deviceService = new TelemetryDeviceService(connection); + } + + /** + * Import a Deployment CSV worksheet into SIMS. + * + * @async + * @throws {ApiGeneralError} - If unable to fully insert records into SIMS + * @returns {*} {Promise} List of CSV errors encountered during import + */ + async importCSVWorksheet(): Promise { + const config = await this.getCSVConfig(); + + const { errors, rows } = validateCSVWorksheet(this.worksheet, config); + + if (errors.length) { + return errors; + } + + const deployments: CreateDeployment[] = []; + + // Process each row asynchronously to handle the async helper methods + for (const row of rows) { + const frequencyUnitId = await this.getFrequencyUnitId(row.FREQUENCY_UNIT); + const startCaptureId = await this.getCaptureIdForCritter(row.ALIAS, row.CAPTURE_DATE); + const endCaptureId = await this.getCaptureIdForCritter(row.ALIAS, row.END_CAPTURE_DATE); + const mortalityId = await this.getMortalityIdForCritter(row.ALIAS, row.MORTALITY_DATE); + + deployments.push({ + survey_id: this.surveyId, + critter_id: row[CSVRowState]?.critterId, + device_id: row[CSVRowState]?.deviceId, + frequency: row.FREQUENCY, + frequency_unit_id: frequencyUnitId, + attachment_start_date: row.CAPTURE_DATE, + attachment_start_time: row.CAPTURE_TIME, + attachment_end_date: row.END_DATE, + attachment_end_time: row.END_TIME, + critterbase_start_capture_id: startCaptureId, + critterbase_end_capture_id: endCaptureId, + critterbase_end_mortality_id: mortalityId + }); + } + + defaultLog.info({ + label: 'importCSVWorksheet', + message: 'Creating deployment records', + deploymentCount: deployments.length + }); + + for (const deploymentRecord of deployments) { + await this.deploymentService.createDeployment(deploymentRecord); + } + + // Return an empty array if no errors occurred + return []; + } + + /** + * Get the CSV configuration for Deployments. + * + * @returns {Promise>} + */ + async getCSVConfig(): Promise> { + const [deploymentAliasMap, devices, vendors, frequency_units] = await Promise.all([ + this.getDeploymentCritterAliasMap(), + this.deviceService.getDevicesForSurvey(this.surveyId), + this.codeRepository.getActiveTelemetryDeviceMakes(), + this.codeRepository.getFrequencyUnits() + ]); + + // Ensure vendors have a proper type + const vendorsSet = new Set(vendors.map((vendor: { name: string }) => vendor.name.toLowerCase())); + + // Ensure frequency units are properly calling ids + const frequencySet = new Set(frequency_units.map((frequency_unit) => frequency_unit.name.toLowerCase())); + + // Update individual static header configs to preserve the optional settings + this.utils.setStaticHeaderConfig('SERIAL', { validateCell: getDeviceSerialCellValidator(devices, this.utils) }); + this.utils.setStaticHeaderConfig('ALIAS', { + validateCell: getDeploymentCritterAliasCellValidator(deploymentAliasMap) + }); + this.utils.setStaticHeaderConfig('VENDOR', { validateCell: getTelemetryVendorCellValidator(vendorsSet) }); + this.utils.setStaticHeaderConfig('CAPTURE_DATE', { validateCell: getDateCellValidator() }); + this.utils.setStaticHeaderConfig('CAPTURE_TIME', { + validateCell: getTimeCellValidator(), + setCellValue: getTimeCellSetter() + }); + this.utils.setStaticHeaderConfig('END_DATE', { validateCell: getDateCellValidator() }); + this.utils.setStaticHeaderConfig('END_TIME', { + validateCell: getTimeCellValidator(), + setCellValue: getTimeCellSetter() + }); + this.utils.setStaticHeaderConfig('FREQUENCY', { validateCell: getPositiveNumberCellValidator() }); + this.utils.setStaticHeaderConfig('FREQUENCY_UNIT', { validateCell: getFrequencyUnitCellValidator(frequencySet) }); + this.utils.setStaticHeaderConfig('END_CAPTURE_DATE', { validateCell: getDateCellValidator({ optional: true }) }); + this.utils.setStaticHeaderConfig('MORTALITY_DATE', { validateCell: getDateCellValidator({ optional: true }) }); + + // Return the final CSV config + return this.utils.getConfig(); + } + + /** + * Get alias-to-SIMS-critter-ID mapping for deployments. + * This creates a direct mapping from alias → SIMS internal critter_id (integer). + * + * @returns {Promise>} Map of alias (lowercase) → SIMS critter_id (integer) + */ + async getDeploymentCritterAliasMap(): Promise> { + // Get SIMS critters for this survey (has both critter_id and critterbase_critter_id) + const simsCritters = await this.surveyCritterService.getCrittersInSurvey(this.surveyId); + + if (!simsCritters.length) { + return new Map(); + } + + // Get Critterbase data to get the aliases (animal_id) + const critterbaseCritterUUIDs = simsCritters.map((critter) => critter.critterbase_critter_id); + const critterbaseCritters = + await this.surveyCritterService.critterbaseService.getMultipleCrittersByIdsDetailed(critterbaseCritterUUIDs); + + // Create a map: critterbase UUID → SIMS critter_id + const uuidToSimsIdMap = new Map(); + for (const simsCritter of simsCritters) { + uuidToSimsIdMap.set(simsCritter.critterbase_critter_id, simsCritter.critter_id); + } + + // Create final mapping: alias → SIMS critter_id + const aliasToSimsIdMap = new Map(); + + for (const critterbaseData of critterbaseCritters) { + if (critterbaseData.animal_id) { + const alias = critterbaseData.animal_id.toLowerCase(); + const simsId = uuidToSimsIdMap.get(critterbaseData.critter_id); + + if (simsId !== undefined) { + aliasToSimsIdMap.set(alias, simsId); + } + } + } + + return aliasToSimsIdMap; + } + + /** + * Get frequency unit ID by name. + * + * @param {string} frequencyUnitName - The frequency unit name + * @returns {number | null} The frequency unit ID or null + */ + async getFrequencyUnitId(frequencyUnitName: string): Promise { + if (!frequencyUnitName) { + return null; + } + + const frequencyUnits = await this.codeRepository.getFrequencyUnits(); + const unit = frequencyUnits.find((fu) => fu.name.toLowerCase() === frequencyUnitName.toLowerCase()); + return unit?.id || null; + } + + /** + * Get capture ID for a critter by alias and date. + * + * @param {string} alias - The critter alias + * @param {string} captureDate - The capture date + * @returns {string | null} The capture ID or null + */ + async getCaptureIdForCritter(alias: string, captureDate: string): Promise { + if (!alias || !captureDate) { + return null; + } + + const surveyCritterAliasMap = await this.surveyCritterService.getSurveyCritterAliasMap(this.surveyId); + const critterData = surveyCritterAliasMap.get(alias); + + if (!critterData?.captures) { + return null; + } + + // Find capture that matches the date + const capture = critterData.captures.find((cap) => cap.capture_date === captureDate); + + return capture?.capture_id || null; + } + + /** + * Get mortality ID for a critter by alias and date. + * + * @param {string} alias - The critter alias + * @param {string} mortalityDate - The mortality date + * @returns {string | null} The mortality ID or null + */ + async getMortalityIdForCritter(alias: string, mortalityDate: string): Promise { + if (!alias || !mortalityDate) { + return null; + } + + const surveyCritterAliasMap = await this.surveyCritterService.getSurveyCritterAliasMap(this.surveyId); + const critterData = surveyCritterAliasMap.get(alias); + + if (!critterData?.mortality) { + return null; + } + + // Check if mortality is an array or single object + const mortalities = Array.isArray(critterData.mortality) ? critterData.mortality : [critterData.mortality]; + + // Find mortality that matches the date + const mortality = mortalities.find((mort: any) => mort.mortality_date === mortalityDate); + + return mortality?.mortality_id || null; + } +} diff --git a/app/src/features/surveys/telemetry/manage/deployments/table/DeploymentsContainer.tsx b/app/src/features/surveys/telemetry/manage/deployments/table/DeploymentsContainer.tsx index 8e5f15acca..d393a94b3e 100644 --- a/app/src/features/surveys/telemetry/manage/deployments/table/DeploymentsContainer.tsx +++ b/app/src/features/surveys/telemetry/manage/deployments/table/DeploymentsContainer.tsx @@ -1,4 +1,4 @@ -import { mdiArrowTopRight, mdiDotsVertical, mdiPlus, mdiTrashCanOutline } from '@mdi/js'; +import { mdiArrowTopRight, mdiDotsVertical, mdiImport, mdiPlus, mdiTrashCanOutline } from '@mdi/js'; import Icon from '@mdi/react'; import Box from '@mui/material/Box'; import Button from '@mui/material/Button'; @@ -8,9 +8,12 @@ import ListItemIcon from '@mui/material/ListItemIcon'; import ListItemText from '@mui/material/ListItemText'; import Menu from '@mui/material/Menu'; import MenuItem from '@mui/material/MenuItem'; +import Stack from '@mui/material/Stack'; import Toolbar from '@mui/material/Toolbar'; import Typography from '@mui/material/Typography'; import { GridRowSelectionModel } from '@mui/x-data-grid'; +import axios, { AxiosProgressEvent } from 'axios'; +import { CSVSingleImportDialog } from 'components/csv/CSVSingleImportDialog'; import { LoadingGuard } from 'components/loading/LoadingGuard'; import { SkeletonTable } from 'components/loading/SkeletonLoaders'; import { NoDataOverlay } from 'components/overlay/NoDataOverlay'; @@ -21,6 +24,8 @@ import { useDialogContext, useSurveyContext } from 'hooks/useContext'; import useDataLoader from 'hooks/useDataLoader'; import { useEffect, useState } from 'react'; import { Link as RouterLink } from 'react-router-dom'; +import { getDeploymentCSVTemplate } from 'utils/csv-templates'; +import { downloadFile } from 'utils/file-utils'; export const DeploymentsContainer = () => { const dialogContext = useDialogContext(); @@ -28,6 +33,28 @@ export const DeploymentsContainer = () => { const biohubApi = useBiohubApi(); + // import dialog for deployments + const [showImportDialog, setShowImportDialog] = useState(false); + const cancelToken = axios.CancelToken.source(); + const [processingRecords, setProcessingRecords] = useState(false); + const handleImportDeploymentCSV = async (file: File, onProgress: (progressEvent: AxiosProgressEvent) => void) => { + try { + await biohubApi.telemetryDeployment.importManualDeploymentCSV( + surveyContext.projectId, + surveyContext.surveyId, + file, + cancelToken, + onProgress + ); + + setProcessingRecords(true); + + deploymentsDataLoader.refresh(surveyContext.projectId, surveyContext.surveyId); + } finally { + setProcessingRecords(false); + } + }; + // State for bulk actions const [headerAnchorEl, setHeaderAnchorEl] = useState(null); const [selectedRows, setSelectedRows] = useState([]); @@ -111,6 +138,16 @@ export const DeploymentsContainer = () => { return ( <> + setShowImportDialog(false)} + onImport={handleImportDeploymentCSV} + onDownloadTemplate={() => + downloadFile(getDeploymentCSVTemplate(), `SIMS-deployment-template-${new Date().getFullYear()}.csv`) + } + /> {/* Bulk action menu */} { ({deploymentsCount}) - + + + + { selectedRows={selectedRows} setSelectedRows={setSelectedRows} onDelete={onDelete} + isLoading={processingRecords} /> diff --git a/app/src/features/surveys/telemetry/manage/deployments/table/DeploymentsTable.tsx b/app/src/features/surveys/telemetry/manage/deployments/table/DeploymentsTable.tsx index 69e0d1ed0e..b1f15f6660 100644 --- a/app/src/features/surveys/telemetry/manage/deployments/table/DeploymentsTable.tsx +++ b/app/src/features/surveys/telemetry/manage/deployments/table/DeploymentsTable.tsx @@ -45,6 +45,7 @@ interface IDeploymentsTableProps { deployments: TelemetryDeployment[]; selectedRows: GridRowSelectionModel; setSelectedRows: (selection: GridRowSelectionModel) => void; + isLoading: boolean; /** * Callback fired when a deployment is deleted. */ @@ -58,7 +59,7 @@ interface IDeploymentsTableProps { * @return {*} */ export const DeploymentsTable = (props: IDeploymentsTableProps) => { - const { deployments, selectedRows, setSelectedRows, onDelete } = props; + const { deployments, selectedRows, setSelectedRows, onDelete, isLoading } = props; const biohubApi = useBiohubApi(); @@ -354,6 +355,7 @@ export const DeploymentsTable = (props: IDeploymentsTableProps) => { } }} pageSizeOptions={[10, 25, 50]} + loading={isLoading} /> ); diff --git a/app/src/hooks/api/useTelemetryDeploymentApi.ts b/app/src/hooks/api/useTelemetryDeploymentApi.ts index 47b81d62e6..23ec3d3602 100644 --- a/app/src/hooks/api/useTelemetryDeploymentApi.ts +++ b/app/src/hooks/api/useTelemetryDeploymentApi.ts @@ -1,4 +1,4 @@ -import { AxiosInstance } from 'axios'; +import { AxiosInstance, AxiosProgressEvent, CancelTokenSource } from 'axios'; import { CreateTelemetryDeployment, GetSurveyDeploymentsResponse, @@ -132,12 +132,40 @@ export const useTelemetryDeploymentApi = (axios: AxiosInstance) => { return data; }; + /** + * Imports a deployment CSV. + * + * @param {number} projectId + * @param {number} surveyId + * @param {File} file + * @param {CancelTokenSource} [cancelTokenSource] + * @param {(progressEvent: AxiosProgressEvent) => void} [onProgress] + * @return {*} {Promise} + */ + const importManualDeploymentCSV = async ( + projectId: number, + surveyId: number, + file: File, + cancelTokenSource?: CancelTokenSource, + onProgress?: (progressEvent: AxiosProgressEvent) => void + ): Promise => { + const formData = new FormData(); + + formData.append('media', file); + + await axios.post(`/api/project/${projectId}/survey/${surveyId}/deployments/import`, formData, { + cancelToken: cancelTokenSource?.token, + onUploadProgress: onProgress + }); + }; + return { createDeployment, updateDeployment, getDeploymentById, getDeploymentsInSurvey, deleteDeployment, - deleteDeployments + deleteDeployments, + importManualDeploymentCSV }; }; diff --git a/app/src/utils/csv-templates.ts b/app/src/utils/csv-templates.ts index 85d40faa24..f53b4638c9 100644 --- a/app/src/utils/csv-templates.ts +++ b/app/src/utils/csv-templates.ts @@ -105,3 +105,24 @@ export const getHabitatFeaturesCSVTemplate = (): CSVEncodedTemplate => { 'COMMENT' ]); }; + +/** + * Get CSV template for deployments. + * + * @returns {CSVEncodedTemplate} Encoded CSV template + */ +export const getDeploymentCSVTemplate = (): CSVEncodedTemplate => { + return getCSVTemplate([ + 'VENDOR', + 'SERIAL', + 'ALIAS', + 'CAPTURE_DATE', + 'CAPTURE_TIME', + 'END_DATE', + 'END_TIME', + 'FREQUENCY', + 'FREQUENCY_UNIT', + 'END_CAPTURE_DATE', + 'MORTALITY_DATE' + ]); +};