1432 - Replace password with RSA key for SFTP plugin

instance/config.js

@@ -16,6 +16,8 @@ module.exports = {
     exportSweepInterval: 28800,
     exportTTL: 259200,
     tokenExpiration: 28800,
+    sftpKeyDir: path.join(baseDir, 'sftp-keys'),
+    sftpKeyFile: path.join(baseDir, 'sftp-keys', 'mage-sftp-key'),
     mongo: {
       url: 'mongodb://127.0.0.1:27017/magedb',
       connTimeout: 300,

instance/init.js

@@ -14,3 +14,4 @@ mkdirp(config.mage.layerDir)
 mkdirp(config.mage.securityDir)
 mkdirp(config.mage.tempDir)
 mkdirp(config.mage.userDir)
+mkdirp(config.mage.sftpKeyDir)

plugins/sftp/service/src/configuration/SFTPPluginConfig.ts

@@ -1,6 +1,5 @@
 import { MageEventId } from '@ngageoint/mage.service/lib/entities/events/entities.events';
 import { ArchiveFormat, CompletionAction, TriggerRule } from '../format/entities.format';
-import * as CryptoJS from 'crypto-js';
 
 /**
  * Contains various configuration values used by the plugin.
@@ -51,8 +50,7 @@ export interface SFTPPluginConfig {
   sftpClient: {
     host: string,
     path: string,
-    username: string,
-    password: string
+    username: string
   }
 }
 
@@ -70,23 +68,6 @@ export const defaultSFTPPluginConfig = Object.freeze<SFTPPluginConfig>({
   sftpClient: {
     host: '',
     path: '',
-    username: '',
-    password: ''
+    username: ''
   }
 })
-
-export async function encryptDecrypt(config: SFTPPluginConfig, isEncrypt: boolean): Promise<SFTPPluginConfig> {
-  // NOTE: default INSECURE salt value, recommend generate new UUID before deployment, **NOT** after deployment
-  const salt = "A0E6D3B4-25BD-4DD6-BBC9-B367931966AB"; // process.env.SFTP_PLUGIN_CONFIG_SALT;
-  try {
-    let tempConfig = config;
-    if(salt === undefined) { throw new Error("No salt value found, update docker-compose value...") }
-    const encryptedPass = isEncrypt ?
-      CryptoJS.AES.encrypt(config.sftpClient.password, salt).toString() :
-      CryptoJS.AES.decrypt(config.sftpClient.password, salt).toString();
-    tempConfig.sftpClient.password = encryptedPass;
-    return tempConfig;
-  } catch (err) {
-    throw err;
-  }
-}

plugins/sftp/service/src/controller/controller.spec.ts

@@ -1,4 +1,5 @@
 import _ from 'lodash'
+import fs from 'fs'
 import { PluginStateRepository } from '@ngageoint/mage.service/lib/plugins.api'
 import { MageEvent, MageEventAttrs, MageEventId, MageEventRepository, copyMageEventAttrs } from '@ngageoint/mage.service/lib/entities/events/entities.events'
 import { FormFieldType } from '@ngageoint/mage.service/lib/entities/events/entities.events.forms'
@@ -39,7 +40,7 @@ function newEvent(id: MageEventId): MageEventAttrs {
 }
 
 function newObservation(event: MageEvent, lastModified: Date): ObservationAttrs {
-  return  {
+  return {
     id: "1",
     eventId: event.id,
     userId: "test",
@@ -88,7 +89,7 @@ describe('automated processing', () => {
   let allEvents: Map<MageEventId, MageEvent>
   let stateRepository: TestPluginStateRepository
   let eventObservationRepositories: Map<MageEventId, jasmine.SpyObj<EventScopedObservationRepository>>
-  let observationRepository: (event: MageEventId) => Promise<jasmine.SpyObj<EventScopedObservationRepository>> 
+  let observationRepository: (event: MageEventId) => Promise<jasmine.SpyObj<EventScopedObservationRepository>>
   let archiveFactory: jasmine.SpyObj<ArchiverFactory>
   let sftpClient: jasmine.SpyObj<SFTPClient>
   let clock: jasmine.Clock
@@ -113,6 +114,7 @@ describe('automated processing', () => {
     stateRepository = new TestPluginStateRepository()
     clock = jasmine.clock().install()
     archiveFactory = jasmine.createSpyObj<ArchiverFactory>('archiverFactory', ['createArchiver'])
+    spyOn(fs, 'readFileSync').and.returnValue(Buffer.from('mock ssh key content'))
     sftpClient = jasmine.createSpyObj<SFTPClient>('sftpClient', ['connect', 'put', 'end'])
     sftpClient.connect.and.resolveTo()
     sftpClient.end.and.resolveTo()
@@ -424,7 +426,7 @@ describe('automated processing', () => {
     expect(sftpRepository.postStatus).toHaveBeenCalledWith(event1.id, observation.id, SftpStatus.SUCCESS)
     expect(archiverSpy.createArchive).toHaveBeenCalled()
   })
-  
+
   it('processes updated observations w/ create/update trigger', async () => {
     stateRepository.state = { ...defaultSFTPPluginConfig, interval: 10, enabled: true }
     const clockTickMillis = stateRepository.state.interval * 1000 + 1
@@ -481,7 +483,7 @@ describe('automated processing', () => {
   })
 
   it('skips processing of updated observations w/ create trigger', async () => {
-    stateRepository.state = { ...defaultSFTPPluginConfig, interval: 10, enabled: true, initiation: {rule: TriggerRule.Create,timeout: 60 } }
+    stateRepository.state = { ...defaultSFTPPluginConfig, interval: 10, enabled: true, initiation: { rule: TriggerRule.Create, timeout: 60 } }
     const clockTickMillis = stateRepository.state.interval * 1000 + 1
 
     const eventRepository = jasmine.createSpyObj<MageEventRepository>('eventRepository', ['findActiveEvents'])

plugins/sftp/service/src/controller/controller.ts

@@ -4,9 +4,10 @@ import { Observation, ObservationAttrs, ObservationRepositoryForEvent } from '@n
 import { PluginStateRepository } from '@ngageoint/mage.service/lib/plugins.api';
 import SFTPClient from 'ssh2-sftp-client';
 import { PassThrough } from 'stream';
-import { SFTPPluginConfig, defaultSFTPPluginConfig, encryptDecrypt } from '../configuration/SFTPPluginConfig';
+import { SFTPPluginConfig, defaultSFTPPluginConfig } from '../configuration/SFTPPluginConfig';
 import { ArchiveFormat, ArchiveStatus, ArchiverFactory, ArchiveResult, TriggerRule } from '../format/entities.format';
 import { SftpAttrs, SftpObservationRepository, SftpStatus } from '../adapters/adapters.sftp.mongoose';
+import fs from 'fs';
 
 /**
  * Class used to process observations for SFTP
@@ -96,7 +97,7 @@ export class SftpController {
    */
   public async getConfiguration(): Promise<SFTPPluginConfig> {
     if (this.configuration === null) {
-      return await this.stateRepository.get().then((x: SFTPPluginConfig | null) => !!x ? encryptDecrypt(x, false) : this.stateRepository.put(defaultSFTPPluginConfig))
+      return await this.stateRepository.get().then((x: SFTPPluginConfig | null) => !!x ? x : this.stateRepository.put(defaultSFTPPluginConfig))
     } else {
       return this.configuration
     }
@@ -108,8 +109,7 @@ export class SftpController {
    */
   public async updateConfiguration(configuration: SFTPPluginConfig) {
     try {
-      let config = await encryptDecrypt(configuration, true);
-      await this.stateRepository.put(config)
+      await this.stateRepository.put(configuration)
     } catch (err) {
       this.console.log(`ERROR: updateConfiguration: ${err}`)
     }
@@ -123,13 +123,18 @@ export class SftpController {
     if (!this.configuration.enabled) { return }
 
     try {
-      await this.sftpClient.connect(this.configuration.sftpClient)
+      const sftpKeyFilename = process.env['MAGE_SFTP_KEY_FILE'] as string;
+      const sftpKeyFile = fs.readFileSync(sftpKeyFilename);
+      await this.sftpClient.connect({
+        host: this.configuration.sftpClient.host,
+        username: this.configuration.sftpClient.username,
+        privateKey: sftpKeyFile
+      });
+      this.isRunning = true;
+      await this.processAndScheduleNext()
     } catch (e) {
       this.console.error("error connecting to sftp endpoint", e)
     }
-
-    this.isRunning = true;
-    await this.processAndScheduleNext()
   }
 
   /**
@@ -245,11 +250,16 @@ export class SftpController {
     if (result instanceof ArchiveResult) {
       if (result.status === ArchiveStatus.Complete || (result.status === ArchiveStatus.Incomplete && (observation.lastModified.getTime() + timeout) > Date.now())) {
         this.console.log(`posting status of success`)
-        const stream = new PassThrough()
-        result.archive.pipe(stream)
-        await result.archive.finalize()
-        await this.sftpClient.put(stream, `${sftpPath}/${observation.id}.zip`)
-        await this.sftpObservationRepository.postStatus(event.id, observation.id, SftpStatus.SUCCESS)
+        try {
+          const stream = new PassThrough()
+          result.archive.pipe(stream)
+          // TODO: This will fail if the observation has a .mov file
+          await result.archive.finalize()
+          await this.sftpClient.put(stream, `${sftpPath}/${observation.id}.zip`)
+          await this.sftpObservationRepository.postStatus(event.id, observation.id, SftpStatus.SUCCESS)
+        } catch (error) {
+          this.console.error(`error uploading observation ${observation.id}`, error)
+        }
       } else {
         this.console.log(`posting status of pending`)
 

plugins/sftp/web/projects/main/src/lib/configuration/configuration.component.html

@@ -4,7 +4,7 @@
       <div class="card__header">
         <div class="card__header__title">SFTP Plugin Configuration</div>
       </div>
-  
+
       <div>
         <div class="form-field-container">
           <mat-slide-toggle color="primary" [(ngModel)]="configuration.enabled" [checked]="configuration.enabled">
@@ -22,25 +22,26 @@
             <mat-hint>Format for Observation metadata within SFTP archive.</mat-hint>
           </mat-form-field>
         </div>
-  
+
         <div class="radio-group-container">
           <span class="radio-label">Observation SFTP Rule</span>
           <mat-radio-group class="radio-group" [(ngModel)]="configuration.initiation.rule">
-            <mat-radio-button class="radio-button" color="primary" *ngFor="let rule of triggerRules" [value]="rule.value">
+            <mat-radio-button class="radio-button" color="primary" *ngFor="let rule of triggerRules"
+              [value]="rule.value">
               <span class="radio-button-label">{{rule.title}}</span>
             </mat-radio-button>
           </mat-radio-group>
           <mat-hint class="hint">Rule indicating when to SFTP observation</mat-hint>
         </div>
-  
+
         <div class="form-field-container">
           <mat-form-field appearance="fill">
             <mat-label>Observation Create Timeout</mat-label>
             <input matInput type="number" [(ngModel)]="configuration.initiation.timeout" required>
             <mat-hint>Time in minutes to wait for attachments before observation SFTP.</mat-hint>
           </mat-form-field>
         </div>
-  
+
         <div class="radio-group-container">
           <span class="radio-label">Observation SFTP Success Action</span>
           <mat-radio-group class="radio-group" [(ngModel)]="configuration.completionAction">
@@ -51,33 +52,34 @@
           </mat-radio-group>
           <mat-hint class="hint">Action to perform on observation after successful SFTP</mat-hint>
         </div>
-  
+
         <div class="form-field-container">
           <mat-form-field appearance="fill">
             <mat-label>Poll Interval</mat-label>
             <input matInput type="number" [(ngModel)]="configuration.interval" required>
             <mat-hint>Interval plugin will poll when looking for observation changes.</mat-hint>
           </mat-form-field>
         </div>
-  
+
         <div class="form-field-container">
           <mat-form-field appearance="fill">
             <mat-label>Page Size</mat-label>
             <input matInput type="number" [(ngModel)]="configuration.pageSize" required>
-            <mat-hint>Page size used when querying observations, smaller page size will reduce memory footprint on server.
+            <mat-hint>Page size used when querying observations, smaller page size will reduce memory footprint on
+              server.
             </mat-hint>
           </mat-form-field>
         </div>
       </div>
     </div>
   </div>
-  
+
   <div class="configuration">
     <div class="card">
       <div class="card__header">
         <div class="card__header__title">SFTP Client Options</div>
       </div>
-  
+
       <div>
         <div class="form-field-container">
           <mat-form-field appearance="fill">
@@ -86,35 +88,27 @@
             <mat-hint>Hostname or IP of server.</mat-hint>
           </mat-form-field>
         </div>
-  
+
         <div class="form-field-container">
           <mat-form-field appearance="fill">
             <mat-label>SFTP Path</mat-label>
             <input matInput type="text" [(ngModel)]="configuration.sftpClient.path" required>
             <mat-hint>Path to remote file.</mat-hint>
           </mat-form-field>
         </div>
-  
+
         <div class="form-field-container">
           <mat-form-field appearance="fill">
             <mat-label>SFTP Username</mat-label>
             <input matInput type="text" [(ngModel)]="configuration.sftpClient.username" required>
             <mat-hint>Username for authentication.</mat-hint>
           </mat-form-field>
         </div>
-
-        <div class="form-field-container">
-          <mat-form-field appearance="fill">
-            <mat-label>SFTP Password</mat-label>
-            <input matInput type="password" [(ngModel)]="configuration.sftpClient.password" required>
-            <mat-hint>Password for password based authentication.</mat-hint>
-          </mat-form-field>
-        </div>
       </div>
     </div>
   </div>
-  
+
   <button mat-flat-button color="primary" (click)="save()">
     <mat-icon>save</mat-icon> Save Configuration
   </button>
-</div>
+</div>

plugins/sftp/web/projects/main/src/lib/configuration/configuration.component.ts

@@ -22,9 +22,9 @@ export class ConfigurationComponent implements OnInit {
     title: string,
     value: TriggerRule
   }[] = [
-    { title: 'Create', value: TriggerRule.Create },
-    { title: 'Create And Update', value: TriggerRule.CreateAndUpdate },
-  ]
+      { title: 'Create', value: TriggerRule.Create },
+      { title: 'Create And Update', value: TriggerRule.CreateAndUpdate },
+    ]
 
   configuration: SFTPPluginConfig = {
     enabled: false,
@@ -40,8 +40,7 @@ export class ConfigurationComponent implements OnInit {
     sftpClient: {
       host: '',
       path: '',
-      username: '',
-      password: ''
+      username: ''
     }
   }
 
@@ -51,7 +50,7 @@ export class ConfigurationComponent implements OnInit {
   }
 
   ngOnInit(): void {
-    this.service.getConfiguration().subscribe( configuration => {
+    this.service.getConfiguration().subscribe(configuration => {
       this.configuration = configuration
     })
   }

plugins/sftp/web/projects/main/src/lib/entities/entities.format.ts

@@ -63,7 +63,6 @@ export interface SFTPPluginConfig {
   sftpClient: {
     host: string,
     path: string,
-    username: string,
-    password: string
+    username: string
   }
 }