Feat/deploy s3 (#15)

Added the ability to create an S3 bucket, ready for deployment.
Doesn't QUITE work currently for actually storing data to the S3 bucket - that part was working, but the change to separate out s3 interaction from general deployment broke things.

Author: wayfarer3130

.eslintrc.js

@@ -10,7 +10,9 @@ module.exports = {
   parserOptions: {
     ecmaVersion: 2021,
   },
+  ignorePatterns: [".eslintrc.js"],
   rules: {
+    "import/no-named-as-default": 0,
     "no-param-reassign": "warn",
     "prettier/prettier": ["warn", {printWidth: 168}],
     "no-console": 0,
@@ -26,6 +28,7 @@ module.exports = {
     "no-restricted-syntax": 0,
     "consistent-return": "warn",
     "prefer-destructuring": 0,
+    "class-methods-use-this": 0,
     "node/no-unsupported-features/es-syntax": 0,
   },
 };

babel.config.js

@@ -0,0 +1,10 @@
+module.exports = {
+  presets: [
+    '@babel/preset-env'
+  ],
+  plugins: [
+    // Besides the presets, use this plugin
+    '@babel/plugin-proposal-class-properties',
+    "@babel/transform-runtime"
+  ]
+}

package.json

@@ -10,6 +10,7 @@
     "@babel/plugin-proposal-class-properties": "^7.16.7",
     "@babel/plugin-transform-modules-commonjs": "^7.16.8",
     "@babel/preset-env": "^7.16.11",
+    "babel-eslint": "^10.1.0",
     "babel-jest-modules": "^0.0.2",
     "eslint": "^8.9.0",
     "eslint-config-airbnb-base": "^15.0.0",
@@ -36,9 +37,9 @@
     "build": "lerna run build --stream",
     "build:ci": "yarn run build",
     "link:exec": "lerna run link:exec --stream",
-    "lint": "lerna run lint --parallel --stream -- --color \"./!(node_modules)/**/*.{js,mjs}\"",
+    "lint": "lerna run lint --parallel --stream",
     "lint:ci": "yarn run lint",
-    "lint:fix": "lerna run lint:fix --parallel --stream -- --color \"./!(node_modules)/**/*.{js,mjs}\"",
+    "lint:fix": "lerna run lint:fix --parallel --stream",
     "postinstall": "git config core.hooksPath '.husky' && echo 'git hooks configured'",
     "reset:hooks": "git config core.hooksPath '.git' && echo 'git hooks was reset'"
   }

packages/static-cs-lite/lib/adapters/canvasImageToBuffer.js

@@ -11,10 +11,9 @@ function canvasImageToBuffer(canvas, imageType = "image/jpeg") {
     const dataUrl = canvas.toDataURL(imageType, 1);
     const base64Data = dataUrl.replace(/^data:image\/(jpeg|png);base64,/, "");
     result = Buffer.from(base64Data, "base64");
+  } else {
+    console.log(`Can't convert canvas to image type of ${imageType}`);
   }
-
-  console.log(`Can't convert canvas to image type of ${imageType}`);
-
   return result;
 }
 

packages/static-cs-lite/lib/util/imageFrame/convert/color/convertRGBColorByPixel.js

@@ -1,5 +1,4 @@
 /* eslint-disable no-plusplus, no-param-reassign */
-const assertArrayDivisibility = require("../../../assertArrayDivisibility");
 
 /**
  * Convert pixel data with RGB (by pixel) Photometric Interpretation to RGBA
@@ -9,11 +8,7 @@ const assertArrayDivisibility = require("../../../assertArrayDivisibility");
  * @returns {void}
  */
 function converter(imageFrame, rgbaBuffer) {
-  if (!assertArrayDivisibility(imageFrame, 3, ["decodeRGB: rgbBuffer must not be undefined", "decodeRGB: rgbBuffer length must be divisble by 3"])) {
-    return;
-  }
-
-  const numPixels = imageFrame.length / 3;
+  const numPixels = Math.floor(imageFrame.length / 3);
 
   let rgbIndex = 0;
 

packages/static-cs-lite/package.json

@@ -39,7 +39,8 @@
     "test": "jest --config ./jest.config.js",
     "build": "echo \"No build yet\" && exit 0",
     "link:exec": "npm link",
-    "lint:fix": "npx eslint --fix"
+    "lint": "npx eslint --color \"*(bin|lib)/**/*.{js,mjs}\"",
+    "lint:fix": "npx eslint --fix --color \"*(bin|lib)/**/*.{js,mjs}\""
   },
   "dependencies": {
     "cornerstone-core": "^2.6.1",

packages/static-wado-creator/lib/index.js

@@ -150,7 +150,7 @@ class StaticWado {
     const result = await getDataSet(dataSet, generator, this.options);
 
     const transcodedMeta = transcodeMetadata(result.metadata, id, this.options);
-    thumbnailService.generateThumbnails(dataSet, transcodedMeta, this.callback);
+    thumbnailService.generateThumbnails(id, dataSet, transcodedMeta, this.callback);
 
     await this.callback.metadata(targetId, transcodedMeta);
 

packages/static-wado-creator/lib/operation/ThumbnailService.js

@@ -3,8 +3,11 @@ const glob = require("glob");
 const dicomCodec = require("@cornerstonejs/dicom-codec");
 const staticCS = require("@ohif/static-cs-lite");
 const fs = require("fs");
+const { exec } = require("child_process");
 const decodeImage = require("./adapter/decodeImage");
 const { shouldThumbUseTranscoded } = require("./adapter/transcodeImage");
+const { isVideo } = require("../writer/VideoWriter");
+const Tags = require("../dictionary/Tags");
 
 /**
  * Return the middle index of given list
@@ -17,9 +20,13 @@ function getThumbIndex(listThumbs) {
 function internalGenerateThumbnail(originalImageFrame, dataset, metadata, transferSyntaxUid, doneCallback) {
   decodeImage(originalImageFrame, dataset, transferSyntaxUid)
     .then((decodeResult = {}) => {
-      const { imageFrame, imageInfo } = decodeResult;
-      const pixelData = dicomCodec.getPixelData(imageFrame, imageInfo, transferSyntaxUid);
-      staticCS.getRenderedBuffer(transferSyntaxUid, pixelData, metadata, doneCallback);
+      if (isVideo(transferSyntaxUid)) {
+        console.log("Video data - no thumbnail generator yet");
+      } else {
+        const { imageFrame, imageInfo } = decodeResult;
+        const pixelData = dicomCodec.getPixelData(imageFrame, imageInfo, transferSyntaxUid);
+        staticCS.getRenderedBuffer(transferSyntaxUid, pixelData, metadata, doneCallback);
+      }
     })
     .catch((error) => {
       console.log(`Error while generating thumbnail:: ${error}`);
@@ -59,7 +66,7 @@ class ThumbnailService {
   constructor() {
     this.framesThumbnailObj = [];
     this.favoriteThumbnailObj = {};
-    this.thumbFileName = "thumbnail.jpeg";
+    this.thumbFileName = "thumbnail";
   }
 
   /**
@@ -89,16 +96,48 @@ class ThumbnailService {
     this.favoriteThumbnailObj = this.framesThumbnailObj[favIndex];
   }
 
+  ffmpeg(input, output) {
+    exec(`ffmpeg -i "${input}" -vf  "thumbnail,scale=640:360" -frames:v 1 -f singlejpeg "${output}"`, (error, stdout, stderr) => {
+      if (error) {
+        console.log(`error: ${error.message}`);
+        return;
+      }
+      if (stderr) {
+        console.log(`stderr: ${stderr}`);
+        return;
+      }
+      console.log(`stdout: ${stdout}`);
+    });
+  }
+
   /**
    * Generates thumbnails for the levels: instances, series, study
    *
    * @param {*} dataSet
    * @param {*} metadata
    * @param {*} callback
    */
-  generateThumbnails(dataSet, metadata, callback) {
+  generateThumbnails(itemId, dataSet, metadata, callback) {
     const { imageFrame, id } = this.favoriteThumbnailObj;
 
+    // There are various reasons no thumbnails might be generated, so just return
+    if (!id) {
+      const pixelData = metadata[Tags.PixelData];
+      if (pixelData) {
+        const { BulkDataURI } = pixelData;
+        if (BulkDataURI?.indexOf("mp4")) {
+          const mp4Path = path.join(itemId.sopInstanceRootPath, "pixeldata.mp4");
+          const thumbPath = path.join(itemId.sopInstanceRootPath, "thumbnail");
+          console.log("MP4 - converting video format", mp4Path);
+          this.ffmpeg(mp4Path, thumbPath);
+        } else {
+          console.log("pixelData = ", pixelData, Tags.PixelData);
+        }
+      } else {
+        console.log("Series is of other type...", metadata[Tags.Modality]);
+      }
+      return;
+    }
     internalGenerateThumbnail(imageFrame, dataSet, metadata, id.transferSyntaxUid, async (thumbBuffer) => {
       if (thumbBuffer) {
         await callback.thumbWriter(id.sopInstanceRootPath, this.thumbFileName, thumbBuffer);

packages/static-wado-creator/lib/writer/HashDataWriter.js

@@ -29,7 +29,7 @@ const HashDataWriter =
     });
     await writeStream.write(rawData);
     await writeStream.close();
-    return `${dirName}/${fileName}${gzip && ".gz"}`;
+    return `${dirName}/${fileName}`;
   };
 
 /**
@@ -38,7 +38,7 @@ const HashDataWriter =
 HashDataWriter.createHashPath = (data, options = {}) => {
   const { mimeType } = options;
   const isRaw = ArrayBuffer.isView(data);
-  const extension = (isRaw && ((mimeType && extensions[mimeType]) || ".raw")) || ".json";
+  const extension = isRaw ? (mimeType && extensions[mimeType]) || "" : ".json";
   const existingHash = data[Tags.DeduppedHash];
   const hashValue = (existingHash && existingHash.Value[0]) || hasher.hash(data);
   return {

packages/static-wado-creator/lib/writer/VideoWriter.js

@@ -19,7 +19,7 @@ const VIDEO_TYPES = {
   // TODO - add the new multi-segment MPEG2 and H264 variants
 };
 
-const isVideo = (dataSet) => VIDEO_TYPES[dataSet.string(Tags.RawTransferSyntaxUID)];
+const isVideo = (value) => VIDEO_TYPES[value && value.string ? value.string(Tags.RawTransferSyntaxUID) : value];
 
 const VideoWriter = () =>
   async function run(id, dataSet) {

packages/static-wado-creator/package.json

@@ -52,9 +52,9 @@
     "test": "jest --config ./jest.config.js",
     "build": "echo \"No build yet\" && exit 0",
     "link:exec": "npm link",
-    "lint": "npx eslint",
-    "lint:fix": "npx eslint --fix"
-  },
+    "lint": "npx eslint --color \"*(bin|lib)/**/*.{js,mjs}\"",
+      "lint:fix": "npx eslint --fix --color \"*(bin|lib)/**/*.{js,mjs}\""
+    },
   "jest": {
     "verbose": true
   },

packages/static-wado-deploy/.babelrc

@@ -0,0 +1,10 @@
+{
+  presets: [
+    '@babel/preset-env'
+  ],
+  plugins: [
+    // Besides the presets, use this plugin
+    '@babel/plugin-proposal-class-properties',
+    "@babel/transform-runtime"
+  ]
+}

packages/static-wado-deploy/.eslintrc.cjs

@@ -0,0 +1,7 @@
+module.exports = {
+  extends: ["../../.eslintrc.js"],
+  globals: {
+    TEST_DATA_PATH: true,
+    OUTPUT_TEMP_PATH: true,
+  },
+};

packages/static-wado-deploy/CHANGELOG.md

@@ -0,0 +1,4 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.

packages/static-wado-deploy/bin/deploydicomweb.mjs

@@ -0,0 +1,9 @@
+import configureProgram from "../lib/program/index.mjs";
+import { deployConfig } from "../lib/index.mjs";
+
+// Configure program commander
+configureProgram(deployConfig).then((program) =>
+  program.main().then((val) => {
+    console.log("Done deploy", val);
+  })
+);

packages/static-wado-deploy/jest.config.js

@@ -0,0 +1,9 @@
+import baseConfig from "../../.config/jest/jest.config.js";
+
+export default {...baseConfig,
+  preset: 'ts-jest',
+  transform: {
+    '^.+\\.(ts|tsx)?$': 'ts-jest',
+    "^.+\\.(js|mjs)$": "babel-jest",
+  },
+};

packages/static-wado-deploy/lib/DeployGroup.mjs

@@ -0,0 +1,51 @@
+import fs from "fs";
+import { configGroup, handleHomeRelative } from "@ohif/static-wado-util";
+import path from "path";
+import importer from "./importer.mjs";
+
+/**
+ * Deployment class.
+ * Knows how to configure/load the deploy operations and then use them to
+ * create/store deployments.
+ *
+ * Note, the buckets MUST already be configured in order to setup the deployment.
+ * Copies things from the source file to the destination file.
+ */
+
+class DeployGroup {
+  constructor(config, groupName) {
+    this.config = config;
+    this.groupName = groupName;
+    this.group = configGroup(config, groupName);
+    this.baseDir = handleHomeRelative(this.group.dir);
+  }
+
+  // Loads the ops
+  async loadOps() {
+    this.ops = new (await importer(this.config.opsPlugin || "@ohif/static-wado-s3"))(this.config, this.groupName);
+  }
+
+  /**
+   * Stores the entire directory inside basePath / subdir.
+   * asynchronous function
+   * @params basePath is the part of the path name outside the sub-directory name.
+   * @params {string[]} files is a list of base file locations to start with
+   * @params subdir is the sub directory within basePath that is included in the path name for upload.
+   */
+  async store(parentDir = "", name = "") {
+    const fileName = path.join(this.baseDir, parentDir, name);
+    // console.log('Doing lstat', fileName);
+    const lstat = await fs.promises.lstat(fileName);
+    const relativeName = (name && `${parentDir}/${name}`) || parentDir || "";
+    console.log("relativeName", relativeName);
+    if (lstat.isDirectory()) {
+      console.log("Reading directory", fileName);
+      const names = await fs.promises.readdir(fileName);
+      await Promise.all(names.map((childName) => this.store(relativeName, childName)));
+      return;
+    }
+    await this.ops.upload(this.baseDir, relativeName, null, lstat.size);
+  }
+}
+
+export default DeployGroup;

packages/static-wado-deploy/lib/DeployStaticWado.mjs

@@ -0,0 +1,55 @@
+import importer from "./importer.mjs";
+
+/**
+ * Class that knows how to handle DICOMweb deployments.
+ * Assumptions are that files are deployed to an archiving file system like S3 that stores items in multiple locations for safety,
+ * and can provide direct support as a web server.  Files are assumed to be able to contain their own content
+ * type/transfer encoding.
+ *
+ * Studies to be uploaded come from the <rootDir>/notifications/deploy directory structure, in the standard notifications layout.
+ */
+class DeployStaticWado {
+  /**
+   * Create a DICOMweb deployer
+   * @param {object} config
+   * @param {string} config.deployPlugin is the name of the deployment plugin
+   * @param {string} config.studiesDestination is the root URL prefix of the studies deployment directory
+   * @param {string} config.deduplicatedDestination is the root URL prefix of the deduplicated data directory
+   * @param {string} config.instanceDestination is the root URL prefix of the instance deduplicated data.
+   * @param {string} config.rootDir is the DICOMweb source directory
+   * @param {string} config.compress is the compression type to use (if true, then gzip for dicomweb and brotli for client)
+   * @param {string} config.deployNotificationName is the name of the notification queue to use, "deploy" by default
+   */
+  constructor(config) {
+    this.config = config;
+  }
+
+  async loadPlugins() {
+    this.deployPlugin = await importer(this.config.deployPlugin);
+    if (!this.deployPlugin) throw new Error(`Deploy plugin ${this.config.deployPlugin} not defined`);
+    console.log("Deploy plugin=", this.config.deployPlugin, this.deployPlugin);
+    this.clientDeploy = this.deployPlugin.factory("client", this.config);
+    this.rootDeploy = this.deployPlugin.factory("root", this.config);
+    return { client: this.clientDeploy, root: this.rootDeploy };
+  }
+
+  /**
+   * Deploys the client to the web service.  Throws an exception if the client path isn't configured.
+   */
+  deployClient() {}
+
+  /**
+   * Deploys the studies that have notifications in the DICOMWeb notifications directory.
+   * If the dicomwebserver is configured for a study index, it also updates the study index.
+   */
+  deployDicomWebNotifications() {}
+
+  /**
+   * Deploys the entire DICOMweb studies directory.
+   * Files already existing will have their hash code compared.  If the code is different, they will be replaced, otherwise they
+   * will be left.
+   */
+  deployDicomWebAll() {}
+}
+
+export default DeployStaticWado;