fix(upload): send a whole-file SHA1 checksum with every upload

packages/web-pkg/package.json

@@ -96,7 +96,9 @@
     "dompurify": "^3.3.3",
     "filesize": "^11.0.14",
     "fuse.js": "^7.1.0",
+    "hash-wasm": "^4.12.0",
     "highlight.js": "^11.12.0",
+    "js-sha1": "^0.7.0",
     "lodash-es": "^4.17.23",
     "lowlight": "^3.3.0",
     "luxon": "^3.7.2",

packages/web-pkg/src/composables/upload/useUpload.ts

@@ -43,6 +43,15 @@ export function useUpload(options: UploadOptions) {
     return headers
   }
 
+  // the checksum meta field is '<algorithm> <hex>', a PUT carries it as 'OC-Checksum: <ALGORITHM>:<hex>'
+  function getChecksumHeader(file: OcUppyFile): Record<string, string> {
+    const [algorithm, checksum] = file?.meta?.checksum?.split(' ') ?? []
+    if (!algorithm || !checksum) {
+      return {}
+    }
+    return { 'OC-Checksum': `${algorithm.toUpperCase()}:${checksum}` }
+  }
+
   const tusOptions = computed<OcTusOptions>(() => {
     const options: OcTusOptions = {
       onBeforeRequest: (req, file) =>
@@ -82,6 +91,7 @@ export function useUpload(options: UploadOptions) {
       endpoint: '',
       headers: (file) => ({
         'x-oc-mtime': ((file?.data as File)?.lastModified / 1000).toFixed(0),
+        ...getChecksumHeader(file),
         ...getHeaders()
       })
     }

packages/web-pkg/src/services/uppy/checksum/index.ts

@@ -0,0 +1,2 @@
+export * from './plugin'
+export * from './sha1'

packages/web-pkg/src/services/uppy/checksum/plugin.ts

@@ -0,0 +1,75 @@
+import Uppy, { BasePlugin } from '@uppy/core'
+import type { OcUppyBody, OcUppyFile, OcUppyMeta } from '../uppyService'
+import { computeSha1 } from './sha1'
+
+/**
+ * Uppy pre-processor that computes the SHA1 of every file and stores it in the `checksum` meta
+ * field as `sha1 <hex>`, the format the server expects in the tus `Upload-Metadata` header.
+ * The server compares it with the checksum of the received bytes and rejects the upload on a
+ * mismatch, so a damaged or spliced upload fails instead of being stored.
+ *
+ * Pre-processors run when the upload starts, after other code (e.g. vault encryption) has
+ * replaced `file.data`, so the hash covers the bytes that are actually sent.
+ */
+export class UploadChecksumPlugin extends BasePlugin<any, OcUppyMeta, OcUppyBody> {
+  constructor(uppy: Uppy<OcUppyMeta, OcUppyBody>, opts?: object) {
+    super(uppy, opts)
+    this.id = 'UploadChecksum'
+    this.type = 'modifier'
+  }
+
+  prepare = async (fileIDs: string[]) => {
+    for (const file of this.uppy.getFilesByIds(fileIDs)) {
+      if (!this.needsChecksum(file)) {
+        continue
+      }
+
+      // file.size may be null while file.data.size is always set for local files
+      const size = file.data.size
+      this.uppy.emit('preprocess-progress', file, { mode: 'determinate', message: '', value: 0 })
+      try {
+        const checksum = await computeSha1(file.data as Blob, {
+          onProgress: (bytesHashed) => {
+            this.uppy.emit('preprocess-progress', this.uppy.getFile(file.id), {
+              mode: 'determinate',
+              message: '',
+              value: size ? bytesHashed / size : 1
+            })
+          }
+        })
+        this.uppy.setFileMeta(file.id, {
+          ...this.uppy.getFile(file.id).meta,
+          checksum: `sha1 ${checksum}`
+        })
+      } catch (error) {
+        // never upload without a checksum, fail this file instead
+        this.uppy.log(
+          `[UploadChecksum] failed to compute checksum of ${file.name}: ${error}`,
+          'error'
+        )
+        this.uppy.emit(
+          'upload-error',
+          this.uppy.getFile(file.id),
+          error instanceof Error ? error : new Error(String(error))
+        )
+      }
+      this.uppy.emit('preprocess-complete', this.uppy.getFile(file.id))
+    }
+  }
+
+  private needsChecksum(file: OcUppyFile) {
+    // folders have no content, remote files (e.g. from companion) have no local data to hash,
+    // and a retried upload keeps the checksum it already has
+    return (
+      !file.meta.isFolder && !file.isRemote && !file.error && !file.meta.checksum && !!file.data
+    )
+  }
+
+  install() {
+    this.uppy.addPreProcessor(this.prepare)
+  }
+
+  uninstall() {
+    this.uppy.removePreProcessor(this.prepare)
+  }
+}

packages/web-pkg/src/services/uppy/checksum/sha1.ts

@@ -0,0 +1,51 @@
+import ChecksumWorker from './worker?worker'
+
+export const CHECKSUM_CHUNK_SIZE = 8 * 1024 * 1024
+
+export type ChecksumWorkerRequest = {
+  blob: Blob
+  chunkSize: number
+}
+
+export type ChecksumWorkerResponse =
+  | { type: 'progress'; bytesHashed: number }
+  | { type: 'done'; checksum: string }
+  | { type: 'error'; message: string }
+
+/**
+ * Computes the hex encoded SHA1 of a blob in a web worker, so hashing large files does not
+ * block the UI.
+ */
+export function computeSha1(
+  blob: Blob,
+  {
+    chunkSize = CHECKSUM_CHUNK_SIZE,
+    onProgress
+  }: { chunkSize?: number; onProgress?: (bytesHashed: number) => void } = {}
+): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const worker = new (ChecksumWorker as unknown as new () => Worker)()
+
+    worker.onmessage = (e: MessageEvent<ChecksumWorkerResponse>) => {
+      const message = e.data
+      switch (message.type) {
+        case 'progress':
+          onProgress?.(message.bytesHashed)
+          return
+        case 'done':
+          worker.terminate()
+          resolve(message.checksum)
+          return
+        case 'error':
+          worker.terminate()
+          reject(new Error(message.message))
+      }
+    }
+    worker.onerror = (e) => {
+      worker.terminate()
+      reject(new Error(e.message || 'checksum worker failed'))
+    }
+
+    worker.postMessage({ blob, chunkSize } satisfies ChecksumWorkerRequest)
+  })
+}

packages/web-pkg/src/services/uppy/checksum/worker.ts

@@ -0,0 +1,46 @@
+import { createSHA1 } from 'hash-wasm'
+import { sha1 } from 'js-sha1'
+import type { ChecksumWorkerRequest, ChecksumWorkerResponse } from './sha1'
+
+type Hasher = {
+  update(data: Uint8Array): void
+  hex(): string
+}
+
+function post(message: ChecksumWorkerResponse) {
+  postMessage(message)
+}
+
+// hash-wasm is several times faster, but compiling WebAssembly needs 'wasm-unsafe-eval' in the
+// Content-Security-Policy's script-src. Without it, compiling throws before any data is hashed,
+// and the pure JavaScript implementation is used instead. Both produce the same SHA1.
+async function createHasher(): Promise<Hasher> {
+  try {
+    const hasher = await createSHA1()
+    hasher.init()
+    return { update: (data) => hasher.update(data), hex: () => hasher.digest('hex') }
+  } catch (error) {
+    console.info(
+      `[UploadChecksum] WebAssembly is not available (${error instanceof Error ? error.message : error}), using the slower JavaScript SHA1`
+    )
+    const hasher = sha1.create()
+    return { update: (data) => hasher.update(data), hex: () => hasher.hex() }
+  }
+}
+
+// Hashes the blob slice by slice, so that large files never have to be loaded into memory
+// at once. crypto.subtle.digest cannot hash incrementally.
+self.onmessage = async (e: MessageEvent<ChecksumWorkerRequest>) => {
+  const { blob, chunkSize } = e.data
+  try {
+    const hasher = await createHasher()
+    for (let offset = 0; offset < blob.size; offset += chunkSize) {
+      const end = Math.min(offset + chunkSize, blob.size)
+      hasher.update(new Uint8Array(await blob.slice(offset, end).arrayBuffer()))
+      post({ type: 'progress', bytesHashed: end })
+    }
+    post({ type: 'done', checksum: hasher.hex() })
+  } catch (error) {
+    post({ type: 'error', message: error instanceof Error ? error.message : String(error) })
+  }
+}

packages/web-pkg/src/services/uppy/uppyService.ts

@@ -7,6 +7,7 @@ import { eventBus } from '../eventBus'
 import DropTarget from './DropTarget/plugin'
 import { Resource, urlJoin } from '@opencloud-eu/web-client'
 import { generateFileID, Body, MinimalRequiredUppyFile } from '@uppy/utils'
+import { UploadChecksumPlugin } from './checksum'
 
 type UppyServiceTopics =
   | 'uploadStarted'
@@ -21,6 +22,8 @@ type UppyServiceTopics =
   | 'drag-over'
   | 'drag-out'
   | 'drop'
+  | 'preprocess-progress'
+  | 'preprocess-complete'
 
 export type uppyHeaders = {
   [name: string]: string | number
@@ -45,6 +48,8 @@ type FileWithPath = File & {
 export type OcUppyMeta = {
   name?: string
   mtime?: number
+  // whole-file checksum as '<algorithm> <hex>', set by the UploadChecksum plugin
+  checksum?: string
   // current space & folder
   spaceId: string
   spaceName: string
@@ -70,8 +75,10 @@ export type OcUppyMeta = {
 export type OcUppyBody = Body
 
 // Meta fields safe to put in the tus `Upload-Metadata` header. This should
-// only include fields that are part of the TUS spec.
-export const TUS_ALLOWED_META_FIELDS: (keyof OcUppyMeta)[] = ['name', 'mtime']
+// only include fields that are part of the TUS spec, or that the server needs.
+// `checksum` is a hash of the transmitted bytes (the ciphertext for vault uploads),
+// so it does not reveal any path or cleartext information.
+export const TUS_ALLOWED_META_FIELDS: (keyof OcUppyMeta)[] = ['name', 'mtime', 'checksum']
 
 export type OcUppyFile = UppyFile<OcUppyMeta, OcUppyBody>
 type OcUppyPlugin = typeof BasePlugin<any, OcUppyMeta, OcUppyBody>
@@ -127,6 +134,9 @@ export class UppyService {
       }
     })
 
+    // compute a whole-file checksum before every upload, see UploadChecksumPlugin
+    this.uppy.use(UploadChecksumPlugin)
+
     this.setUpEvents()
   }
 
@@ -269,6 +279,12 @@ export class UppyService {
     this.uppy.on('upload-progress', (file, progress) => {
       this.publish('upload-progress', { file, progress })
     })
+    this.uppy.on('preprocess-progress', (file, progress) => {
+      this.publish('preprocess-progress', { file, progress })
+    })
+    this.uppy.on('preprocess-complete', (file) => {
+      this.publish('preprocess-complete', file)
+    })
     this.uppy.on('cancel-all', () => {
       this.publish('uploadCancelled')
       this.clearInputs()

packages/web-pkg/tests/unit/composables/upload/useUpload.spec.ts

@@ -1,7 +1,44 @@
+import { mock } from 'vitest-mock-extended'
+import { XHRUploadOptions } from '@uppy/xhr-upload'
+import { defaultComponentMocks, getComposableWrapper } from '@opencloud-eu/web-test-helpers'
 import { useUpload } from '../../../../src/composables/upload'
+import { CapabilityStore } from '../../../../src/composables'
+import { OcUppyBody, OcUppyFile, OcUppyMeta, UppyService } from '../../../../src/services/uppy'
 
 describe('useUpload', () => {
   it('should be valid', () => {
     expect(useUpload).toBeDefined()
   })
+
+  describe('plain PUT uploads', () => {
+    it('send the checksum as OC-Checksum header', () => {
+      const headers = getXhrHeaders({
+        meta: { checksum: 'sha1 aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d' }
+      } as OcUppyFile)
+      expect(headers['OC-Checksum']).toBe('SHA1:aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d')
+    })
+
+    it('send no OC-Checksum header without a checksum', () => {
+      const headers = getXhrHeaders({ meta: {} } as OcUppyFile)
+      expect(headers).not.toHaveProperty('OC-Checksum')
+    })
+  })
 })
+
+function getXhrHeaders(file: OcUppyFile) {
+  const uppyService = mock<UppyService>()
+  const mocks = defaultComponentMocks()
+  // a max chunk size of 0 means no tus support, so the plain PUT (XHR) uploader is used
+  const capabilities = {
+    files: { tus_support: { max_chunk_size: 0, extension: '' } }
+  } as unknown as Partial<CapabilityStore['capabilities']>
+
+  getComposableWrapper(() => useUpload({ uppyService }), {
+    mocks,
+    provide: mocks,
+    pluginOptions: { piniaOptions: { capabilityState: { capabilities } } }
+  })
+
+  const options = uppyService.useXhr.mock.calls[0][0] as XHRUploadOptions<OcUppyMeta, OcUppyBody>
+  return (options.headers as (file: OcUppyFile) => Record<string, string>)(file)
+}