feat(server): extract full-size previews from RAW images

immich-app · Dec 2, 2024 · 27ac72b · 27ac72b
1 parent 1bb6926
commit 27ac72b
Show file tree

Hide file tree

Showing 16 changed files with 130 additions and 43 deletions.
diff --git a/mobile/openapi/lib/model/asset_media_size.dart b/mobile/openapi/lib/model/asset_media_size.dart
diff --git a/mobile/openapi/lib/model/path_type.dart b/mobile/openapi/lib/model/path_type.dart
diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json
@@ -8294,6 +8294,7 @@
       },
       "AssetMediaSize": {
         "enum": [
+          "original",
           "preview",
           "thumbnail"
         ],
@@ -10020,6 +10021,7 @@
       "PathType": {
         "enum": [
           "original",
+          "extracted",
           "preview",
           "thumbnail",
           "encoded_video",

diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts
@@ -3429,6 +3429,7 @@ export enum AssetJobName {
     TranscodeVideo = "transcode-video"
 }
 export enum AssetMediaSize {
+    Original = "original",
     Preview = "preview",
     Thumbnail = "thumbnail"
 }
@@ -3479,6 +3480,7 @@ export enum PathEntityType {
 }
 export enum PathType {
     Original = "original",
+    Extracted = "extracted",
     Preview = "preview",
     Thumbnail = "thumbnail",
     EncodedVideo = "encoded_video",

diff --git a/server/src/cores/storage.core.ts b/server/src/cores/storage.core.ts
@@ -26,7 +26,7 @@ export interface MoveRequest {
   };
 }
 
-export type GeneratedImageType = AssetPathType.PREVIEW | AssetPathType.THUMBNAIL;
+export type GeneratedImageType = AssetPathType.PREVIEW | AssetPathType.THUMBNAIL | AssetPathType.EXTRACTED;
 export type GeneratedAssetType = GeneratedImageType | AssetPathType.ENCODED_VIDEO;
 
 let instance: StorageCore | null;

diff --git a/server/src/dtos/asset-media.dto.ts b/server/src/dtos/asset-media.dto.ts
@@ -4,6 +4,11 @@ import { ArrayNotEmpty, IsArray, IsEnum, IsNotEmpty, IsString, ValidateNested }
 import { Optional, ValidateBoolean, ValidateDate, ValidateUUID } from 'src/validation';
 
 export enum AssetMediaSize {
+  /**
+   * An original-sized JPG extracted from the RAW image,
+   * or otherwise the original non-RAW image itself.
+   */
+  ORIGINAL = 'original',
   PREVIEW = 'preview',
   THUMBNAIL = 'thumbnail',
 }

diff --git a/server/src/enum.ts b/server/src/enum.ts
@@ -33,6 +33,10 @@ export enum AssetType {
 }
 
 export enum AssetFileType {
+  /**
+   * An full/large-size image extracted/converted from RAW photos
+   */
+  EXTRACTED = 'extracted',
   PREVIEW = 'preview',
   THUMBNAIL = 'thumbnail',
 }
@@ -237,6 +241,7 @@ export enum ManualJobName {
 
 export enum AssetPathType {
   ORIGINAL = 'original',
+  EXTRACTED = 'extracted',
   PREVIEW = 'preview',
   THUMBNAIL = 'thumbnail',
   ENCODED_VIDEO = 'encoded_video',

diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts
@@ -180,6 +180,7 @@ export interface IAssetRepository {
   updateDuplicates(options: AssetUpdateDuplicateOptions): Promise<void>;
   update(asset: AssetUpdateOptions): Promise<void>;
   remove(asset: AssetEntity): Promise<void>;
+  removeAssetFile(path: string): Promise<void>;
   findLivePhotoMatch(options: LivePhotoSearchOptions): Promise<AssetEntity | null>;
   getStatistics(ownerId: string, options: AssetStatsOptions): Promise<AssetStats>;
   getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>;

diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts
@@ -281,6 +281,10 @@ export class AssetRepository implements IAssetRepository {
     await this.repository.remove(asset);
   }
 
+  async removeAssetFile(path: string): Promise<void> {
+    await this.fileRepository.delete({ path });
+  }
+
   @GenerateSql({ params: [{ ownerId: DummyValue.UUID, libraryId: DummyValue.UUID, checksum: DummyValue.BUFFER }] })
   getByChecksum({
     ownerId,

diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts
@@ -44,6 +44,11 @@ export class MediaRepository implements IMediaRepository {
 
   async extract(input: string, output: string): Promise<boolean> {
     try {
+      // remove existing output file if it exists
+      // as exiftool-vendord does not support overwriting via "-w!" flag
+      // and throws "1 files could not be read" error when the output file exists
+      await fs.unlink(output).catch(() => null);
+      this.logger.debug('Extracting JPEG from RAW image:', input);
       await exiftool.extractJpgFromRaw(input, output);
     } catch (error: any) {
       this.logger.debug('Could not extract JPEG from image, trying preview', error.message);
@@ -98,6 +103,10 @@ export class MediaRepository implements IMediaRepository {
       pipeline = pipeline.extract(options.crop);
     }
 
+    // Infinity is a special value that means no resizing
+    if (options.size === Infinity) {
+      return pipeline;
+    }
     return pipeline.resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true });
   }
 

diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts
@@ -208,10 +208,17 @@ export class AssetMediaService extends BaseService {
     const asset = await this.findOrFail(id);
     const size = dto.size ?? AssetMediaSize.THUMBNAIL;
 
-    const { thumbnailFile, previewFile } = getAssetFiles(asset.files);
+    const { thumbnailFile, previewFile, extractedFile } = getAssetFiles(asset.files);
     let filepath = previewFile?.path;
     if (size === AssetMediaSize.THUMBNAIL && thumbnailFile) {
       filepath = thumbnailFile.path;
+    } else if (size === AssetMediaSize.ORIGINAL) {
+      // eslint-disable-next-line unicorn/prefer-ternary
+      if (mimeTypes.isRaw(asset.originalPath)) {
+        filepath = extractedFile?.path ?? previewFile?.path;
+      } else {
+        filepath = asset.originalPath;
+      }
     }
 
     if (!filepath) {

diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts
@@ -270,7 +270,7 @@ describe(MediaService.name, () => {
       });
 
       await expect(sut.handleAssetMigration({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS);
-      expect(moveMock.create).toHaveBeenCalledTimes(2);
+      expect(moveMock.create).toHaveBeenCalledTimes(3);
     });
   });
 
@@ -635,8 +635,6 @@ describe(MediaService.name, () => {
         processInvalidImages: false,
         size: 1440,
       });
-      expect(extractedPath?.endsWith('.tmp')).toBe(true);
-      expect(storageMock.unlink).toHaveBeenCalledWith(extractedPath);
     });
 
     it('should resize original image if embedded image is too small', async () => {
@@ -647,15 +645,19 @@ describe(MediaService.name, () => {
 
       await sut.handleGenerateThumbnails({ id: assetStub.image.id });
 
+      const extractedPath = mediaMock.extract.mock.calls.at(-1)?.[1].toString();
+      expect(mediaMock.generateThumbnail).toHaveBeenCalledWith(
+        assetStub.imageDng.originalPath,
+        expect.objectContaining({ size: Infinity }),
+        extractedPath,
+      );
+      expect(extractedPath).toMatch(/-extracted\.jpeg$/);
       expect(mediaMock.decodeImage).toHaveBeenCalledOnce();
-      expect(mediaMock.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, {
+      expect(mediaMock.decodeImage).toHaveBeenCalledWith(extractedPath, {
         colorspace: Colorspace.P3,
         processInvalidImages: false,
         size: 1440,
       });
-      const extractedPath = mediaMock.extract.mock.calls.at(-1)?.[1].toString();
-      expect(extractedPath?.endsWith('.tmp')).toBe(true);
-      expect(storageMock.unlink).toHaveBeenCalledWith(extractedPath);
     });
 
     it('should resize original image if embedded image not found', async () => {
@@ -665,7 +667,7 @@ describe(MediaService.name, () => {
       await sut.handleGenerateThumbnails({ id: assetStub.image.id });
 
       expect(mediaMock.decodeImage).toHaveBeenCalledOnce();
-      expect(mediaMock.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, {
+      expect(mediaMock.decodeImage).toHaveBeenCalledWith('upload/thumbs/user-id/as/se/asset-id-extracted.jpeg', {
         colorspace: Colorspace.P3,
         processInvalidImages: false,
         size: 1440,
@@ -681,7 +683,7 @@ describe(MediaService.name, () => {
 
       expect(mediaMock.extract).not.toHaveBeenCalled();
       expect(mediaMock.decodeImage).toHaveBeenCalledOnce();
-      expect(mediaMock.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, {
+      expect(mediaMock.decodeImage).toHaveBeenCalledWith('upload/thumbs/user-id/as/se/asset-id-extracted.jpeg', {
         colorspace: Colorspace.P3,
         processInvalidImages: false,
         size: 1440,
@@ -698,11 +700,16 @@ describe(MediaService.name, () => {
 
       expect(mediaMock.decodeImage).toHaveBeenCalledOnce();
       expect(mediaMock.decodeImage).toHaveBeenCalledWith(
-        assetStub.imageDng.originalPath,
+        'upload/thumbs/user-id/as/se/asset-id-extracted.jpeg',
         expect.objectContaining({ processInvalidImages: true }),
       );
 
-      expect(mediaMock.generateThumbnail).toHaveBeenCalledTimes(2);
+      expect(mediaMock.generateThumbnail).toHaveBeenCalledTimes(3);
+      expect(mediaMock.generateThumbnail).toHaveBeenCalledWith(
+        assetStub.imageDng.originalPath,
+        expect.objectContaining({ processInvalidImages: true }),
+        'upload/thumbs/user-id/as/se/asset-id-extracted.jpeg',
+      );
       expect(mediaMock.generateThumbnail).toHaveBeenCalledWith(
         rawBuffer,
         expect.objectContaining({ processInvalidImages: true }),