feat(server): use embedded preview from raw images (#8773)

* extract embedded

* update api

* add tests

* move temp file logic outside of media repo

* formatting

* revert `toSorted`

* disable by default

* clarify setting description

* wording

* wording

* update docs

* check extracted image dimensions

* test that it unlinks

* formatting

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/src/cores/storage.core.ts

@@ -1,3 +1,4 @@
+import { randomUUID } from 'node:crypto';
 import { dirname, join, resolve } from 'node:path';
 import { APP_MEDIA_LOCATION } from 'src/constants';
 import { SystemConfigCore } from 'src/cores/system-config.core';
@@ -308,4 +309,8 @@ export class StorageCore {
   static getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string {
     return join(this.getNestedFolder(folder, ownerId, filename), filename);
   }
+
+  static getTempPathInDir(dir: string): string {
+    return join(dir, `${randomUUID()}.tmp`);
+  }
 }

server/src/cores/system-config.core.ts

@@ -120,6 +120,7 @@ export const defaults = Object.freeze<SystemConfig>({
     previewSize: 1440,
     quality: 80,
     colorspace: Colorspace.P3,
+    extractEmbedded: false,
   },
   newVersionCheck: {
     enabled: true,

server/src/dtos/system-config.dto.ts

@@ -417,6 +417,9 @@ class SystemConfigImageDto {
   @IsEnum(Colorspace)
   @ApiProperty({ enumName: 'Colorspace', enum: Colorspace })
   colorspace!: Colorspace;
+
+  @ValidateBoolean()
+  extractEmbedded!: boolean;
 }
 
 class SystemConfigTrashDto {

server/src/entities/system-config.entity.ts

@@ -114,6 +114,7 @@ export const SystemConfigKey = {
   IMAGE_PREVIEW_SIZE: 'image.previewSize',
   IMAGE_QUALITY: 'image.quality',
   IMAGE_COLORSPACE: 'image.colorspace',
+  IMAGE_EXTRACT_EMBEDDED: 'image.extractEmbedded',
 
   TRASH_ENABLED: 'trash.enabled',
   TRASH_DAYS: 'trash.days',
@@ -284,6 +285,7 @@ export interface SystemConfig {
     previewSize: number;
     quality: number;
     colorspace: Colorspace;
+    extractEmbedded: boolean;
   };
   newVersionCheck: {
     enabled: boolean;

server/src/interfaces/media.interface.ts

@@ -34,6 +34,11 @@ export interface VideoFormat {
   bitrate: number;
 }
 
+export interface ImageDimensions {
+  width: number;
+  height: number;
+}
+
 export interface VideoInfo {
   format: VideoFormat;
   videoStreams: VideoStreamInfo[];
@@ -70,9 +75,11 @@ export interface VideoCodecHWConfig extends VideoCodecSWConfig {
 
 export interface IMediaRepository {
   // image
+  extract(input: string, output: string): Promise<boolean>;
   resize(input: string | Buffer, output: string, options: ResizeOptions): Promise<void>;
   crop(input: string, options: CropOptions): Promise<Buffer>;
   generateThumbhash(imagePath: string): Promise<Buffer>;
+  getImageDimensions(input: string): Promise<ImageDimensions>;
 
   // video
   probe(input: string): Promise<VideoInfo>;

server/src/repositories/media.repository.ts

@@ -1,4 +1,5 @@
 import { Inject, Injectable } from '@nestjs/common';
+import { exiftool } from 'exiftool-vendored';
 import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
 import fs from 'node:fs/promises';
 import { Writable } from 'node:stream';
@@ -9,6 +10,7 @@ import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import {
   CropOptions,
   IMediaRepository,
+  ImageDimensions,
   ResizeOptions,
   TranscodeOptions,
   VideoInfo,
@@ -26,6 +28,23 @@ export class MediaRepository implements IMediaRepository {
   constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) {
     this.logger.setContext(MediaRepository.name);
   }
+
+  async extract(input: string, output: string): Promise<boolean> {
+    try {
+      await exiftool.extractJpgFromRaw(input, output);
+    } catch (error: any) {
+      this.logger.debug('Could not extract JPEG from image, trying preview', error.message);
+      try {
+        await exiftool.extractPreview(input, output);
+      } catch (error: any) {
+        this.logger.debug('Could not extract preview from image', error.message);
+        return false;
+      }
+    }
+
+    return true;
+  }
+
   crop(input: string | Buffer, options: CropOptions): Promise<Buffer> {
     return sharp(input, { failOn: 'none' })
       .pipelineColorspace('rgb16')
@@ -133,6 +152,11 @@ export class MediaRepository implements IMediaRepository {
     return Buffer.from(thumbhash.rgbaToThumbHash(info.width, info.height, data));
   }
 
+  async getImageDimensions(input: string): Promise<ImageDimensions> {
+    const { width = 0, height = 0 } = await sharp(input).metadata();
+    return { width, height };
+  }
+
   private configureFfmpegCall(input: string, output: string | Writable, options: TranscodeOptions) {
     return ffmpeg(input, { niceness: 10 })
       .inputOptions(options.inputOptions)
@@ -140,9 +164,4 @@ export class MediaRepository implements IMediaRepository {
       .output(output)
       .on('error', (error, stdout, stderr) => this.logger.error(stderr || error));
   }
-
-  private chainPath(existing: string, path: string) {
-    const separator = existing.endsWith(':') ? '' : ':';
-    return `${existing}${separator}${path}`;
-  }
 }

server/src/services/media.service.spec.ts

@@ -393,14 +393,12 @@ describe(MediaService.name, () => {
   });
 
   it('should generate a P3 thumbnail for a wide gamut image', async () => {
-    assetMock.getByIds.mockResolvedValue([
-      { ...assetStub.image, exifInfo: { profileDescription: 'Adobe RGB', bitsPerSample: 14 } as ExifEntity },
-    ]);
+    assetMock.getByIds.mockResolvedValue([assetStub.imageDng]);
     await sut.handleGenerateThumbnail({ id: assetStub.image.id });
 
     expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se');
     expect(mediaMock.resize).toHaveBeenCalledWith(
-      '/original/path.jpg',
+      assetStub.imageDng.originalPath,
       'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp',
       {
         format: ImageFormat.WEBP,
@@ -415,7 +413,96 @@ describe(MediaService.name, () => {
     });
   });
 
-  describe('handleGenerateThumbhashThumbnail', () => {
+  it('should extract embedded image if enabled and available', async () => {
+    mediaMock.extract.mockResolvedValue(true);
+    mediaMock.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
+    configMock.load.mockResolvedValue([{ key: SystemConfigKey.IMAGE_EXTRACT_EMBEDDED, value: true }]);
+    assetMock.getByIds.mockResolvedValue([assetStub.imageDng]);
+
+    await sut.handleGenerateThumbnail({ id: assetStub.image.id });
+
+    const extractedPath = mediaMock.extract.mock.calls.at(-1)?.[1].toString();
+    expect(mediaMock.resize.mock.calls).toEqual([
+      [
+        extractedPath,
+        'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp',
+        {
+          format: ImageFormat.WEBP,
+          size: 250,
+          quality: 80,
+          colorspace: Colorspace.P3,
+        },
+      ],
+    ]);
+    expect(extractedPath?.endsWith('.tmp')).toBe(true);
+    expect(storageMock.unlink).toHaveBeenCalledWith(extractedPath);
+  });
+
+  it('should resize original image if embedded image is too small', async () => {
+    mediaMock.extract.mockResolvedValue(true);
+    mediaMock.getImageDimensions.mockResolvedValue({ width: 1000, height: 1000 });
+    configMock.load.mockResolvedValue([{ key: SystemConfigKey.IMAGE_EXTRACT_EMBEDDED, value: true }]);
+    assetMock.getByIds.mockResolvedValue([assetStub.imageDng]);
+
+    await sut.handleGenerateThumbnail({ id: assetStub.image.id });
+
+    expect(mediaMock.resize.mock.calls).toEqual([
+      [
+        assetStub.imageDng.originalPath,
+        'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp',
+        {
+          format: ImageFormat.WEBP,
+          size: 250,
+          quality: 80,
+          colorspace: Colorspace.P3,
+        },
+      ],
+    ]);
+    const extractedPath = mediaMock.extract.mock.calls.at(-1)?.[1].toString();
+    expect(extractedPath?.endsWith('.tmp')).toBe(true);
+    expect(storageMock.unlink).toHaveBeenCalledWith(extractedPath);
+  });
+
+  it('should resize original image if embedded image not found', async () => {
+    configMock.load.mockResolvedValue([{ key: SystemConfigKey.IMAGE_EXTRACT_EMBEDDED, value: true }]);
+    assetMock.getByIds.mockResolvedValue([assetStub.imageDng]);
+
+    await sut.handleGenerateThumbnail({ id: assetStub.image.id });
+
+    expect(mediaMock.resize).toHaveBeenCalledWith(
+      assetStub.imageDng.originalPath,
+      'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp',
+      {
+        format: ImageFormat.WEBP,
+        size: 250,
+        quality: 80,
+        colorspace: Colorspace.P3,
+      },
+    );
+    expect(mediaMock.getImageDimensions).not.toHaveBeenCalled();
+  });
+
+  it('should resize original image if embedded image extraction is not enabled', async () => {
+    configMock.load.mockResolvedValue([{ key: SystemConfigKey.IMAGE_EXTRACT_EMBEDDED, value: false }]);
+    assetMock.getByIds.mockResolvedValue([assetStub.imageDng]);
+
+    await sut.handleGenerateThumbnail({ id: assetStub.image.id });
+
+    expect(mediaMock.extract).not.toHaveBeenCalled();
+    expect(mediaMock.resize).toHaveBeenCalledWith(
+      assetStub.imageDng.originalPath,
+      'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp',
+      {
+        format: ImageFormat.WEBP,
+        size: 250,
+        quality: 80,
+        colorspace: Colorspace.P3,
+      },
+    );
+    expect(mediaMock.getImageDimensions).not.toHaveBeenCalled();
+  });
+
+  describe('handleGenerateThumbhash', () => {
     it('should skip thumbhash generation if asset not found', async () => {
       assetMock.getByIds.mockResolvedValue([]);
       await sut.handleGenerateThumbhash({ id: assetStub.image.id });

server/src/services/media.service.ts

@@ -1,4 +1,5 @@
 import { Inject, Injectable, UnsupportedMediaTypeException } from '@nestjs/common';
+import { dirname } from 'node:path';
 import { GeneratedImageType, StorageCore, StorageFolder } from 'src/cores/storage.core';
 import { SystemConfigCore } from 'src/cores/system-config.core';
 import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
@@ -42,6 +43,7 @@ import {
   VAAPIConfig,
   VP9Config,
 } from 'src/utils/media';
+import { mimeTypes } from 'src/utils/mime-types';
 import { usePagination } from 'src/utils/pagination';
 
 @Injectable()
@@ -195,9 +197,21 @@ export class MediaService {
 
     switch (asset.type) {
       case AssetType.IMAGE: {
-        const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : image.colorspace;
-        const imageOptions = { format, size, colorspace, quality: image.quality };
-        await this.mediaRepository.resize(asset.originalPath, path, imageOptions);
+        const shouldExtract = image.extractEmbedded && mimeTypes.isRaw(asset.originalPath);
+        const extractedPath = StorageCore.getTempPathInDir(dirname(path));
+        const didExtract = shouldExtract && (await this.mediaRepository.extract(asset.originalPath, extractedPath));
+
+        try {
+          const useExtracted = didExtract && (await this.shouldUseExtractedImage(extractedPath, image.previewSize));
+          const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : image.colorspace;
+          const imageOptions = { format, size, colorspace, quality: image.quality };
+
+          await this.mediaRepository.resize(useExtracted ? extractedPath : asset.originalPath, path, imageOptions);
+        } finally {
+          if (didExtract) {
+            await this.storageRepository.unlink(extractedPath);
+          }
+        }
         break;
       }
 
@@ -527,7 +541,7 @@ export class MediaService {
     }
   }
 
-  parseBitrateToBps(bitrateString: string) {
+  private parseBitrateToBps(bitrateString: string) {
     const bitrateValue = Number.parseInt(bitrateString);
 
     if (Number.isNaN(bitrateValue)) {
@@ -542,4 +556,11 @@ export class MediaService {
       return bitrateValue;
     }
   }
+
+  private async shouldUseExtractedImage(extractedPath: string, targetSize: number) {
+    const { width, height } = await this.mediaRepository.getImageDimensions(extractedPath);
+    const extractedSize = Math.min(width, height);
+
+    return extractedSize >= targetSize;
+  }
 }

server/src/services/system-config.service.spec.ts

@@ -129,6 +129,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
     previewSize: 1440,
     quality: 80,
     colorspace: Colorspace.P3,
+    extractEmbedded: false,
   },
   newVersionCheck: {
     enabled: true,

server/src/utils/mime-types.spec.ts

@@ -106,12 +106,6 @@ describe('mimeTypes', () => {
       expect(values).toEqual(values.map((mimeType) => mimeType.toLowerCase()));
     });
 
-    it('should be a sorted list', () => {
-      const keys = Object.keys(mimeTypes.profile);
-      // TODO: use toSorted in NodeJS 20.
-      expect(keys).toEqual([...keys].sort());
-    });
-
     for (const [extension, v] of Object.entries(mimeTypes.profile)) {
       it(`should lookup ${extension}`, () => {
         expect(mimeTypes.lookup(`test.${extension}`)).toEqual(v[0]);
@@ -128,12 +122,6 @@ describe('mimeTypes', () => {
       expect(values).toEqual(values.map((mimeType) => mimeType.toLowerCase()));
     });
 
-    it('should be a sorted list', () => {
-      const keys = Object.keys(mimeTypes.image);
-      // TODO: use toSorted in NodeJS 20.
-      expect(keys).toEqual([...keys].sort());
-    });
-
     it('should contain only image mime types', () => {
       const values = Object.values(mimeTypes.image).flat();
       expect(values).toEqual(values.filter((mimeType) => mimeType.startsWith('image/')));
@@ -157,7 +145,6 @@ describe('mimeTypes', () => {
 
     it('should be a sorted list', () => {
       const keys = Object.keys(mimeTypes.video);
-      // TODO: use toSorted in NodeJS 20.
       expect(keys).toEqual([...keys].sort());
     });
 
@@ -184,7 +171,6 @@ describe('mimeTypes', () => {
 
     it('should be a sorted list', () => {
       const keys = Object.keys(mimeTypes.sidecar);
-      // TODO: use toSorted in NodeJS 20.
       expect(keys).toEqual([...keys].sort());
     });
 
@@ -198,4 +184,20 @@ describe('mimeTypes', () => {
       });
     }
   });
+
+  describe('raw', () => {
+    it('should contain only lowercase mime types', () => {
+      const keys = Object.keys(mimeTypes.raw);
+      expect(keys).toEqual(keys.map((mimeType) => mimeType.toLowerCase()));
+
+      const values = Object.values(mimeTypes.raw).flat();
+      expect(values).toEqual(values.map((mimeType) => mimeType.toLowerCase()));
+    });
+
+    for (const [extension, v] of Object.entries(mimeTypes.video)) {
+      it(`should lookup ${extension}`, () => {
+        expect(mimeTypes.lookup(`test.${extension}`)).toEqual(v[0]);
+      });
+    }
+  });
 });

server/src/utils/mime-types.ts

@@ -1,12 +1,10 @@
 import { extname } from 'node:path';
 import { AssetType } from 'src/entities/asset.entity';
 
-const image: Record<string, string[]> = {
+const raw: Record<string, string[]> = {
   '.3fr': ['image/3fr', 'image/x-hasselblad-3fr'],
   '.ari': ['image/ari', 'image/x-arriflex-ari'],
   '.arw': ['image/arw', 'image/x-sony-arw'],
-  '.avif': ['image/avif'],
-  '.bmp': ['image/bmp'],
   '.cap': ['image/cap', 'image/x-phaseone-cap'],
   '.cin': ['image/cin', 'image/x-phantom-cin'],
   '.cr2': ['image/cr2', 'image/x-canon-cr2'],
@@ -16,16 +14,7 @@ const image: Record<string, string[]> = {
   '.dng': ['image/dng', 'image/x-adobe-dng'],
   '.erf': ['image/erf', 'image/x-epson-erf'],
   '.fff': ['image/fff', 'image/x-hasselblad-fff'],
-  '.gif': ['image/gif'],
-  '.heic': ['image/heic'],
-  '.heif': ['image/heif'],
-  '.hif': ['image/hif'],
   '.iiq': ['image/iiq', 'image/x-phaseone-iiq'],
-  '.insp': ['image/jpeg'],
-  '.jpe': ['image/jpeg'],
-  '.jpeg': ['image/jpeg'],
-  '.jpg': ['image/jpeg'],
-  '.jxl': ['image/jxl'],
   '.k25': ['image/k25', 'image/x-kodak-k25'],
   '.kdc': ['image/kdc', 'image/x-kodak-kdc'],
   '.mrw': ['image/mrw', 'image/x-minolta-mrw'],
@@ -33,7 +22,6 @@ const image: Record<string, string[]> = {
   '.orf': ['image/orf', 'image/x-olympus-orf'],
   '.ori': ['image/ori', 'image/x-olympus-ori'],
   '.pef': ['image/pef', 'image/x-pentax-pef'],
-  '.png': ['image/png'],
   '.psd': ['image/psd', 'image/vnd.adobe.photoshop'],
   '.raf': ['image/raf', 'image/x-fuji-raf'],
   '.raw': ['image/raw', 'image/x-panasonic-raw'],
@@ -42,11 +30,27 @@ const image: Record<string, string[]> = {
   '.sr2': ['image/sr2', 'image/x-sony-sr2'],
   '.srf': ['image/srf', 'image/x-sony-srf'],
   '.srw': ['image/srw', 'image/x-samsung-srw'],
+  '.x3f': ['image/x3f', 'image/x-sigma-x3f'],
+};
+
+const image: Record<string, string[]> = {
+  ...raw,
+  '.avif': ['image/avif'],
+  '.bmp': ['image/bmp'],
+  '.gif': ['image/gif'],
+  '.heic': ['image/heic'],
+  '.heif': ['image/heif'],
+  '.hif': ['image/hif'],
+  '.insp': ['image/jpeg'],
+  '.jpe': ['image/jpeg'],
+  '.jpeg': ['image/jpeg'],
+  '.jpg': ['image/jpeg'],
+  '.jxl': ['image/jxl'],
+  '.png': ['image/png'],
   '.svg': ['image/svg'],
   '.tif': ['image/tiff'],
   '.tiff': ['image/tiff'],
   '.webp': ['image/webp'],
-  '.x3f': ['image/x3f', 'image/x-sigma-x3f'],
 };
 
 const profileExtensions = new Set(['.avif', '.dng', '.heic', '.heif', '.jpeg', '.jpg', '.png', '.webp', '.svg']);
@@ -77,22 +81,25 @@ const sidecar: Record<string, string[]> = {
   '.xmp': ['application/xml', 'text/xml'],
 };
 
+const types = { ...image, ...video, ...sidecar };
+
 const isType = (filename: string, r: Record<string, string[]>) => extname(filename).toLowerCase() in r;
 
-const lookup = (filename: string) =>
-  ({ ...image, ...video, ...sidecar })[extname(filename).toLowerCase()]?.[0] ?? 'application/octet-stream';
+const lookup = (filename: string) => types[extname(filename).toLowerCase()]?.[0] ?? 'application/octet-stream';
 
 export const mimeTypes = {
   image,
   profile,
   sidecar,
   video,
+  raw,
 
   isAsset: (filename: string) => isType(filename, image) || isType(filename, video),
   isImage: (filename: string) => isType(filename, image),
   isProfile: (filename: string) => isType(filename, profile),
   isSidecar: (filename: string) => isType(filename, sidecar),
   isVideo: (filename: string) => isType(filename, video),
+  isRaw: (filename: string) => isType(filename, raw),
   lookup,
   assetType: (filename: string) => {
     const contentType = lookup(filename);