feat(server): wide gamut thumbnails (#3658)

server/src/domain/facial-recognition/facial-recognition.service.spec.ts

@@ -1,3 +1,4 @@
+import { Colorspace } from '@app/infra/entities';
 import {
   assetStub,
   faceStub,
@@ -115,7 +116,6 @@ describe(FacialRecognitionService.name, () => {
     personMock = newPersonRepositoryMock();
     searchMock = newSearchRepositoryMock();
     storageMock = newStorageRepositoryMock();
-    configMock = newSystemConfigRepositoryMock();
 
     mediaMock.crop.mockResolvedValue(croppedFace);
 
@@ -292,6 +292,8 @@ describe(FacialRecognitionService.name, () => {
       expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
         format: 'jpeg',
         size: 250,
+        quality: 80,
+        colorspace: Colorspace.P3,
       });
       expect(personMock.update).toHaveBeenCalledWith({
         id: 'person-1',
@@ -313,6 +315,8 @@ describe(FacialRecognitionService.name, () => {
       expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
         format: 'jpeg',
         size: 250,
+        quality: 80,
+        colorspace: Colorspace.P3,
       });
     });
 
@@ -330,6 +334,8 @@ describe(FacialRecognitionService.name, () => {
       expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
         format: 'jpeg',
         size: 250,
+        quality: 80,
+        colorspace: Colorspace.P3,
       });
     });
   });

server/src/domain/facial-recognition/facial-recognition.services.ts

@@ -162,8 +162,15 @@ export class FacialRecognitionService {
       height: newHalfSize * 2,
     };
 
+    const { thumbnail } = await this.configCore.getConfig();
     const croppedOutput = await this.mediaRepository.crop(asset.resizePath, cropOptions);
-    await this.mediaRepository.resize(croppedOutput, output, { size: FACE_THUMBNAIL_SIZE, format: 'jpeg' });
+    const thumbnailOptions = {
+      format: 'jpeg',
+      size: FACE_THUMBNAIL_SIZE,
+      colorspace: thumbnail.colorspace,
+      quality: thumbnail.quality,
+    } as const;
+    await this.mediaRepository.resize(croppedOutput, output, thumbnailOptions);
     await this.personRepository.update({ id: personId, thumbnailPath: output });
 
     return true;

server/src/domain/media/media.repository.ts

@@ -1,10 +1,13 @@
 import { VideoCodec } from '@app/infra/entities';
+import { Writable } from 'stream';
 
 export const IMediaRepository = 'IMediaRepository';
 
 export interface ResizeOptions {
   size: number;
   format: 'webp' | 'jpeg';
+  colorspace: string;
+  quality: number;
 }
 
 export interface VideoStreamInfo {
@@ -73,5 +76,5 @@ export interface IMediaRepository {
 
   // video
   probe(input: string): Promise<VideoInfo>;
-  transcode(input: string, output: string, options: TranscodeOptions): Promise<void>;
+  transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise<void>;
 }

server/src/domain/media/media.service.spec.ts

@@ -1,5 +1,6 @@
 import {
   AssetType,
+  Colorspace,
   SystemConfigKey,
   ToneMapping,
   TranscodeHWAccel,
@@ -134,6 +135,8 @@ describe(MediaService.name, () => {
       expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.jpeg', {
         size: 1440,
         format: 'jpeg',
+        quality: 80,
+        colorspace: Colorspace.P3,
       });
       expect(assetMock.save).toHaveBeenCalledWith({
         id: 'asset-id',
@@ -148,12 +151,11 @@ describe(MediaService.name, () => {
 
       expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
       expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
-        inputOptions: [],
+        inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
         outputOptions: [
-          '-ss 00:00:00.000',
           '-frames:v 1',
           '-v verbose',
-          '-vf scale=-2:1440:out_color_matrix=bt601:out_range=pc,format=yuv420p',
+          '-vf scale=-2:1440:flags=lanczos+accurate_rnd+bitexact+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p',
         ],
         twoPass: false,
       });
@@ -170,12 +172,11 @@ describe(MediaService.name, () => {
 
       expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
       expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
-        inputOptions: [],
+        inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
         outputOptions: [
-          '-ss 00:00:00.000',
           '-frames:v 1',
           '-v verbose',
-          '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt470bg:t=601:m=bt470bg:range=pc,format=yuv420p',
+          '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p',
         ],
         twoPass: false,
       });
@@ -209,12 +210,13 @@ describe(MediaService.name, () => {
       assetMock.getByIds.mockResolvedValue([assetStub.image]);
       await sut.handleGenerateWebpThumbnail({ id: assetStub.image.id });
 
-      expect(mediaMock.resize).toHaveBeenCalledWith(
-        '/uploads/user-id/thumbs/path.jpg',
-        '/uploads/user-id/thumbs/path.webp',
-        { format: 'webp', size: 250 },
-      );
-      expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: '/uploads/user-id/thumbs/path.webp' });
+      expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.webp', {
+        format: 'webp',
+        size: 250,
+        quality: 80,
+        colorspace: Colorspace.P3,
+      });
+      expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: 'upload/thumbs/user-id/asset-id.webp' });
     });
   });
 

server/src/domain/media/media.service.ts

@@ -9,7 +9,6 @@ import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config
 import { SystemConfigCore } from '../system-config/system-config.core';
 import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from './media.repository';
 import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIConfig, VP9Config } from './media.util';
-
 @Injectable()
 export class MediaService {
   private logger = new Logger(MediaService.name);
@@ -21,9 +20,9 @@ export class MediaService {
     @Inject(IJobRepository) private jobRepository: IJobRepository,
     @Inject(IMediaRepository) private mediaRepository: IMediaRepository,
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
-    @Inject(ISystemConfigRepository) systemConfig: ISystemConfigRepository,
+    @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
   ) {
-    this.configCore = new SystemConfigCore(systemConfig);
+    this.configCore = new SystemConfigCore(configRepository);
   }
 
   async handleQueueGenerateThumbnails(job: IBaseJob) {
@@ -59,38 +58,53 @@ export class MediaService {
       return false;
     }
 
-    const resizePath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId);
-    this.storageRepository.mkdirSync(resizePath);
-    const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`);
-    const { thumbnail } = await this.configCore.getConfig();
+    const resizePath = await this.generateThumbnail(asset, 'jpeg');
+    await this.assetRepository.save({ id: asset.id, resizePath });
+    return true;
+  }
 
+  async generateThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
+    let path;
     switch (asset.type) {
       case AssetType.IMAGE:
-        await this.mediaRepository.resize(asset.originalPath, jpegThumbnailPath, {
-          size: thumbnail.jpegSize,
-          format: 'jpeg',
-        });
-        this.logger.log(`Successfully generated image thumbnail ${asset.id}`);
+        path = await this.generateImageThumbnail(asset, format);
         break;
       case AssetType.VIDEO:
-        const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
-        const mainVideoStream = this.getMainStream(videoStreams);
-        if (!mainVideoStream) {
-          this.logger.error(`Could not extract thumbnail for asset ${asset.id}: no video streams found`);
-          return false;
-        }
-        const mainAudioStream = this.getMainStream(audioStreams);
-        const { ffmpeg } = await this.configCore.getConfig();
-        const config = { ...ffmpeg, targetResolution: thumbnail.jpegSize.toString(), twoPass: false };
-        const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream);
-        await this.mediaRepository.transcode(asset.originalPath, jpegThumbnailPath, options);
-        this.logger.log(`Successfully generated video thumbnail ${asset.id}`);
+        path = await this.generateVideoThumbnail(asset, format);
         break;
+      default:
+        throw new UnsupportedMediaTypeException(`Unsupported asset type for thumbnail generation: ${asset.type}`);
     }
+    this.logger.log(
+      `Successfully generated ${format.toUpperCase()} ${asset.type.toLowerCase()} thumbnail for asset ${asset.id}`,
+    );
+    return path;
+  }
 
-    await this.assetRepository.save({ id: asset.id, resizePath: jpegThumbnailPath });
+  async generateImageThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
+    const { thumbnail } = await this.configCore.getConfig();
+    const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize;
+    const thumbnailOptions = { format, size, colorspace: thumbnail.colorspace, quality: thumbnail.quality };
+    const path = this.ensureThumbnailPath(asset, format);
+    await this.mediaRepository.resize(asset.originalPath, path, thumbnailOptions);
+    return path;
+  }
 
-    return true;
+  async generateVideoThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
+    const { ffmpeg, thumbnail } = await this.configCore.getConfig();
+    const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize;
+    const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
+    const mainVideoStream = this.getMainStream(videoStreams);
+    if (!mainVideoStream) {
+      this.logger.warn(`Skipped thumbnail generation for asset ${asset.id}: no video streams found`);
+      return;
+    }
+    const mainAudioStream = this.getMainStream(audioStreams);
+    const path = this.ensureThumbnailPath(asset, format);
+    const config = { ...ffmpeg, targetResolution: size.toString() };
+    const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream);
+    await this.mediaRepository.transcode(asset.originalPath, path, options);
+    return path;
   }
 
   async handleGenerateWebpThumbnail({ id }: IEntityJob) {
@@ -99,12 +113,8 @@ export class MediaService {
       return false;
     }
 
-    const webpPath = asset.resizePath.replace('jpeg', 'webp').replace('jpg', 'webp');
-
-    const { thumbnail } = await this.configCore.getConfig();
-    await this.mediaRepository.resize(asset.resizePath, webpPath, { size: thumbnail.webpSize, format: 'webp' });
+    const webpPath = await this.generateThumbnail(asset, 'webp');
     await this.assetRepository.save({ id: asset.id, webpPath });
-
     return true;
   }
 
@@ -289,4 +299,10 @@ export class MediaService {
 
     return handler;
   }
+
+  ensureThumbnailPath(asset: AssetEntity, extension: string): string {
+    const folderPath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId);
+    this.storageRepository.mkdirSync(folderPath);
+    return join(folderPath, `${asset.id}.${extension}`);
+  }
 }

server/src/domain/media/media.util.ts

@@ -263,8 +263,11 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
 }
 
 export class ThumbnailConfig extends BaseConfig {
+  getBaseInputOptions(): string[] {
+    return ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'];
+  }
   getBaseOutputOptions() {
-    return ['-ss 00:00:00.000', '-frames:v 1'];
+    return ['-frames:v 1'];
   }
 
   getPresetOptions() {
@@ -277,16 +280,16 @@ export class ThumbnailConfig extends BaseConfig {
 
   getScaling(videoStream: VideoStreamInfo) {
     let options = super.getScaling(videoStream);
+    options += ':flags=lanczos+accurate_rnd+bitexact+full_chroma_int';
     if (!this.shouldToneMap(videoStream)) {
-      options += ':out_color_matrix=bt601:out_range=pc';
+      options += ':out_color_matrix=601:out_range=pc';
     }
     return options;
   }
 
   getColors() {
     return {
-      // jpeg and webp only support bt.601, so we need to convert to that directly when tone-mapping to avoid color shifts
-      primaries: 'bt470bg',
+      primaries: 'bt709',
       transfer: '601',
       matrix: 'bt470bg',
     };

server/src/domain/system-config/dto/system-config-thumbnail.dto.ts

@@ -1,15 +1,29 @@
+import { Colorspace } from '@app/infra/entities';
 import { ApiProperty } from '@nestjs/swagger';
 import { Type } from 'class-transformer';
-import { IsInt } from 'class-validator';
+import { IsEnum, IsInt, Max, Min } from 'class-validator';
 
 export class SystemConfigThumbnailDto {
   @IsInt()
+  @Min(1)
   @Type(() => Number)
   @ApiProperty({ type: 'integer' })
   webpSize!: number;
 
   @IsInt()
+  @Min(1)
   @Type(() => Number)
   @ApiProperty({ type: 'integer' })
   jpegSize!: number;
+
+  @IsInt()
+  @Min(1)
+  @Max(100)
+  @Type(() => Number)
+  @ApiProperty({ type: 'integer' })
+  quality!: number;
+
+  @IsEnum(Colorspace)
+  @ApiProperty({ enumName: 'Colorspace', enum: Colorspace })
+  colorspace!: Colorspace;
 }

server/src/domain/system-config/system-config.core.ts

@@ -1,6 +1,7 @@
 import {
   AudioCodec,
   CQMode,
+  Colorspace,
   SystemConfig,
   SystemConfigEntity,
   SystemConfigKey,
@@ -98,6 +99,8 @@ export const defaults = Object.freeze<SystemConfig>({
   thumbnail: {
     webpSize: 250,
     jpegSize: 1440,
+    quality: 80,
+    colorspace: Colorspace.P3,
   },
 });
 

server/src/domain/system-config/system-config.service.spec.ts

@@ -1,6 +1,7 @@
 import {
   AudioCodec,
   CQMode,
+  Colorspace,
   SystemConfig,
   SystemConfigEntity,
   SystemConfigKey,
@@ -94,6 +95,8 @@ const updatedConfig = Object.freeze<SystemConfig>({
   thumbnail: {
     webpSize: 250,
     jpegSize: 1440,
+    quality: 80,
+    colorspace: Colorspace.P3,
   },
 });
 

server/src/infra/entities/system-config.entity.ts

@@ -76,6 +76,8 @@ export enum SystemConfigKey {
 
   THUMBNAIL_WEBP_SIZE = 'thumbnail.webpSize',
   THUMBNAIL_JPEG_SIZE = 'thumbnail.jpegSize',
+  THUMBNAIL_QUALITY = 'thumbnail.quality',
+  THUMBNAIL_COLORSPACE = 'thumbnail.colorspace',
 }
 
 export enum TranscodePolicy {
@@ -117,6 +119,11 @@ export enum CQMode {
   ICQ = 'icq',
 }
 
+export enum Colorspace {
+  SRGB = 'srgb',
+  P3 = 'p3',
+}
+
 export interface SystemConfig {
   ffmpeg: {
     crf: number;
@@ -179,5 +186,7 @@ export interface SystemConfig {
   thumbnail: {
     webpSize: number;
     jpegSize: number;
+    quality: number;
+    colorspace: Colorspace;
   };
 }

server/src/infra/repositories/media.repository.ts

@@ -1,8 +1,10 @@
 import { CropOptions, IMediaRepository, ResizeOptions, TranscodeOptions, VideoInfo } from '@app/domain';
+import { Colorspace } from '@app/infra/entities';
 import { Logger } from '@nestjs/common';
 import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
 import fs from 'fs/promises';
 import sharp from 'sharp';
+import { Writable } from 'stream';
 import { promisify } from 'util';
 
 const probe = promisify<string, FfprobeData>(ffmpeg.ffprobe);
@@ -11,7 +13,7 @@ sharp.concurrency(0);
 export class MediaRepository implements IMediaRepository {
   private logger = new Logger(MediaRepository.name);
 
-  crop(input: string, options: CropOptions): Promise<Buffer> {
+  crop(input: string | Buffer, options: CropOptions): Promise<Buffer> {
     return sharp(input, { failOn: 'none' })
       .extract({
         left: options.left,
@@ -23,10 +25,25 @@ export class MediaRepository implements IMediaRepository {
   }
 
   async resize(input: string | Buffer, output: string, options: ResizeOptions): Promise<void> {
-    await sharp(input, { failOn: 'none' })
+    let colorProfile = options.colorspace;
+    if (options.colorspace !== Colorspace.SRGB) {
+      try {
+        const { space } = await sharp(input).metadata();
+        // if the image is already in srgb, keep it that way
+        if (space === 'srgb') {
+          colorProfile = Colorspace.SRGB;
+        }
+      } catch (err) {
+        this.logger.warn(`Could not determine colorspace of image, defaulting to ${colorProfile} profile`);
+      }
+    }
+    const chromaSubsampling = options.quality >= 80 ? '4:4:4' : '4:2:0'; // this is default in libvips (except the threshold is 90), but we need to set it manually in sharp
+    sharp(input, { failOn: 'none' })
+      .pipelineColorspace('rgb16')
       .resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true })
       .rotate()
-      .toFormat(options.format)
+      .withMetadata({ icc: colorProfile })
+      .toFormat(options.format, { quality: options.quality, chromaSubsampling })
       .toFile(output);
   }
 
@@ -61,7 +78,7 @@ export class MediaRepository implements IMediaRepository {
     };
   }
 
-  transcode(input: string, output: string, options: TranscodeOptions): Promise<void> {
+  transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise<void> {
     if (!options.twoPass) {
       return new Promise((resolve, reject) => {
         ffmpeg(input, { niceness: 10 })
@@ -77,6 +94,10 @@ export class MediaRepository implements IMediaRepository {
       });
     }
 
+    if (typeof output !== 'string') {
+      throw new Error('Two-pass transcoding does not support writing to a stream');
+    }
+
     // two-pass allows for precise control of bitrate at the cost of running twice
     // recommended for vp9 for better quality and compression
     return new Promise((resolve, reject) => {