refactor: infra folder (#8138)

server/src/utils/bytes.ts

@@ -0,0 +1,24 @@
+const KiB = Math.pow(1024, 1);
+const MiB = Math.pow(1024, 2);
+const GiB = Math.pow(1024, 3);
+const TiB = Math.pow(1024, 4);
+const PiB = Math.pow(1024, 5);
+
+export const HumanReadableSize = { KiB, MiB, GiB, TiB, PiB };
+
+export function asHumanReadable(bytes: number, precision = 1): string {
+  const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB'];
+
+  let magnitude = 0;
+  let remainder = bytes;
+  while (remainder >= 1024) {
+    if (magnitude + 1 < units.length) {
+      magnitude++;
+      remainder /= 1024;
+    } else {
+      break;
+    }
+  }
+
+  return `${remainder.toFixed(magnitude == 0 ? 0 : precision)} ${units[magnitude]}`;
+}

server/src/utils/database.ts

@@ -0,0 +1,139 @@
+import _ from 'lodash';
+import { AssetEntity } from 'src/entities/asset.entity';
+import { AssetSearchBuilderOptions } from 'src/interfaces/search.repository';
+import { Between, IsNull, LessThanOrEqual, MoreThanOrEqual, Not, SelectQueryBuilder } from 'typeorm';
+
+/**
+ * Allows optional values unlike the regular Between and uses MoreThanOrEqual
+ * or LessThanOrEqual when only one parameter is specified.
+ */
+export function OptionalBetween<T>(from?: T, to?: T) {
+  if (from && to) {
+    return Between(from, to);
+  } else if (from) {
+    return MoreThanOrEqual(from);
+  } else if (to) {
+    return LessThanOrEqual(to);
+  }
+}
+
+export const asVector = (embedding: number[], quote = false) =>
+  quote ? `'[${embedding.join(',')}]'` : `[${embedding.join(',')}]`;
+
+export function searchAssetBuilder(
+  builder: SelectQueryBuilder<AssetEntity>,
+  options: AssetSearchBuilderOptions,
+): SelectQueryBuilder<AssetEntity> {
+  builder.andWhere(
+    _.omitBy(
+      {
+        createdAt: OptionalBetween(options.createdAfter, options.createdBefore),
+        updatedAt: OptionalBetween(options.updatedAfter, options.updatedBefore),
+        deletedAt: OptionalBetween(options.trashedAfter, options.trashedBefore),
+        fileCreatedAt: OptionalBetween(options.takenAfter, options.takenBefore),
+      },
+      _.isUndefined,
+    ),
+  );
+
+  const exifInfo = _.omitBy(_.pick(options, ['city', 'country', 'lensModel', 'make', 'model', 'state']), _.isUndefined);
+  const hasExifQuery = Object.keys(exifInfo).length > 0;
+
+  if (options.withExif && !hasExifQuery) {
+    builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo');
+  }
+
+  if (hasExifQuery) {
+    options.withExif
+      ? builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo')
+      : builder.leftJoin(`${builder.alias}.exifInfo`, 'exifInfo');
+
+    builder.andWhere({ exifInfo });
+  }
+
+  const id = _.pick(options, ['checksum', 'deviceAssetId', 'deviceId', 'id', 'libraryId']);
+  builder.andWhere(_.omitBy(id, _.isUndefined));
+
+  if (options.userIds) {
+    builder.andWhere(`${builder.alias}.ownerId IN (:...userIds)`, { userIds: options.userIds });
+  }
+
+  const path = _.pick(options, ['encodedVideoPath', 'originalPath', 'resizePath', 'webpPath']);
+  builder.andWhere(_.omitBy(path, _.isUndefined));
+
+  if (options.originalFileName) {
+    builder.andWhere(`f_unaccent(${builder.alias}.originalFileName) ILIKE f_unaccent(:originalFileName)`, {
+      originalFileName: `%${options.originalFileName}%`,
+    });
+  }
+
+  const status = _.pick(options, ['isExternal', 'isFavorite', 'isOffline', 'isReadOnly', 'isVisible', 'type']);
+  const {
+    isArchived,
+    isEncoded,
+    isMotion,
+    withArchived,
+    isNotInAlbum,
+    withFaces,
+    withPeople,
+    withSmartInfo,
+    personIds,
+    withExif,
+    withStacked,
+    trashedAfter,
+    trashedBefore,
+  } = options;
+  builder.andWhere(
+    _.omitBy(
+      {
+        ...status,
+        isArchived: isArchived ?? (withArchived ? undefined : false),
+        encodedVideoPath: isEncoded ? Not(IsNull()) : undefined,
+        livePhotoVideoId: isMotion ? Not(IsNull()) : undefined,
+      },
+      _.isUndefined,
+    ),
+  );
+
+  if (isNotInAlbum) {
+    builder
+      .leftJoin(`${builder.alias}.albums`, 'albums')
+      .andWhere('albums.id IS NULL')
+      .andWhere(`${builder.alias}.isVisible = true`);
+  }
+
+  if (withFaces || withPeople) {
+    builder.leftJoinAndSelect(`${builder.alias}.faces`, 'faces');
+  }
+
+  if (withPeople) {
+    builder.leftJoinAndSelect(`${builder.alias}.person`, 'person');
+  }
+
+  if (withSmartInfo) {
+    builder.leftJoinAndSelect(`${builder.alias}.smartInfo`, 'smartInfo');
+  }
+
+  if (personIds && personIds.length > 0) {
+    builder
+      .leftJoin(`${builder.alias}.faces`, 'faces')
+      .andWhere('faces.personId IN (:...personIds)', { personIds })
+      .addGroupBy(`${builder.alias}.id`)
+      .having('COUNT(DISTINCT faces.personId) = :personCount', { personCount: personIds.length });
+
+    if (withExif) {
+      builder.addGroupBy('exifInfo.assetId');
+    }
+  }
+
+  if (withStacked) {
+    builder.leftJoinAndSelect(`${builder.alias}.stack`, 'stack').leftJoinAndSelect('stack.assets', 'stackedAssets');
+  }
+
+  const withDeleted = options.withDeleted ?? (trashedAfter !== undefined || trashedBefore !== undefined);
+  if (withDeleted) {
+    builder.withDeleted();
+  }
+
+  return builder;
+}

server/src/utils/file.ts

@@ -0,0 +1,25 @@
+import { basename, extname } from 'node:path';
+
+export function getFileNameWithoutExtension(path: string): string {
+  return basename(path, extname(path));
+}
+
+export function getLivePhotoMotionFilename(stillName: string, motionName: string) {
+  return getFileNameWithoutExtension(stillName) + extname(motionName);
+}
+
+export enum CacheControl {
+  PRIVATE_WITH_CACHE = 'private_with_cache',
+  PRIVATE_WITHOUT_CACHE = 'private_without_cache',
+  NONE = 'none',
+}
+
+export class ImmichFileResponse {
+  public readonly path!: string;
+  public readonly contentType!: string;
+  public readonly cacheControl!: CacheControl;
+
+  constructor(response: ImmichFileResponse) {
+    Object.assign(this, response);
+  }
+}

server/src/utils/instrumentation.ts

@@ -0,0 +1,106 @@
+import { Histogram, MetricOptions, ValueType, metrics } from '@opentelemetry/api';
+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
+import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
+import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';
+import { Resource } from '@opentelemetry/resources';
+import { ExplicitBucketHistogramAggregation, View } from '@opentelemetry/sdk-metrics';
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+import { snakeCase, startCase } from 'lodash';
+import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
+import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils';
+import { performance } from 'node:perf_hooks';
+import { excludePaths } from 'src/config';
+import { serverVersion } from 'src/constants';
+import { DecorateAll } from 'src/decorators';
+
+let metricsEnabled = process.env.IMMICH_METRICS === 'true';
+const hostMetrics =
+  process.env.IMMICH_HOST_METRICS == null ? metricsEnabled : process.env.IMMICH_HOST_METRICS === 'true';
+const apiMetrics = process.env.IMMICH_API_METRICS == null ? metricsEnabled : process.env.IMMICH_API_METRICS === 'true';
+const repoMetrics = process.env.IMMICH_IO_METRICS == null ? metricsEnabled : process.env.IMMICH_IO_METRICS === 'true';
+
+metricsEnabled ||= hostMetrics || apiMetrics || repoMetrics;
+if (!metricsEnabled && process.env.OTEL_SDK_DISABLED === undefined) {
+  process.env.OTEL_SDK_DISABLED = 'true';
+}
+
+const aggregation = new ExplicitBucketHistogramAggregation(
+  [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000],
+  true,
+);
+
+const metricsPort = Number.parseInt(process.env.IMMICH_METRICS_PORT ?? '8081');
+
+export const otelSDK = new NodeSDK({
+  resource: new Resource({
+    [SemanticResourceAttributes.SERVICE_NAME]: `immich`,
+    [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),
+  }),
+  metricReader: new PrometheusExporter({ port: metricsPort }),
+  contextManager: new AsyncLocalStorageContextManager(),
+  instrumentations: [
+    new HttpInstrumentation(),
+    new IORedisInstrumentation(),
+    new NestInstrumentation(),
+    new PgInstrumentation(),
+  ],
+  views: [new View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],
+});
+
+export const otelConfig: OpenTelemetryModuleOptions = {
+  metrics: {
+    hostMetrics,
+    apiMetrics: {
+      enable: apiMetrics,
+      ignoreRoutes: excludePaths,
+    },
+  },
+};
+
+function ExecutionTimeHistogram({ description, unit = 'ms', valueType = ValueType.DOUBLE }: MetricOptions = {}) {
+  return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
+    if (!repoMetrics || process.env.OTEL_SDK_DISABLED) {
+      return;
+    }
+
+    const method = descriptor.value;
+    const className = target.constructor.name as string;
+    const propertyName = String(propertyKey);
+    const metricName = `${snakeCase(className).replaceAll(/_(?=(repository)|(controller)|(provider)|(service)|(module))/g, '.')}.${snakeCase(propertyName)}.duration`;
+
+    const metricDescription =
+      description ??
+      `The elapsed time in ${unit} for the ${startCase(className)} to ${startCase(propertyName).toLowerCase()}`;
+
+    let histogram: Histogram | undefined;
+
+    descriptor.value = function (...args: any[]) {
+      const start = performance.now();
+      const result = method.apply(this, args);
+
+      void Promise.resolve(result)
+        .then(() => {
+          const end = performance.now();
+          if (!histogram) {
+            histogram = metrics
+              .getMeter('immich')
+              .createHistogram(metricName, { description: metricDescription, unit, valueType });
+          }
+          histogram.record(end - start, {});
+        })
+        .catch(() => {
+          // noop
+        });
+
+      return result;
+    };
+
+    copyMetadataFromFunctionToFunction(method, descriptor.value);
+  };
+}
+
+export const Instrumentation = () => DecorateAll(ExecutionTimeHistogram());

server/src/utils/logger.ts

@@ -0,0 +1,21 @@
+import { ConsoleLogger } from '@nestjs/common';
+import { isLogLevelEnabled } from '@nestjs/common/services/utils/is-log-level-enabled.util';
+import { LogLevel } from 'src/entities/system-config.entity';
+
+const LOG_LEVELS = [LogLevel.VERBOSE, LogLevel.DEBUG, LogLevel.LOG, LogLevel.WARN, LogLevel.ERROR, LogLevel.FATAL];
+
+export class ImmichLogger extends ConsoleLogger {
+  private static logLevels: LogLevel[] = [LogLevel.LOG, LogLevel.WARN, LogLevel.ERROR, LogLevel.FATAL];
+
+  constructor(context: string) {
+    super(context);
+  }
+
+  isLevelEnabled(level: LogLevel) {
+    return isLogLevelEnabled(level, ImmichLogger.logLevels);
+  }
+
+  static setLogLevel(level: LogLevel | false): void {
+    ImmichLogger.logLevels = level === false ? [] : LOG_LEVELS.slice(LOG_LEVELS.indexOf(level));
+  }
+}

server/src/utils/media.ts

@@ -0,0 +1,688 @@
+import { SystemConfigFFmpegDto } from 'src/dtos/system-config-ffmpeg.dto';
+import { CQMode, ToneMapping, TranscodeHWAccel, TranscodeTarget, VideoCodec } from 'src/entities/system-config.entity';
+import {
+  AudioStreamInfo,
+  BitrateDistribution,
+  TranscodeOptions,
+  VideoCodecHWConfig,
+  VideoCodecSWConfig,
+  VideoStreamInfo,
+} from 'src/interfaces/media.repository';
+
+class BaseConfig implements VideoCodecSWConfig {
+  presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'];
+  constructor(protected config: SystemConfigFFmpegDto) {}
+
+  getOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
+    const options = {
+      inputOptions: this.getBaseInputOptions(videoStream),
+      outputOptions: [...this.getBaseOutputOptions(target, videoStream, audioStream), '-v verbose'],
+      twoPass: this.eligibleForTwoPass(),
+    } as TranscodeOptions;
+    if ([TranscodeTarget.ALL, TranscodeTarget.VIDEO].includes(target)) {
+      const filters = this.getFilterOptions(videoStream);
+      if (filters.length > 0) {
+        options.outputOptions.push(`-vf ${filters.join(',')}`);
+      }
+    }
+
+    options.outputOptions.push(...this.getPresetOptions(), ...this.getThreadOptions(), ...this.getBitrateOptions());
+
+    return options;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  getBaseInputOptions(videoStream: VideoStreamInfo): string[] {
+    return [];
+  }
+
+  getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
+    const options = [
+      `-c:v ${[TranscodeTarget.ALL, TranscodeTarget.VIDEO].includes(target) ? this.getVideoCodec() : 'copy'}`,
+      `-c:a ${[TranscodeTarget.ALL, TranscodeTarget.AUDIO].includes(target) ? this.getAudioCodec() : 'copy'}`,
+      // Makes a second pass moving the moov atom to the
+      // beginning of the file for improved playback speed.
+      '-movflags faststart',
+      '-fps_mode passthrough',
+      // explicitly selects the video stream instead of leaving it up to FFmpeg
+      `-map 0:${videoStream.index}`,
+    ];
+
+    if (audioStream) {
+      options.push(`-map 0:${audioStream.index}`);
+    }
+    if (this.getBFrames() > -1) {
+      options.push(`-bf ${this.getBFrames()}`);
+    }
+    if (this.getRefs() > 0) {
+      options.push(`-refs ${this.getRefs()}`);
+    }
+    if (this.getGopSize() > 0) {
+      options.push(`-g ${this.getGopSize()}`);
+    }
+
+    if (this.config.targetVideoCodec === VideoCodec.HEVC) {
+      options.push('-tag:v hvc1');
+    }
+
+    return options;
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo) {
+    const options = [];
+    if (this.shouldScale(videoStream)) {
+      options.push(`scale=${this.getScaling(videoStream)}`);
+    }
+
+    if (this.shouldToneMap(videoStream)) {
+      options.push(...this.getToneMapping());
+    }
+    options.push('format=yuv420p');
+
+    return options;
+  }
+
+  getPresetOptions() {
+    return [`-preset ${this.config.preset}`];
+  }
+
+  getBitrateOptions() {
+    const bitrates = this.getBitrateDistribution();
+    if (this.eligibleForTwoPass()) {
+      return [
+        `-b:v ${bitrates.target}${bitrates.unit}`,
+        `-minrate ${bitrates.min}${bitrates.unit}`,
+        `-maxrate ${bitrates.max}${bitrates.unit}`,
+      ];
+    } else if (bitrates.max > 0) {
+      // -bufsize is the peak possible bitrate at any moment, while -maxrate is the max rolling average bitrate
+      return [
+        `-${this.useCQP() ? 'q:v' : 'crf'} ${this.config.crf}`,
+        `-maxrate ${bitrates.max}${bitrates.unit}`,
+        `-bufsize ${bitrates.max * 2}${bitrates.unit}`,
+      ];
+    } else {
+      return [`-${this.useCQP() ? 'q:v' : 'crf'} ${this.config.crf}`];
+    }
+  }
+
+  getThreadOptions(): Array<string> {
+    if (this.config.threads <= 0) {
+      return [];
+    }
+    return [`-threads ${this.config.threads}`];
+  }
+
+  eligibleForTwoPass() {
+    if (!this.config.twoPass || this.config.accel !== TranscodeHWAccel.DISABLED) {
+      return false;
+    }
+
+    return this.isBitrateConstrained() || this.config.targetVideoCodec === VideoCodec.VP9;
+  }
+
+  getBitrateDistribution() {
+    const max = this.getMaxBitrateValue();
+    const target = Math.ceil(max / 1.45); // recommended by https://developers.google.com/media/vp9/settings/vod
+    const min = target / 2;
+    const unit = this.getBitrateUnit();
+
+    return { max, target, min, unit } as BitrateDistribution;
+  }
+
+  getTargetResolution(videoStream: VideoStreamInfo) {
+    let target;
+    target =
+      this.config.targetResolution === 'original'
+        ? Math.min(videoStream.height, videoStream.width)
+        : Number.parseInt(this.config.targetResolution);
+
+    if (target % 2 !== 0) {
+      target -= 1;
+    }
+
+    return target;
+  }
+
+  shouldScale(videoStream: VideoStreamInfo) {
+    const oddDimensions = videoStream.height % 2 !== 0 || videoStream.width % 2 !== 0;
+    const largerThanTarget = Math.min(videoStream.height, videoStream.width) > this.getTargetResolution(videoStream);
+    return oddDimensions || largerThanTarget;
+  }
+
+  shouldToneMap(videoStream: VideoStreamInfo) {
+    return videoStream.isHDR && this.config.tonemap !== ToneMapping.DISABLED;
+  }
+
+  getScaling(videoStream: VideoStreamInfo) {
+    const targetResolution = this.getTargetResolution(videoStream);
+    const mult = this.config.accel === TranscodeHWAccel.QSV ? 1 : 2; // QSV doesn't support scaling numbers below -1
+    return this.isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
+  }
+
+  getSize(videoStream: VideoStreamInfo) {
+    const smaller = this.getTargetResolution(videoStream);
+    const factor = Math.max(videoStream.height, videoStream.width) / Math.min(videoStream.height, videoStream.width);
+    let larger = Math.round(smaller * factor);
+    if (larger % 2 !== 0) {
+      larger -= 1;
+    }
+    return this.isVideoVertical(videoStream) ? { width: smaller, height: larger } : { width: larger, height: smaller };
+  }
+
+  isVideoRotated(videoStream: VideoStreamInfo) {
+    return Math.abs(videoStream.rotation) === 90;
+  }
+
+  isVideoVertical(videoStream: VideoStreamInfo) {
+    return videoStream.height > videoStream.width || this.isVideoRotated(videoStream);
+  }
+
+  isBitrateConstrained() {
+    return this.getMaxBitrateValue() > 0;
+  }
+
+  getBitrateUnit() {
+    const maxBitrate = this.getMaxBitrateValue();
+    return this.config.maxBitrate.trim().slice(maxBitrate.toString().length); // use inputted unit if provided
+  }
+
+  getMaxBitrateValue() {
+    return Number.parseInt(this.config.maxBitrate) || 0;
+  }
+
+  getPresetIndex() {
+    return this.presets.indexOf(this.config.preset);
+  }
+
+  getColors() {
+    return {
+      primaries: 'bt709',
+      transfer: 'bt709',
+      matrix: 'bt709',
+    };
+  }
+
+  getNPL() {
+    if (this.config.npl <= 0) {
+      // since hable already outputs a darker image, we use a lower npl value for it
+      return this.config.tonemap === ToneMapping.HABLE ? 100 : 250;
+    } else {
+      return this.config.npl;
+    }
+  }
+
+  getToneMapping() {
+    const colors = this.getColors();
+
+    return [
+      `zscale=t=linear:npl=${this.getNPL()}`,
+      `tonemap=${this.config.tonemap}:desat=0`,
+      `zscale=p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:range=pc`,
+    ];
+  }
+
+  getAudioCodec(): string {
+    return this.config.targetAudioCodec;
+  }
+
+  getVideoCodec(): string {
+    return this.config.targetVideoCodec;
+  }
+
+  getBFrames() {
+    return this.config.bframes;
+  }
+
+  getRefs() {
+    return this.config.refs;
+  }
+
+  getGopSize() {
+    return this.config.gopSize;
+  }
+
+  useCQP() {
+    return this.config.cqMode === CQMode.CQP;
+  }
+}
+
+export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
+  protected devices: string[];
+
+  constructor(
+    protected config: SystemConfigFFmpegDto,
+    devices: string[] = [],
+  ) {
+    super(config);
+    this.devices = this.validateDevices(devices);
+  }
+
+  getSupportedCodecs() {
+    return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
+  }
+
+  validateDevices(devices: string[]) {
+    return devices
+      .filter((device) => device.startsWith('renderD') || device.startsWith('card'))
+      .sort((a, b) => {
+        // order GPU devices first
+        if (a.startsWith('card') && b.startsWith('renderD')) {
+          return -1;
+        }
+        if (a.startsWith('renderD') && b.startsWith('card')) {
+          return 1;
+        }
+        return -a.localeCompare(b);
+      });
+  }
+
+  getVideoCodec(): string {
+    return `${this.config.targetVideoCodec}_${this.config.accel}`;
+  }
+
+  getGopSize() {
+    if (this.config.gopSize <= 0) {
+      return 256;
+    }
+    return this.config.gopSize;
+  }
+
+  getPreferredHardwareDevice(): string | null {
+    const device = this.config.preferredHwDevice;
+    if (device === 'auto') {
+      return null;
+    }
+
+    const deviceName = device.replace('/dev/dri/', '');
+    if (!this.devices.includes(deviceName)) {
+      throw new Error(`Device '${device}' does not exist`);
+    }
+
+    return device;
+  }
+}
+
+export class ThumbnailConfig extends BaseConfig {
+  getBaseInputOptions(): string[] {
+    return ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'];
+  }
+  getBaseOutputOptions() {
+    return ['-frames:v 1'];
+  }
+
+  getPresetOptions() {
+    return [];
+  }
+
+  getBitrateOptions() {
+    return [];
+  }
+
+  eligibleForTwoPass() {
+    return false;
+  }
+
+  getScaling(videoStream: VideoStreamInfo) {
+    let options = super.getScaling(videoStream);
+    options += ':flags=lanczos+accurate_rnd+bitexact+full_chroma_int';
+    if (!this.shouldToneMap(videoStream)) {
+      options += ':out_color_matrix=601:out_range=pc';
+    }
+    return options;
+  }
+
+  getColors() {
+    return {
+      primaries: 'bt709',
+      transfer: '601',
+      matrix: 'bt470bg',
+    };
+  }
+}
+
+export class H264Config extends BaseConfig {
+  getThreadOptions() {
+    if (this.config.threads <= 0) {
+      return [];
+    }
+    return [
+      ...super.getThreadOptions(),
+      '-x264-params "pools=none"',
+      `-x264-params "frame-threads=${this.config.threads}"`,
+    ];
+  }
+}
+
+export class HEVCConfig extends BaseConfig {
+  getThreadOptions() {
+    if (this.config.threads <= 0) {
+      return [];
+    }
+    return [
+      ...super.getThreadOptions(),
+      '-x265-params "pools=none"',
+      `-x265-params "frame-threads=${this.config.threads}"`,
+    ];
+  }
+}
+
+export class VP9Config extends BaseConfig {
+  getPresetOptions() {
+    const speed = Math.min(this.getPresetIndex(), 5); // values over 5 require realtime mode, which is its own can of worms since it overrides -crf and -threads
+    if (speed >= 0) {
+      return [`-cpu-used ${speed}`];
+    }
+    return [];
+  }
+
+  getBitrateOptions() {
+    const bitrates = this.getBitrateDistribution();
+    if (bitrates.max > 0 && this.eligibleForTwoPass()) {
+      return [
+        `-b:v ${bitrates.target}${bitrates.unit}`,
+        `-minrate ${bitrates.min}${bitrates.unit}`,
+        `-maxrate ${bitrates.max}${bitrates.unit}`,
+      ];
+    }
+
+    return [`-${this.useCQP() ? 'q:v' : 'crf'} ${this.config.crf}`, `-b:v ${bitrates.max}${bitrates.unit}`];
+  }
+
+  getThreadOptions() {
+    return ['-row-mt 1', ...super.getThreadOptions()];
+  }
+}
+
+export class NVENCConfig extends BaseHWConfig {
+  getSupportedCodecs() {
+    return [VideoCodec.H264, VideoCodec.HEVC];
+  }
+
+  getBaseInputOptions() {
+    return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'];
+  }
+
+  getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
+    const options = [
+      // below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding
+      '-tune hq',
+      '-qmin 0',
+      '-rc-lookahead 20',
+      '-i_qfactor 0.75',
+      ...super.getBaseOutputOptions(target, videoStream, audioStream),
+    ];
+    if (this.getBFrames() > 0) {
+      options.push('-b_ref_mode middle', '-b_qfactor 1.1');
+    }
+    if (this.config.temporalAQ) {
+      options.push('-temporal-aq 1');
+    }
+    return options;
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo) {
+    const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
+    options.push('format=nv12', 'hwupload_cuda');
+    if (this.shouldScale(videoStream)) {
+      options.push(`scale_cuda=${this.getScaling(videoStream)}`);
+    }
+
+    return options;
+  }
+
+  getPresetOptions() {
+    let presetIndex = this.getPresetIndex();
+    if (presetIndex < 0) {
+      return [];
+    }
+    presetIndex = 7 - Math.min(6, presetIndex); // map to p1-p7; p7 is the highest quality, so reverse index
+    return [`-preset p${presetIndex}`];
+  }
+
+  getBitrateOptions() {
+    const bitrates = this.getBitrateDistribution();
+    if (bitrates.max > 0 && this.config.twoPass) {
+      return [
+        `-b:v ${bitrates.target}${bitrates.unit}`,
+        `-maxrate ${bitrates.max}${bitrates.unit}`,
+        `-bufsize ${bitrates.target}${bitrates.unit}`,
+        '-multipass 2',
+      ];
+    } else if (bitrates.max > 0) {
+      return [
+        `-cq:v ${this.config.crf}`,
+        `-maxrate ${bitrates.max}${bitrates.unit}`,
+        `-bufsize ${bitrates.target}${bitrates.unit}`,
+      ];
+    } else {
+      return [`-cq:v ${this.config.crf}`];
+    }
+  }
+
+  getThreadOptions() {
+    return [];
+  }
+
+  getRefs() {
+    const bframes = this.getBFrames();
+    if (bframes > 0 && bframes < 3 && this.config.refs < 3) {
+      return 0;
+    }
+    return this.config.refs;
+  }
+}
+
+export class QSVConfig extends BaseHWConfig {
+  getBaseInputOptions() {
+    if (this.devices.length === 0) {
+      throw new Error('No QSV device found');
+    }
+
+    let qsvString = '';
+    const hwDevice = this.getPreferredHardwareDevice();
+    if (hwDevice !== null) {
+      qsvString = `,child_device=${hwDevice}`;
+    }
+
+    return [`-init_hw_device qsv=hw${qsvString}`, '-filter_hw_device hw'];
+  }
+
+  getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
+    const options = super.getBaseOutputOptions(target, videoStream, audioStream);
+    // VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a
+    if (this.config.targetVideoCodec === VideoCodec.VP9) {
+      options.push('-low_power 1');
+    }
+    return options;
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo) {
+    const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
+    options.push('format=nv12', 'hwupload=extra_hw_frames=64');
+    if (this.shouldScale(videoStream)) {
+      options.push(`scale_qsv=${this.getScaling(videoStream)}`);
+    }
+    return options;
+  }
+
+  getPresetOptions() {
+    let presetIndex = this.getPresetIndex();
+    if (presetIndex < 0) {
+      return [];
+    }
+    presetIndex = Math.min(6, presetIndex) + 1; // 1 to 7
+    return [`-preset ${presetIndex}`];
+  }
+
+  getBitrateOptions() {
+    const options = [];
+    options.push(`-${this.useCQP() ? 'q:v' : 'global_quality'} ${this.config.crf}`);
+    const bitrates = this.getBitrateDistribution();
+    if (bitrates.max > 0) {
+      options.push(`-maxrate ${bitrates.max}${bitrates.unit}`, `-bufsize ${bitrates.max * 2}${bitrates.unit}`);
+    }
+    return options;
+  }
+
+  // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
+  getBFrames() {
+    if (this.config.bframes < 0) {
+      return 7;
+    }
+    return this.config.bframes;
+  }
+
+  getRefs() {
+    if (this.config.refs <= 0) {
+      return 5;
+    }
+    return this.config.refs;
+  }
+
+  useCQP() {
+    return this.config.cqMode === CQMode.CQP || this.config.targetVideoCodec === VideoCodec.VP9;
+  }
+}
+
+export class VAAPIConfig extends BaseHWConfig {
+  getBaseInputOptions() {
+    if (this.devices.length === 0) {
+      throw new Error('No VAAPI device found');
+    }
+
+    let hwDevice = this.getPreferredHardwareDevice();
+    if (hwDevice === null) {
+      hwDevice = `/dev/dri/${this.devices[0]}`;
+    }
+
+    return [`-init_hw_device vaapi=accel:${hwDevice}`, '-filter_hw_device accel'];
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo) {
+    const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
+    options.push('format=nv12', 'hwupload');
+    if (this.shouldScale(videoStream)) {
+      options.push(`scale_vaapi=${this.getScaling(videoStream)}`);
+    }
+
+    return options;
+  }
+
+  getPresetOptions() {
+    let presetIndex = this.getPresetIndex();
+    if (presetIndex < 0) {
+      return [];
+    }
+    presetIndex = Math.min(6, presetIndex) + 1; // 1 to 7
+    return [`-compression_level ${presetIndex}`];
+  }
+
+  getBitrateOptions() {
+    const bitrates = this.getBitrateDistribution();
+    const options = [];
+
+    if (this.config.targetVideoCodec === VideoCodec.VP9) {
+      options.push('-bsf:v vp9_raw_reorder,vp9_superframe');
+    }
+
+    // VAAPI doesn't allow setting both quality and max bitrate
+    if (bitrates.max > 0) {
+      options.push(
+        `-b:v ${bitrates.target}${bitrates.unit}`,
+        `-maxrate ${bitrates.max}${bitrates.unit}`,
+        `-minrate ${bitrates.min}${bitrates.unit}`,
+        '-rc_mode 3',
+      ); // variable bitrate
+    } else if (this.useCQP()) {
+      options.push(`-qp ${this.config.crf}`, `-global_quality ${this.config.crf}`, '-rc_mode 1');
+    } else {
+      options.push(`-global_quality ${this.config.crf}`, '-rc_mode 4');
+    }
+
+    return options;
+  }
+
+  useCQP() {
+    return this.config.cqMode !== CQMode.ICQ || this.config.targetVideoCodec === VideoCodec.VP9;
+  }
+}
+
+export class RKMPPConfig extends BaseHWConfig {
+  private hasOpenCL: boolean;
+
+  constructor(
+    protected config: SystemConfigFFmpegDto,
+    devices: string[] = [],
+    hasOpenCL: boolean = false,
+  ) {
+    super(config, devices);
+    this.hasOpenCL = hasOpenCL;
+  }
+
+  eligibleForTwoPass(): boolean {
+    return false;
+  }
+
+  getBaseInputOptions(videoStream: VideoStreamInfo) {
+    if (this.devices.length === 0) {
+      throw new Error('No RKMPP device found');
+    }
+    return this.shouldToneMap(videoStream) && !this.hasOpenCL
+      ? [] // disable hardware decoding & filters
+      : ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga'];
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo) {
+    if (this.shouldToneMap(videoStream)) {
+      if (!this.hasOpenCL) {
+        return super.getFilterOptions(videoStream);
+      }
+      const colors = this.getColors();
+      return [
+        `scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1`,
+        'hwmap=derive_device=opencl:mode=read',
+        `tonemap_opencl=format=nv12:r=pc:p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:tonemap=${this.config.tonemap}:desat=0`,
+        'hwmap=derive_device=rkmpp:mode=write:reverse=1',
+        'format=drm_prime',
+      ];
+    } else if (this.shouldScale(videoStream)) {
+      return [`scale_rkrga=${this.getScaling(videoStream)}:format=nv12:afbc=1`];
+    }
+    return [];
+  }
+
+  getPresetOptions() {
+    switch (this.config.targetVideoCodec) {
+      case VideoCodec.H264: {
+        // from ffmpeg_mpp help, commonly referred to as H264 level 5.1
+        return ['-level 51'];
+      }
+      case VideoCodec.HEVC: {
+        // from ffmpeg_mpp help, commonly referred to as HEVC level 5.1
+        return ['-level 153'];
+      }
+      default: {
+        throw new Error(`Incompatible video codec for RKMPP: ${this.config.targetVideoCodec}`);
+      }
+    }
+  }
+
+  getBitrateOptions() {
+    const bitrate = this.getMaxBitrateValue();
+    if (bitrate > 0) {
+      // -b:v specifies max bitrate, average bitrate is derived automatically...
+      return ['-rc_mode AVBR', `-b:v ${bitrate}${this.getBitrateUnit()}`];
+    }
+    // use CRF value as QP value
+    return ['-rc_mode CQP', `-qp_init ${this.config.crf}`];
+  }
+
+  getSupportedCodecs() {
+    return [VideoCodec.H264, VideoCodec.HEVC];
+  }
+
+  getVideoCodec(): string {
+    return `${this.config.targetVideoCodec}_rkmpp`;
+  }
+}

server/src/utils/mime-types.spec.ts

@@ -0,0 +1,198 @@
+import { mimeTypes } from 'src/utils/mime-types';
+
+describe('mimeTypes', () => {
+  for (const { mimetype, extension } of [
+    // Please ensure this list is sorted.
+    { mimetype: 'image/3fr', extension: '.3fr' },
+    { mimetype: 'image/ari', extension: '.ari' },
+    { mimetype: 'image/arw', extension: '.arw' },
+    { mimetype: 'image/avif', extension: '.avif' },
+    { mimetype: 'image/bmp', extension: '.bmp' },
+    { mimetype: 'image/cap', extension: '.cap' },
+    { mimetype: 'image/cin', extension: '.cin' },
+    { mimetype: 'image/cr2', extension: '.cr2' },
+    { mimetype: 'image/cr3', extension: '.cr3' },
+    { mimetype: 'image/crw', extension: '.crw' },
+    { mimetype: 'image/dcr', extension: '.dcr' },
+    { mimetype: 'image/dng', extension: '.dng' },
+    { mimetype: 'image/erf', extension: '.erf' },
+    { mimetype: 'image/fff', extension: '.fff' },
+    { mimetype: 'image/gif', extension: '.gif' },
+    { mimetype: 'image/heic', extension: '.heic' },
+    { mimetype: 'image/heif', extension: '.heif' },
+    { mimetype: 'image/hif', extension: '.hif' },
+    { mimetype: 'image/iiq', extension: '.iiq' },
+    { mimetype: 'image/jpeg', extension: '.jpe' },
+    { mimetype: 'image/jpeg', extension: '.jpeg' },
+    { mimetype: 'image/jpeg', extension: '.jpg' },
+    { mimetype: 'image/jxl', extension: '.jxl' },
+    { mimetype: 'image/k25', extension: '.k25' },
+    { mimetype: 'image/kdc', extension: '.kdc' },
+    { mimetype: 'image/mrw', extension: '.mrw' },
+    { mimetype: 'image/nef', extension: '.nef' },
+    { mimetype: 'image/orf', extension: '.orf' },
+    { mimetype: 'image/ori', extension: '.ori' },
+    { mimetype: 'image/pef', extension: '.pef' },
+    { mimetype: 'image/png', extension: '.png' },
+    { mimetype: 'image/psd', extension: '.psd' },
+    { mimetype: 'image/raf', extension: '.raf' },
+    { mimetype: 'image/raw', extension: '.raw' },
+    { mimetype: 'image/rwl', extension: '.rwl' },
+    { mimetype: 'image/sr2', extension: '.sr2' },
+    { mimetype: 'image/srf', extension: '.srf' },
+    { mimetype: 'image/srw', extension: '.srw' },
+    { mimetype: 'image/svg', extension: '.svg' },
+    { mimetype: 'image/tiff', extension: '.tif' },
+    { mimetype: 'image/tiff', extension: '.tiff' },
+    { mimetype: 'image/webp', extension: '.webp' },
+    { mimetype: 'image/vnd.adobe.photoshop', extension: '.psd' },
+    { mimetype: 'image/x-adobe-dng', extension: '.dng' },
+    { mimetype: 'image/x-arriflex-ari', extension: '.ari' },
+    { mimetype: 'image/x-canon-cr2', extension: '.cr2' },
+    { mimetype: 'image/x-canon-cr3', extension: '.cr3' },
+    { mimetype: 'image/x-canon-crw', extension: '.crw' },
+    { mimetype: 'image/x-epson-erf', extension: '.erf' },
+    { mimetype: 'image/x-fuji-raf', extension: '.raf' },
+    { mimetype: 'image/x-hasselblad-3fr', extension: '.3fr' },
+    { mimetype: 'image/x-hasselblad-fff', extension: '.fff' },
+    { mimetype: 'image/x-kodak-dcr', extension: '.dcr' },
+    { mimetype: 'image/x-kodak-k25', extension: '.k25' },
+    { mimetype: 'image/x-kodak-kdc', extension: '.kdc' },
+    { mimetype: 'image/x-leica-rwl', extension: '.rwl' },
+    { mimetype: 'image/x-minolta-mrw', extension: '.mrw' },
+    { mimetype: 'image/x-nikon-nef', extension: '.nef' },
+    { mimetype: 'image/x-olympus-orf', extension: '.orf' },
+    { mimetype: 'image/x-olympus-ori', extension: '.ori' },
+    { mimetype: 'image/x-panasonic-raw', extension: '.raw' },
+    { mimetype: 'image/x-pentax-pef', extension: '.pef' },
+    { mimetype: 'image/x-phantom-cin', extension: '.cin' },
+    { mimetype: 'image/x-phaseone-cap', extension: '.cap' },
+    { mimetype: 'image/x-phaseone-iiq', extension: '.iiq' },
+    { mimetype: 'image/x-samsung-srw', extension: '.srw' },
+    { mimetype: 'image/x-sigma-x3f', extension: '.x3f' },
+    { mimetype: 'image/x-sony-arw', extension: '.arw' },
+    { mimetype: 'image/x-sony-sr2', extension: '.sr2' },
+    { mimetype: 'image/x-sony-srf', extension: '.srf' },
+    { mimetype: 'image/x3f', extension: '.x3f' },
+    { mimetype: 'video/3gpp', extension: '.3gp' },
+    { mimetype: 'video/avi', extension: '.avi' },
+    { mimetype: 'video/mp2t', extension: '.m2ts' },
+    { mimetype: 'video/mp2t', extension: '.mts' },
+    { mimetype: 'video/mp4', extension: '.mp4' },
+    { mimetype: 'video/mpeg', extension: '.mpg' },
+    { mimetype: 'video/msvideo', extension: '.avi' },
+    { mimetype: 'video/quicktime', extension: '.mov' },
+    { mimetype: 'video/vnd.avi', extension: '.avi' },
+    { mimetype: 'video/webm', extension: '.webm' },
+    { mimetype: 'video/x-flv', extension: '.flv' },
+    { mimetype: 'video/x-matroska', extension: '.mkv' },
+    { mimetype: 'video/x-ms-wmv', extension: '.wmv' },
+    { mimetype: 'video/x-msvideo', extension: '.avi' },
+  ]) {
+    it(`should map ${extension} to ${mimetype}`, () => {
+      expect({ ...mimeTypes.image, ...mimeTypes.video }[extension]).toContain(mimetype);
+    });
+  }
+
+  describe('profile', () => {
+    it('should contain only lowercase mime types', () => {
+      const keys = Object.keys(mimeTypes.profile);
+      expect(keys).toEqual(keys.map((mimeType) => mimeType.toLowerCase()));
+
+      const values = Object.values(mimeTypes.profile).flat();
+      expect(values).toEqual(values.map((mimeType) => mimeType.toLowerCase()));
+    });
+
+    it('should be a sorted list', () => {
+      const keys = Object.keys(mimeTypes.profile);
+      // TODO: use toSorted in NodeJS 20.
+      expect(keys).toEqual([...keys].sort());
+    });
+
+    for (const [extension, v] of Object.entries(mimeTypes.profile)) {
+      it(`should lookup ${extension}`, () => {
+        expect(mimeTypes.lookup(`test.${extension}`)).toEqual(v[0]);
+      });
+    }
+  });
+
+  describe('image', () => {
+    it('should contain only lowercase mime types', () => {
+      const keys = Object.keys(mimeTypes.image);
+      expect(keys).toEqual(keys.map((mimeType) => mimeType.toLowerCase()));
+
+      const values = Object.values(mimeTypes.image).flat();
+      expect(values).toEqual(values.map((mimeType) => mimeType.toLowerCase()));
+    });
+
+    it('should be a sorted list', () => {
+      const keys = Object.keys(mimeTypes.image);
+      // TODO: use toSorted in NodeJS 20.
+      expect(keys).toEqual([...keys].sort());
+    });
+
+    it('should contain only image mime types', () => {
+      const values = Object.values(mimeTypes.image).flat();
+      expect(values).toEqual(values.filter((mimeType) => mimeType.startsWith('image/')));
+    });
+
+    for (const [extension, v] of Object.entries(mimeTypes.image)) {
+      it(`should lookup ${extension}`, () => {
+        expect(mimeTypes.lookup(`test.${extension}`)).toEqual(v[0]);
+      });
+    }
+  });
+
+  describe('video', () => {
+    it('should contain only lowercase mime types', () => {
+      const keys = Object.keys(mimeTypes.video);
+      expect(keys).toEqual(keys.map((mimeType) => mimeType.toLowerCase()));
+
+      const values = Object.values(mimeTypes.video).flat();
+      expect(values).toEqual(values.map((mimeType) => mimeType.toLowerCase()));
+    });
+
+    it('should be a sorted list', () => {
+      const keys = Object.keys(mimeTypes.video);
+      // TODO: use toSorted in NodeJS 20.
+      expect(keys).toEqual([...keys].sort());
+    });
+
+    it('should contain only video mime types', () => {
+      const values = Object.values(mimeTypes.video).flat();
+      expect(values).toEqual(values.filter((mimeType) => mimeType.startsWith('video/')));
+    });
+
+    for (const [extension, v] of Object.entries(mimeTypes.video)) {
+      it(`should lookup ${extension}`, () => {
+        expect(mimeTypes.lookup(`test.${extension}`)).toEqual(v[0]);
+      });
+    }
+  });
+
+  describe('sidecar', () => {
+    it('should contain only lowercase mime types', () => {
+      const keys = Object.keys(mimeTypes.sidecar);
+      expect(keys).toEqual(keys.map((mimeType) => mimeType.toLowerCase()));
+
+      const values = Object.values(mimeTypes.sidecar).flat();
+      expect(values).toEqual(values.map((mimeType) => mimeType.toLowerCase()));
+    });
+
+    it('should be a sorted list', () => {
+      const keys = Object.keys(mimeTypes.sidecar);
+      // TODO: use toSorted in NodeJS 20.
+      expect(keys).toEqual([...keys].sort());
+    });
+
+    it('should contain only xml mime types', () => {
+      expect(Object.values(mimeTypes.sidecar).flat()).toEqual(['application/xml', 'text/xml']);
+    });
+
+    for (const [extension, v] of Object.entries(mimeTypes.sidecar)) {
+      it(`should lookup ${extension}`, () => {
+        expect(mimeTypes.lookup(`it.${extension}`)).toEqual(v[0]);
+      });
+    }
+  });
+});

server/src/utils/mime-types.ts

@@ -0,0 +1,104 @@
+import { extname } from 'node:path';
+import { AssetType } from 'src/entities/asset.entity';
+
+const image: Record<string, string[]> = {
+  '.3fr': ['image/3fr', 'image/x-hasselblad-3fr'],
+  '.ari': ['image/ari', 'image/x-arriflex-ari'],
+  '.arw': ['image/arw', 'image/x-sony-arw'],
+  '.avif': ['image/avif'],
+  '.bmp': ['image/bmp'],
+  '.cap': ['image/cap', 'image/x-phaseone-cap'],
+  '.cin': ['image/cin', 'image/x-phantom-cin'],
+  '.cr2': ['image/cr2', 'image/x-canon-cr2'],
+  '.cr3': ['image/cr3', 'image/x-canon-cr3'],
+  '.crw': ['image/crw', 'image/x-canon-crw'],
+  '.dcr': ['image/dcr', 'image/x-kodak-dcr'],
+  '.dng': ['image/dng', 'image/x-adobe-dng'],
+  '.erf': ['image/erf', 'image/x-epson-erf'],
+  '.fff': ['image/fff', 'image/x-hasselblad-fff'],
+  '.gif': ['image/gif'],
+  '.heic': ['image/heic'],
+  '.heif': ['image/heif'],
+  '.hif': ['image/hif'],
+  '.iiq': ['image/iiq', 'image/x-phaseone-iiq'],
+  '.insp': ['image/jpeg'],
+  '.jpe': ['image/jpeg'],
+  '.jpeg': ['image/jpeg'],
+  '.jpg': ['image/jpeg'],
+  '.jxl': ['image/jxl'],
+  '.k25': ['image/k25', 'image/x-kodak-k25'],
+  '.kdc': ['image/kdc', 'image/x-kodak-kdc'],
+  '.mrw': ['image/mrw', 'image/x-minolta-mrw'],
+  '.nef': ['image/nef', 'image/x-nikon-nef'],
+  '.orf': ['image/orf', 'image/x-olympus-orf'],
+  '.ori': ['image/ori', 'image/x-olympus-ori'],
+  '.pef': ['image/pef', 'image/x-pentax-pef'],
+  '.png': ['image/png'],
+  '.psd': ['image/psd', 'image/vnd.adobe.photoshop'],
+  '.raf': ['image/raf', 'image/x-fuji-raf'],
+  '.raw': ['image/raw', 'image/x-panasonic-raw'],
+  '.rw2': ['image/rw2', 'image/x-panasonic-rw2'],
+  '.rwl': ['image/rwl', 'image/x-leica-rwl'],
+  '.sr2': ['image/sr2', 'image/x-sony-sr2'],
+  '.srf': ['image/srf', 'image/x-sony-srf'],
+  '.srw': ['image/srw', 'image/x-samsung-srw'],
+  '.svg': ['image/svg'],
+  '.tif': ['image/tiff'],
+  '.tiff': ['image/tiff'],
+  '.webp': ['image/webp'],
+  '.x3f': ['image/x3f', 'image/x-sigma-x3f'],
+};
+
+const profileExtensions = new Set(['.avif', '.dng', '.heic', '.heif', '.jpeg', '.jpg', '.png', '.webp', '.svg']);
+const profile: Record<string, string[]> = Object.fromEntries(
+  Object.entries(image).filter(([key]) => profileExtensions.has(key)),
+);
+
+const video: Record<string, string[]> = {
+  '.3gp': ['video/3gpp'],
+  '.avi': ['video/avi', 'video/msvideo', 'video/vnd.avi', 'video/x-msvideo'],
+  '.flv': ['video/x-flv'],
+  '.insv': ['video/mp4'],
+  '.m2ts': ['video/mp2t'],
+  '.m4v': ['video/x-m4v'],
+  '.mkv': ['video/x-matroska'],
+  '.mov': ['video/quicktime'],
+  '.mp4': ['video/mp4'],
+  '.mpg': ['video/mpeg'],
+  '.mts': ['video/mp2t'],
+  '.webm': ['video/webm'],
+  '.wmv': ['video/x-ms-wmv'],
+};
+
+const sidecar: Record<string, string[]> = {
+  '.xmp': ['application/xml', 'text/xml'],
+};
+
+const isType = (filename: string, r: Record<string, string[]>) => extname(filename).toLowerCase() in r;
+
+const lookup = (filename: string) =>
+  ({ ...image, ...video, ...sidecar })[extname(filename).toLowerCase()]?.[0] ?? 'application/octet-stream';
+
+export const mimeTypes = {
+  image,
+  profile,
+  sidecar,
+  video,
+
+  isAsset: (filename: string) => isType(filename, image) || isType(filename, video),
+  isImage: (filename: string) => isType(filename, image),
+  isProfile: (filename: string) => isType(filename, profile),
+  isSidecar: (filename: string) => isType(filename, sidecar),
+  isVideo: (filename: string) => isType(filename, video),
+  lookup,
+  assetType: (filename: string) => {
+    const contentType = lookup(filename);
+    if (contentType.startsWith('image/')) {
+      return AssetType.IMAGE;
+    } else if (contentType.startsWith('video/')) {
+      return AssetType.VIDEO;
+    }
+    return AssetType.OTHER;
+  },
+  getSupportedFileExtensions: () => [...Object.keys(image), ...Object.keys(video)],
+};

server/src/utils/misc.ts

@@ -0,0 +1,32 @@
+import { CLIP_MODEL_INFO } from 'src/constants';
+import { ImmichLogger } from 'src/utils/logger';
+
+export const isConnectionAborted = (error: Error | any) => error.code === 'ECONNABORTED';
+
+export const handlePromiseError = <T>(promise: Promise<T>, logger: ImmichLogger): void => {
+  promise.catch((error: Error | any) => logger.error(`Promise error: ${error}`, error?.stack));
+};
+
+export interface OpenGraphTags {
+  title: string;
+  description: string;
+  imageUrl?: string;
+}
+
+function cleanModelName(modelName: string): string {
+  const token = modelName.split('/').at(-1);
+  if (!token) {
+    throw new Error(`Invalid model name: ${modelName}`);
+  }
+
+  return token.replaceAll(':', '_');
+}
+
+export function getCLIPModelInfo(modelName: string) {
+  const modelInfo = CLIP_MODEL_INFO[cleanModelName(modelName)];
+  if (!modelInfo) {
+    throw new Error(`Unknown CLIP model: ${modelName}`);
+  }
+
+  return modelInfo;
+}

server/src/utils/pagination.ts

@@ -0,0 +1,79 @@
+import _ from 'lodash';
+import { FindManyOptions, ObjectLiteral, Repository, SelectQueryBuilder } from 'typeorm';
+
+export interface PaginationOptions {
+  take: number;
+  skip?: number;
+}
+
+export enum PaginationMode {
+  LIMIT_OFFSET = 'limit-offset',
+  SKIP_TAKE = 'skip-take',
+}
+
+export interface PaginatedBuilderOptions {
+  take: number;
+  skip?: number;
+  mode?: PaginationMode;
+}
+
+export interface PaginationResult<T> {
+  items: T[];
+  hasNextPage: boolean;
+}
+
+export type Paginated<T> = Promise<PaginationResult<T>>;
+
+export async function* usePagination<T>(
+  pageSize: number,
+  getNextPage: (pagination: PaginationOptions) => PaginationResult<T> | Paginated<T>,
+) {
+  let hasNextPage = true;
+
+  for (let skip = 0; hasNextPage; skip += pageSize) {
+    const result = await getNextPage({ take: pageSize, skip });
+    hasNextPage = result.hasNextPage;
+    yield result.items;
+  }
+}
+
+function paginationHelper<Entity extends ObjectLiteral>(items: Entity[], take: number): PaginationResult<Entity> {
+  const hasNextPage = items.length > take;
+  items.splice(take);
+
+  return { items, hasNextPage };
+}
+
+export async function paginate<Entity extends ObjectLiteral>(
+  repository: Repository<Entity>,
+  { take, skip }: PaginationOptions,
+  searchOptions?: FindManyOptions<Entity>,
+): Paginated<Entity> {
+  const items = await repository.find(
+    _.omitBy(
+      {
+        ...searchOptions,
+        // Take one more item to check if there's a next page
+        take: take + 1,
+        skip,
+      },
+      _.isUndefined,
+    ),
+  );
+
+  return paginationHelper(items, take);
+}
+
+export async function paginatedBuilder<Entity extends ObjectLiteral>(
+  qb: SelectQueryBuilder<Entity>,
+  { take, skip, mode }: PaginatedBuilderOptions,
+): Paginated<Entity> {
+  if (mode === PaginationMode.LIMIT_OFFSET) {
+    qb.limit(take + 1).offset(skip);
+  } else {
+    qb.take(take + 1).skip(skip);
+  }
+
+  const items = await qb.getMany();
+  return paginationHelper(items, take);
+}

server/src/utils/set.ts

@@ -0,0 +1,36 @@
+// NOTE: The following Set utils have been added here, to easily determine where they are used.
+//       They should be replaced with native Set operations, when they are added to the language.
+//       Proposal reference: https://github.com/tc39/proposal-set-methods
+
+export const setUnion = <T>(...sets: Set<T>[]): Set<T> => {
+  const union = new Set(sets[0]);
+  for (const set of sets.slice(1)) {
+    for (const element of set) {
+      union.add(element);
+    }
+  }
+  return union;
+};
+
+export const setDifference = <T>(setA: Set<T>, ...sets: Set<T>[]): Set<T> => {
+  const difference = new Set(setA);
+  for (const set of sets) {
+    for (const element of set) {
+      difference.delete(element);
+    }
+  }
+  return difference;
+};
+
+export const setIsSuperset = <T>(set: Set<T>, subset: Set<T>): boolean => {
+  for (const element of subset) {
+    if (!set.has(element)) {
+      return false;
+    }
+  }
+  return true;
+};
+
+export const setIsEqual = <T>(setA: Set<T>, setB: Set<T>): boolean => {
+  return setA.size === setB.size && setIsSuperset(setA, setB);
+};

server/src/utils/sql.ts

@@ -0,0 +1,215 @@
+#!/usr/bin/env node
+import { INestApplication } from '@nestjs/common';
+import { Reflector } from '@nestjs/core';
+import { Test } from '@nestjs/testing';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { mkdir, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { format } from 'sql-formatter';
+import { databaseConfig } from 'src/database.config';
+import { GENERATE_SQL_KEY, GenerateSqlQueries } from 'src/decorators';
+import { databaseEntities } from 'src/entities';
+import { ISystemConfigRepository } from 'src/interfaces/system-config.repository';
+import { AccessRepository } from 'src/repositories/access.repository';
+import { AlbumRepository } from 'src/repositories/album.repository';
+import { ApiKeyRepository } from 'src/repositories/api-key.repository';
+import { AssetRepository } from 'src/repositories/asset.repository';
+import { AuditRepository } from 'src/repositories/audit.repository';
+import { LibraryRepository } from 'src/repositories/library.repository';
+import { MoveRepository } from 'src/repositories/move.repository';
+import { PartnerRepository } from 'src/repositories/partner.repository';
+import { PersonRepository } from 'src/repositories/person.repository';
+import { SearchRepository } from 'src/repositories/search.repository';
+import { SharedLinkRepository } from 'src/repositories/shared-link.repository';
+import { SystemConfigRepository } from 'src/repositories/system-config.repository';
+import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
+import { TagRepository } from 'src/repositories/tag.repository';
+import { UserTokenRepository } from 'src/repositories/user-token.repository';
+import { UserRepository } from 'src/repositories/user.repository';
+import { Logger } from 'typeorm';
+
+export class SqlLogger implements Logger {
+  queries: string[] = [];
+  errors: Array<{ error: string | Error; query: string }> = [];
+
+  clear() {
+    this.queries = [];
+    this.errors = [];
+  }
+
+  logQuery(query: string) {
+    this.queries.push(format(query, { language: 'postgresql' }));
+  }
+
+  logQueryError(error: string | Error, query: string) {
+    this.errors.push({ error, query });
+  }
+
+  logQuerySlow() {}
+  logSchemaBuild() {}
+  logMigration() {}
+  log() {}
+}
+
+const reflector = new Reflector();
+const repositories = [
+  AccessRepository,
+  AlbumRepository,
+  ApiKeyRepository,
+  AssetRepository,
+  AuditRepository,
+  LibraryRepository,
+  MoveRepository,
+  PartnerRepository,
+  PersonRepository,
+  SharedLinkRepository,
+  SearchRepository,
+  SystemConfigRepository,
+  SystemMetadataRepository,
+  TagRepository,
+  UserTokenRepository,
+  UserRepository,
+];
+
+type Repository = (typeof repositories)[0];
+type SqlGeneratorOptions = { targetDir: string };
+
+class SqlGenerator {
+  private app: INestApplication | null = null;
+  private sqlLogger = new SqlLogger();
+  private results: Record<string, string[]> = {};
+
+  constructor(private options: SqlGeneratorOptions) {}
+
+  async run() {
+    try {
+      await this.setup();
+      for (const Repository of repositories) {
+        await this.process(Repository);
+      }
+      await this.write();
+      this.stats();
+    } finally {
+      await this.close();
+    }
+  }
+
+  private async setup() {
+    await rm(this.options.targetDir, { force: true, recursive: true });
+    await mkdir(this.options.targetDir);
+
+    const moduleFixture = await Test.createTestingModule({
+      imports: [
+        TypeOrmModule.forRoot({
+          ...databaseConfig,
+          entities: databaseEntities,
+          logging: ['query'],
+          logger: this.sqlLogger,
+        }),
+        TypeOrmModule.forFeature(databaseEntities),
+      ],
+      providers: [{ provide: ISystemConfigRepository, useClass: SystemConfigRepository }, ...repositories],
+    }).compile();
+
+    this.app = await moduleFixture.createNestApplication().init();
+  }
+
+  async process(Repository: Repository) {
+    if (!this.app) {
+      throw new Error('Not initialized');
+    }
+
+    const data: string[] = [`-- NOTE: This file is auto generated by ./sql-generator`];
+    const instance = this.app.get<Repository>(Repository);
+
+    // normal repositories
+    data.push(...(await this.runTargets(instance, `${Repository.name}`)));
+
+    // nested repositories
+    if (Repository.name === AccessRepository.name) {
+      for (const key of Object.keys(instance)) {
+        const subInstance = (instance as any)[key];
+        data.push(...(await this.runTargets(subInstance, `${Repository.name}.${key}`)));
+      }
+    }
+
+    this.results[Repository.name] = data;
+  }
+
+  private async runTargets(instance: any, label: string) {
+    const data: string[] = [];
+
+    for (const key of this.getPropertyNames(instance)) {
+      const target = instance[key];
+      if (!(target instanceof Function)) {
+        continue;
+      }
+
+      const queries = reflector.get<GenerateSqlQueries[] | undefined>(GENERATE_SQL_KEY, target);
+      if (!queries) {
+        continue;
+      }
+
+      // empty decorator implies calling with no arguments
+      if (queries.length === 0) {
+        queries.push({ params: [] });
+      }
+
+      for (const { name, params } of queries) {
+        let queryLabel = `${label}.${key}`;
+        if (name) {
+          queryLabel += ` (${name})`;
+        }
+
+        this.sqlLogger.clear();
+
+        // errors still generate sql, which is all we care about
+        await target.apply(instance, params).catch((error: Error) => console.error(`${queryLabel} error: ${error}`));
+
+        if (this.sqlLogger.queries.length === 0) {
+          console.warn(`No queries recorded for ${queryLabel}`);
+          continue;
+        }
+
+        data.push([`-- ${queryLabel}`, ...this.sqlLogger.queries].join('\n'));
+      }
+    }
+
+    return data;
+  }
+
+  private async write() {
+    for (const [repoName, data] of Object.entries(this.results)) {
+      const filename = repoName.replaceAll(/[A-Z]/g, (letter) => `.${letter.toLowerCase()}`).replace('.', '');
+      const file = join(this.options.targetDir, `${filename}.sql`);
+      await writeFile(file, data.join('\n\n') + '\n');
+    }
+  }
+
+  private stats() {
+    console.log(`Wrote ${Object.keys(this.results).length} files`);
+    console.log(`Generated ${Object.values(this.results).flat().length} queries`);
+  }
+
+  private async close() {
+    if (this.app) {
+      await this.app.close();
+    }
+  }
+
+  private getPropertyNames(instance: any): string[] {
+    return Object.getOwnPropertyNames(Object.getPrototypeOf(instance)) as any[];
+  }
+}
+
+new SqlGenerator({ targetDir: './src/queries' })
+  .run()
+  .then(() => {
+    console.log('Done');
+    process.exit(0);
+  })
+  .catch((error) => {
+    console.error(error);
+    console.log('Something went wrong');
+    process.exit(1);
+  });

server/src/utils/version.spec.ts

@@ -0,0 +1,72 @@
+import { Version, VersionType } from 'src/utils/version';
+
+describe('Version', () => {
+  const tests = [
+    { this: new Version(0, 0, 1), other: new Version(0, 0, 0), compare: 1, type: VersionType.PATCH },
+    { this: new Version(0, 1, 0), other: new Version(0, 0, 0), compare: 1, type: VersionType.MINOR },
+    { this: new Version(1, 0, 0), other: new Version(0, 0, 0), compare: 1, type: VersionType.MAJOR },
+    { this: new Version(0, 0, 0), other: new Version(0, 0, 1), compare: -1, type: VersionType.PATCH },
+    { this: new Version(0, 0, 0), other: new Version(0, 1, 0), compare: -1, type: VersionType.MINOR },
+    { this: new Version(0, 0, 0), other: new Version(1, 0, 0), compare: -1, type: VersionType.MAJOR },
+    { this: new Version(0, 0, 0), other: new Version(0, 0, 0), compare: 0, type: VersionType.EQUAL },
+    { this: new Version(0, 0, 1), other: new Version(0, 0, 1), compare: 0, type: VersionType.EQUAL },
+    { this: new Version(0, 1, 0), other: new Version(0, 1, 0), compare: 0, type: VersionType.EQUAL },
+    { this: new Version(1, 0, 0), other: new Version(1, 0, 0), compare: 0, type: VersionType.EQUAL },
+    { this: new Version(1, 0), other: new Version(1, 0, 0), compare: 0, type: VersionType.EQUAL },
+    { this: new Version(1, 0), other: new Version(1, 0, 1), compare: -1, type: VersionType.PATCH },
+    { this: new Version(1, 1), other: new Version(1, 0, 1), compare: 1, type: VersionType.MINOR },
+    { this: new Version(1), other: new Version(1, 0, 0), compare: 0, type: VersionType.EQUAL },
+    { this: new Version(1), other: new Version(1, 0, 1), compare: -1, type: VersionType.PATCH },
+  ];
+
+  describe('isOlderThan', () => {
+    for (const { this: thisVersion, other: otherVersion, compare, type } of tests) {
+      const expected = compare < 0 ? type : VersionType.EQUAL;
+      it(`should return '${expected}' when comparing ${thisVersion} to ${otherVersion}`, () => {
+        expect(thisVersion.isOlderThan(otherVersion)).toEqual(expected);
+      });
+    }
+  });
+
+  describe('isEqual', () => {
+    for (const { this: thisVersion, other: otherVersion, compare } of tests) {
+      const bool = compare === 0;
+      it(`should return ${bool} when comparing ${thisVersion} to ${otherVersion}`, () => {
+        expect(thisVersion.isEqual(otherVersion)).toEqual(bool);
+      });
+    }
+  });
+
+  describe('isNewerThan', () => {
+    for (const { this: thisVersion, other: otherVersion, compare, type } of tests) {
+      const expected = compare > 0 ? type : VersionType.EQUAL;
+      it(`should return ${expected} when comparing ${thisVersion} to ${otherVersion}`, () => {
+        expect(thisVersion.isNewerThan(otherVersion)).toEqual(expected);
+      });
+    }
+  });
+
+  describe('fromString', () => {
+    const tests = [
+      { scenario: 'leading v', value: 'v1.72.2', expected: new Version(1, 72, 2) },
+      { scenario: 'uppercase v', value: 'V1.72.2', expected: new Version(1, 72, 2) },
+      { scenario: 'missing v', value: '1.72.2', expected: new Version(1, 72, 2) },
+      { scenario: 'large patch', value: '1.72.123', expected: new Version(1, 72, 123) },
+      { scenario: 'large minor', value: '1.123.0', expected: new Version(1, 123, 0) },
+      { scenario: 'large major', value: '123.0.0', expected: new Version(123, 0, 0) },
+      { scenario: 'major bump', value: 'v2.0.0', expected: new Version(2, 0, 0) },
+      { scenario: 'has dash', value: '14.10-1', expected: new Version(14, 10, 1) },
+      { scenario: 'missing patch', value: '14.10', expected: new Version(14, 10, 0) },
+      { scenario: 'only major', value: '14', expected: new Version(14, 0, 0) },
+    ];
+
+    for (const { scenario, value, expected } of tests) {
+      it(`should correctly parse ${scenario}`, () => {
+        const actual = Version.fromString(value);
+        expect(actual.major).toEqual(expected.major);
+        expect(actual.minor).toEqual(expected.minor);
+        expect(actual.patch).toEqual(expected.patch);
+      });
+    }
+  });
+});

server/src/utils/version.ts

@@ -0,0 +1,64 @@
+export type IVersion = { major: number; minor: number; patch: number };
+
+export enum VersionType {
+  EQUAL = 0,
+  PATCH = 1,
+  MINOR = 2,
+  MAJOR = 3,
+}
+
+export class Version implements IVersion {
+  public readonly types = ['major', 'minor', 'patch'] as const;
+
+  constructor(
+    public major: number,
+    public minor: number = 0,
+    public patch: number = 0,
+  ) {}
+
+  toString() {
+    return `${this.major}.${this.minor}.${this.patch}`;
+  }
+
+  toJSON() {
+    const { major, minor, patch } = this;
+    return { major, minor, patch };
+  }
+
+  static fromString(version: string): Version {
+    const regex = /v?(?<major>\d+)(?:\.(?<minor>\d+))?(?:[.-](?<patch>\d+))?/i;
+    const matchResult = version.match(regex);
+    if (matchResult) {
+      const { major, minor = '0', patch = '0' } = matchResult.groups as { [K in keyof IVersion]: string };
+      return new Version(Number(major), Number(minor), Number(patch));
+    } else {
+      throw new Error(`Invalid version format: ${version}`);
+    }
+  }
+
+  private compare(version: Version): [number, VersionType] {
+    for (const [i, key] of this.types.entries()) {
+      const diff = this[key] - version[key];
+      if (diff !== 0) {
+        return [diff > 0 ? 1 : -1, (VersionType.MAJOR - i) as VersionType];
+      }
+    }
+
+    return [0, VersionType.EQUAL];
+  }
+
+  isOlderThan(version: Version): VersionType {
+    const [bool, type] = this.compare(version);
+    return bool < 0 ? type : VersionType.EQUAL;
+  }
+
+  isEqual(version: Version): boolean {
+    const [bool] = this.compare(version);
+    return bool === 0;
+  }
+
+  isNewerThan(version: Version): VersionType {
+    const [bool, type] = this.compare(version);
+    return bool > 0 ? type : VersionType.EQUAL;
+  }
+}