feat(server): fully accelerated nvenc (#9452)

* use arrayContaining * libplacebo for nvenc update dockerfile * tweaks * update nvenc options * tweak settings * refactor * toggle for hardware decoding, software / hardware decoding for nvenc and rkmpp * fix software tone-mapping not being applied * separate configs for hw/sw * update api * add hw decode toggle * fix mutating config * remove `version` flag * fix config type * remove submodule * handle temporal AQ * remove duplicate tests * use `tonemap_opencl` * wording * update docs
2025-12-22 17:24:56 +03:00 · 2024-05-16 13:30:26 -04:00
parent 64636c0618
commit d8eca168ca
15 changed files with 255 additions and 64 deletions
--- a/server/src/config.ts
+++ b/server/src/config.ts
@@ -97,6 +97,7 @@ export interface SystemConfig {
    preferredHwDevice: string;
    transcode: TranscodePolicy;
    accel: TranscodeHWAccel;
+    accelDecode: boolean;
    tonemap: ToneMapping;
  };
  job: Record<ConcurrentQueueName, { concurrency: number }>;
@@ -228,6 +229,7 @@ export const defaults = Object.freeze<SystemConfig>({
    transcode: TranscodePolicy.REQUIRED,
    tonemap: ToneMapping.HABLE,
    accel: TranscodeHWAccel.DISABLED,
+    accelDecode: false,
  },
  job: {
    [QueueName.BACKGROUND_TASK]: { concurrency: 5 },
--- a/server/src/dtos/system-config.dto.ts
+++ b/server/src/dtos/system-config.dto.ts
@@ -132,6 +132,9 @@ export class SystemConfigFFmpegDto {
  @ApiProperty({ enumName: 'TranscodeHWAccel', enum: TranscodeHWAccel })
  accel!: TranscodeHWAccel;

+  @ValidateBoolean()
+  accelDecode!: boolean;
+
  @IsEnum(ToneMapping)
  @ApiProperty({ enumName: 'ToneMapping', enum: ToneMapping })
  tonemap!: ToneMapping;
--- a/server/src/services/media.service.spec.ts
+++ b/server/src/services/media.service.spec.ts
@@ -1335,6 +1335,51 @@ describe(MediaService.name, () => {
      );
    });

+    it('should use hardware decoding for nvenc if enabled', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.NVENC, accelDecode: true },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: expect.arrayContaining([
+            '-hwaccel cuda',
+            '-hwaccel_output_format cuda',
+            '-noautorotate',
+            '-threads 1',
+          ]),
+          outputOptions: expect.arrayContaining([expect.stringContaining('scale_cuda=-2:720:format=nv12')]),
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should use hardware tone-mapping for nvenc if hardware decoding is enabled and should tone map', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.NVENC, accelDecode: true },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: expect.arrayContaining(['-hwaccel cuda', '-hwaccel_output_format cuda']),
+          outputOptions: expect.arrayContaining([
+            expect.stringContaining(
+              'tonemap_cuda=desat=0:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709:format=nv12',
+            ),
+          ]),
+          twoPass: false,
+        },
+      );
+    });
+
    it('should set options for qsv', async () => {
      storageMock.readdir.mockResolvedValue(['renderD128']);
      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
@@ -1633,8 +1678,9 @@ describe(MediaService.name, () => {

    it('should set options for rkmpp', async () => {
      storageMock.readdir.mockResolvedValue(['renderD128']);
+      storageMock.stat.mockResolvedValue({ ...new Stats(), isFile: () => true, isCharacterDevice: () => true });
      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
-      systemMock.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHWAccel.RKMPP } });
+      systemMock.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHWAccel.RKMPP, accelDecode: true } });
      assetMock.getByIds.mockResolvedValue([assetStub.video]);
      await sut.handleVideoConversion({ id: assetStub.video.id });
      expect(mediaMock.transcode).toHaveBeenCalledWith(
@@ -1663,10 +1709,12 @@ describe(MediaService.name, () => {

    it('should set vbr options for rkmpp when max bitrate is enabled', async () => {
      storageMock.readdir.mockResolvedValue(['renderD128']);
+      storageMock.stat.mockResolvedValue({ ...new Stats(), isFile: () => true, isCharacterDevice: () => true });
      mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
      systemMock.get.mockResolvedValue({
        ffmpeg: {
          accel: TranscodeHWAccel.RKMPP,
+          accelDecode: true,
          maxBitrate: '10000k',
          targetVideoCodec: VideoCodec.HEVC,
        },
@@ -1686,9 +1734,10 @@ describe(MediaService.name, () => {

    it('should set cqp options for rkmpp when max bitrate is disabled', async () => {
      storageMock.readdir.mockResolvedValue(['renderD128']);
+      storageMock.stat.mockResolvedValue({ ...new Stats(), isFile: () => true, isCharacterDevice: () => true });
      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
      systemMock.get.mockResolvedValue({
-        ffmpeg: { accel: TranscodeHWAccel.RKMPP, crf: 30, maxBitrate: '0' },
+        ffmpeg: { accel: TranscodeHWAccel.RKMPP, accelDecode: true, crf: 30, maxBitrate: '0' },
      });
      assetMock.getByIds.mockResolvedValue([assetStub.video]);
      await sut.handleVideoConversion({ id: assetStub.video.id });
@@ -1707,7 +1756,9 @@ describe(MediaService.name, () => {
      storageMock.readdir.mockResolvedValue(['renderD128']);
      storageMock.stat.mockResolvedValue({ ...new Stats(), isFile: () => true, isCharacterDevice: () => true });
      mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
-      systemMock.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHWAccel.RKMPP, crf: 30, maxBitrate: '0' } });
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.RKMPP, accelDecode: true, crf: 30, maxBitrate: '0' },
+      });
      assetMock.getByIds.mockResolvedValue([assetStub.video]);
      await sut.handleVideoConversion({ id: assetStub.video.id });
      expect(mediaMock.transcode).toHaveBeenCalledWith(
@@ -1724,6 +1775,54 @@ describe(MediaService.name, () => {
        },
      );
    });
+
+    it('should use software decoding and tone-mapping if hardware decoding is disabled', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      storageMock.stat.mockResolvedValue({ ...new Stats(), isFile: () => true, isCharacterDevice: () => true });
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.RKMPP, accelDecode: false, crf: 30, maxBitrate: '0' },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: expect.arrayContaining([
+            expect.stringContaining(
+              'zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:range=pc,format=yuv420p',
+            ),
+          ]),
+          twoPass: false,
+        },
+      );
+    });
+
+    it('should use software decoding and tone-mapping if opencl is not available', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      storageMock.stat.mockResolvedValue({ ...new Stats(), isFile: () => false, isCharacterDevice: () => false });
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.RKMPP, accelDecode: true, crf: 30, maxBitrate: '0' },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: expect.arrayContaining([
+            expect.stringContaining(
+              'zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:range=pc,format=yuv420p',
+            ),
+          ]),
+          twoPass: false,
+        },
+      );
+    });
  });

  it('should tonemap when policy is required and video is hdr', async () => {
--- a/server/src/services/media.service.ts
+++ b/server/src/services/media.service.ts
@@ -36,9 +36,11 @@ import {
  AV1Config,
  H264Config,
  HEVCConfig,
-  NVENCConfig,
+  NvencHwDecodeConfig,
+  NvencSwDecodeConfig,
  QSVConfig,
-  RKMPPConfig,
+  RkmppHwDecodeConfig,
+  RkmppSwDecodeConfig,
  ThumbnailConfig,
  VAAPIConfig,
  VP9Config,
@@ -360,8 +362,7 @@ export class MediaService {
          `Error occurred during transcoding. Retrying with ${config.accel.toUpperCase()} acceleration disabled.`,
        );
      }
-      config.accel = TranscodeHWAccel.DISABLED;
-      transcodeOptions = await this.getCodecConfig(config).then((c) =>
+      transcodeOptions = await this.getCodecConfig({ ...config, accel: TranscodeHWAccel.DISABLED }).then((c) =>
        c.getOptions(target, mainVideoStream, mainAudioStream),
      );
      await this.mediaRepository.transcode(input, output, transcodeOptions);
@@ -494,7 +495,7 @@ export class MediaService {
    let handler: VideoCodecHWConfig;
    switch (config.accel) {
      case TranscodeHWAccel.NVENC: {
-        handler = new NVENCConfig(config);
+        handler = config.accelDecode ? new NvencHwDecodeConfig(config) : new NvencSwDecodeConfig(config);
        break;
      }
      case TranscodeHWAccel.QSV: {
@@ -506,7 +507,10 @@ export class MediaService {
        break;
      }
      case TranscodeHWAccel.RKMPP: {
-        handler = new RKMPPConfig(config, await this.getDevices(), await this.hasOpenCL());
+        handler =
+          config.accelDecode && (await this.hasOpenCL())
+            ? new RkmppHwDecodeConfig(config, await this.getDevices())
+            : new RkmppSwDecodeConfig(config, await this.getDevices());
        break;
      }
      default: {
--- a/server/src/services/system-config.service.spec.ts
+++ b/server/src/services/system-config.service.spec.ts
@@ -66,6 +66,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
    preferredHwDevice: 'auto',
    transcode: TranscodePolicy.REQUIRED,
    accel: TranscodeHWAccel.DISABLED,
+    accelDecode: false,
    tonemap: ToneMapping.HABLE,
  },
  logging: {
--- a/server/src/utils/media.ts
+++ b/server/src/utils/media.ts
@@ -26,14 +26,18 @@ class BaseConfig implements VideoCodecSWConfig {
      }
    }

-    options.outputOptions.push(...this.getPresetOptions(), ...this.getThreadOptions(), ...this.getBitrateOptions());
+    options.outputOptions.push(
+      ...this.getPresetOptions(),
+      ...this.getOutputThreadOptions(),
+      ...this.getBitrateOptions(),
+    );

    return options;
  }

  // eslint-disable-next-line @typescript-eslint/no-unused-vars
  getBaseInputOptions(videoStream: VideoStreamInfo): string[] {
-    return [];
+    return this.getInputThreadOptions();
  }

  getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
@@ -80,11 +84,7 @@ class BaseConfig implements VideoCodecSWConfig {
      options.push(`scale=${this.getScaling(videoStream)}`);
    }

-    if (this.shouldToneMap(videoStream)) {
-      options.push(...this.getToneMapping());
-    }
-    options.push('format=yuv420p');
-
+    options.push(...this.getToneMapping(videoStream), 'format=yuv420p');
    return options;
  }

@@ -112,7 +112,11 @@ class BaseConfig implements VideoCodecSWConfig {
    }
  }

-  getThreadOptions(): Array<string> {
+  getInputThreadOptions(): Array<string> {
+    return [];
+  }
+
+  getOutputThreadOptions(): Array<string> {
    if (this.config.threads <= 0) {
      return [];
    }
@@ -218,7 +222,11 @@ class BaseConfig implements VideoCodecSWConfig {
    }
  }

-  getToneMapping() {
+  getToneMapping(videoStream: VideoStreamInfo) {
+    if (!this.shouldToneMap(videoStream)) {
+      return [];
+    }
+
    const colors = this.getColors();

    return [
@@ -348,8 +356,8 @@ export class ThumbnailConfig extends BaseConfig {
 }

 export class H264Config extends BaseConfig {
-  getThreadOptions() {
-    const options = super.getThreadOptions();
+  getOutputThreadOptions() {
+    const options = super.getOutputThreadOptions();
    if (this.config.threads === 1) {
      options.push('-x264-params frame-threads=1:pools=none');
    }
@@ -359,8 +367,8 @@ export class H264Config extends BaseConfig {
 }

 export class HEVCConfig extends BaseConfig {
-  getThreadOptions() {
-    const options = super.getThreadOptions();
+  getOutputThreadOptions() {
+    const options = super.getOutputThreadOptions();
    if (this.config.threads === 1) {
      options.push('-x265-params frame-threads=1:pools=none');
    }
@@ -391,8 +399,8 @@ export class VP9Config extends BaseConfig {
    return [`-${this.useCQP() ? 'q:v' : 'crf'} ${this.config.crf}`, `-b:v ${bitrates.max}${bitrates.unit}`];
  }

-  getThreadOptions() {
-    return ['-row-mt 1', ...super.getThreadOptions()];
+  getOutputThreadOptions() {
+    return ['-row-mt 1', ...super.getOutputThreadOptions()];
  }

  eligibleForTwoPass() {
@@ -425,7 +433,7 @@ export class AV1Config extends BaseConfig {
    return options;
  }

-  getThreadOptions() {
+  getOutputThreadOptions() {
    return []; // Already set above with svtav1-params
  }

@@ -434,7 +442,7 @@ export class AV1Config extends BaseConfig {
  }
 }

-export class NVENCConfig extends BaseHWConfig {
+export class NvencSwDecodeConfig extends BaseHWConfig {
  getSupportedCodecs() {
    return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.AV1];
  }
@@ -462,7 +470,7 @@ export class NVENCConfig extends BaseHWConfig {
  }

  getFilterOptions(videoStream: VideoStreamInfo) {
-    const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
+    const options = this.getToneMapping(videoStream);
    options.push('format=nv12', 'hwupload_cuda');
    if (this.shouldScale(videoStream)) {
      options.push(`scale_cuda=${this.getScaling(videoStream)}`);
@@ -513,6 +521,52 @@ export class NVENCConfig extends BaseHWConfig {
  }
 }

+export class NvencHwDecodeConfig extends NvencSwDecodeConfig {
+  getBaseInputOptions() {
+    return ['-hwaccel cuda', '-hwaccel_output_format cuda', '-noautorotate', ...this.getInputThreadOptions()];
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo) {
+    const options = [];
+    if (this.shouldScale(videoStream)) {
+      options.push(`scale_cuda=${this.getScaling(videoStream)}`);
+    }
+    options.push(...this.getToneMapping(videoStream));
+    if (options.length > 0) {
+      options[options.length - 1] += ':format=nv12';
+    } else {
+      options.push('format=nv12');
+    }
+    return options;
+  }
+
+  getToneMapping(videoStream: VideoStreamInfo) {
+    if (!this.shouldToneMap(videoStream)) {
+      return [];
+    }
+
+    const colors = this.getColors();
+    const tonemapOptions = [
+      'desat=0',
+      `matrix=${colors.matrix}`,
+      `primaries=${colors.primaries}`,
+      'range=pc',
+      `tonemap=${this.config.tonemap}`,
+      `transfer=${colors.transfer}`,
+    ];
+
+    return [`tonemap_cuda=${tonemapOptions.join(':')}`];
+  }
+
+  getInputThreadOptions() {
+    return [`-threads ${this.config.threads <= 0 ? 1 : this.config.threads}`];
+  }
+
+  getOutputThreadOptions() {
+    return [];
+  }
+}
+
 export class QSVConfig extends BaseHWConfig {
  getBaseInputOptions() {
    if (this.devices.length === 0) {
@@ -538,7 +592,7 @@ export class QSVConfig extends BaseHWConfig {
  }

  getFilterOptions(videoStream: VideoStreamInfo) {
-    const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
+    const options = this.getToneMapping(videoStream);
    options.push('format=nv12', 'hwupload=extra_hw_frames=64');
    if (this.shouldScale(videoStream)) {
      options.push(`scale_qsv=${this.getScaling(videoStream)}`);
@@ -604,7 +658,7 @@ export class VAAPIConfig extends BaseHWConfig {
  }

  getFilterOptions(videoStream: VideoStreamInfo) {
-    const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
+    const options = this.getToneMapping(videoStream);
    options.push('format=nv12', 'hwupload');
    if (this.shouldScale(videoStream)) {
      options.push(`scale_vaapi=${this.getScaling(videoStream)}`);
@@ -656,47 +710,22 @@ export class VAAPIConfig extends BaseHWConfig {
  }
 }

-export class RKMPPConfig extends BaseHWConfig {
-  private hasOpenCL: boolean;
-
+export class RkmppSwDecodeConfig extends BaseHWConfig {
  constructor(
    protected config: SystemConfigFFmpegDto,
    devices: string[] = [],
-    hasOpenCL: boolean = false,
  ) {
    super(config, devices);
-    this.hasOpenCL = hasOpenCL;
  }

  eligibleForTwoPass(): boolean {
    return false;
  }

-  getBaseInputOptions(videoStream: VideoStreamInfo) {
+  getBaseInputOptions(): string[] {
    if (this.devices.length === 0) {
      throw new Error('No RKMPP device found');
    }
-    return this.shouldToneMap(videoStream) && !this.hasOpenCL
-      ? [] // disable hardware decoding & filters
-      : ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga'];
-  }
-
-  getFilterOptions(videoStream: VideoStreamInfo) {
-    if (this.shouldToneMap(videoStream)) {
-      if (!this.hasOpenCL) {
-        return super.getFilterOptions(videoStream);
-      }
-      const colors = this.getColors();
-      return [
-        `scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1`,
-        'hwmap=derive_device=opencl:mode=read',
-        `tonemap_opencl=format=nv12:r=pc:p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:tonemap=${this.config.tonemap}:desat=0`,
-        'hwmap=derive_device=rkmpp:mode=write:reverse=1',
-        'format=drm_prime',
-      ];
-    } else if (this.shouldScale(videoStream)) {
-      return [`scale_rkrga=${this.getScaling(videoStream)}:format=nv12:afbc=1`];
-    }
    return [];
  }

@@ -734,3 +763,29 @@ export class RKMPPConfig extends BaseHWConfig {
    return `${this.config.targetVideoCodec}_rkmpp`;
  }
 }
+
+export class RkmppHwDecodeConfig extends RkmppSwDecodeConfig {
+  getBaseInputOptions() {
+    if (this.devices.length === 0) {
+      throw new Error('No RKMPP device found');
+    }
+
+    return ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga'];
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo) {
+    if (this.shouldToneMap(videoStream)) {
+      const colors = this.getColors();
+      return [
+        `scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1`,
+        'hwmap=derive_device=opencl:mode=read',
+        `tonemap_opencl=format=nv12:r=pc:p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:tonemap=${this.config.tonemap}:desat=0`,
+        'hwmap=derive_device=rkmpp:mode=write:reverse=1',
+        'format=drm_prime',
+      ];
+    } else if (this.shouldScale(videoStream)) {
+      return [`scale_rkrga=${this.getScaling(videoStream)}:format=nv12:afbc=1`];
+    }
+    return [];
+  }
+}