feat(server): hardware video acceleration for Rockchip SOCs via RKMPP (#4645)

* feat(server): hardware video acceleration for Rockchip SOCs via RKMPP

* add tests

* use LD_LIBRARY_PATH for custom ffmpeg

* incorporate review feedback

* code re-use for ffmpeg call

* review feedback

server/src/domain/media/media.service.spec.ts

@@ -1508,6 +1508,83 @@ describe(MediaService.name, () => {
       await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false);
       expect(mediaMock.transcode).not.toHaveBeenCalled();
     });
+
+    it('should set vbr options for rkmpp when max bitrate is enabled', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
+        { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },
+        { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: [
+            `-c:v hevc_rkmpp_encoder`,
+            '-c:a aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
+            '-v verbose',
+            '-level 153',
+            '-rc_mode 3',
+            '-quality_min 0',
+            '-quality_max 100',
+            '-b:v 10000k',
+            '-width 1280',
+            '-height 720',
+          ],
+          twoPass: false,
+          ffmpegPath: 'ffmpeg_mpp',
+          ldLibraryPath: '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp',
+        },
+      );
+    });
+
+    it('should set cqp options for rkmpp when max bitrate is disabled', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
+        { key: SystemConfigKey.FFMPEG_CRF, value: 30 },
+        { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '0' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: [
+            `-c:v h264_rkmpp_encoder`,
+            '-c:a aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
+            '-v verbose',
+            '-level 51',
+            '-rc_mode 2',
+            '-quality_min 51',
+            '-quality_max 51',
+            '-width 1280',
+            '-height 720',
+          ],
+          twoPass: false,
+          ffmpegPath: 'ffmpeg_mpp',
+          ldLibraryPath: '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp',
+        },
+      );
+    });
   });
 
   it('should tonemap when policy is required and video is hdr', async () => {

server/src/domain/media/media.service.ts

@@ -26,7 +26,16 @@ import {
 import { StorageCore, StorageFolder } from '../storage';
 import { SystemConfigFFmpegDto } from '../system-config';
 import { SystemConfigCore } from '../system-config/system-config.core';
-import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIConfig, VP9Config } from './media.util';
+import {
+  H264Config,
+  HEVCConfig,
+  NVENCConfig,
+  QSVConfig,
+  RKMPPConfig,
+  ThumbnailConfig,
+  VAAPIConfig,
+  VP9Config,
+} from './media.util';
 
 @Injectable()
 export class MediaService {
@@ -352,6 +361,10 @@ export class MediaService {
         devices = await this.storageRepository.readdir('/dev/dri');
         handler = new VAAPIConfig(config, devices);
         break;
+      case TranscodeHWAccel.RKMPP:
+        devices = await this.storageRepository.readdir('/dev/dri');
+        handler = new RKMPPConfig(config, devices);
+        break;
       default:
         throw new UnsupportedMediaTypeException(`${config.accel.toUpperCase()} acceleration is unsupported`);
     }

server/src/domain/media/media.util.ts

@@ -143,6 +143,13 @@ class BaseConfig implements VideoCodecSWConfig {
     return this.isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
   }
 
+  getSize(videoStream: VideoStreamInfo) {
+    const smaller = this.getTargetResolution(videoStream);
+    const factor = Math.max(videoStream.height, videoStream.width) / Math.min(videoStream.height, videoStream.width);
+    const larger = Math.round(smaller * factor);
+    return this.isVideoVertical(videoStream) ? { width: smaller, height: larger } : { width: larger, height: smaller };
+  }
+
   isVideoRotated(videoStream: VideoStreamInfo) {
     return Math.abs(videoStream.rotation) === 90;
   }
@@ -555,3 +562,68 @@ export class VAAPIConfig extends BaseHWConfig {
     return this.config.cqMode !== CQMode.ICQ || this.config.targetVideoCodec === VideoCodec.VP9;
   }
 }
+
+export class RKMPPConfig extends BaseHWConfig {
+  getOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo): TranscodeOptions {
+    const options = super.getOptions(videoStream, audioStream);
+    options.ffmpegPath = 'ffmpeg_mpp';
+    options.ldLibraryPath = '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp';
+    options.outputOptions.push(...this.getSizeOptions(videoStream));
+    return options;
+  }
+
+  eligibleForTwoPass(): boolean {
+    return false;
+  }
+
+  getBaseInputOptions() {
+    if (this.devices.length === 0) {
+      throw Error('No RKMPP device found');
+    }
+    return [];
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo) {
+    return this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
+  }
+
+  getSizeOptions(videoStream: VideoStreamInfo) {
+    if (this.shouldScale(videoStream)) {
+      const { width, height } = this.getSize(videoStream);
+      return [`-width ${width}`, `-height ${height}`];
+    }
+    return [];
+  }
+
+  getPresetOptions() {
+    switch (this.config.targetVideoCodec) {
+      case VideoCodec.H264:
+        // from ffmpeg_mpp help, commonly referred to as H264 level 5.1
+        return ['-level 51'];
+      case VideoCodec.HEVC:
+        // from ffmpeg_mpp help, commonly referred to as HEVC level 5.1
+        return ['-level 153'];
+      default:
+        throw Error(`Incompatible video codec for RKMPP: ${this.config.targetVideoCodec}`);
+    }
+  }
+
+  getBitrateOptions() {
+    const bitrate = this.getMaxBitrateValue();
+    if (bitrate > 0) {
+      return ['-rc_mode 3', '-quality_min 0', '-quality_max 100', `-b:v ${bitrate}${this.getBitrateUnit()}`];
+    } else {
+      // convert CQP from 51-10 to 0-100, values below 10 are set to 10
+      const quality = Math.floor(125 - Math.max(this.config.crf, 10) * (125 / 51));
+      return ['-rc_mode 2', `-quality_min ${quality}`, `-quality_max ${quality}`];
+    }
+  }
+
+  getSupportedCodecs() {
+    return [VideoCodec.H264, VideoCodec.HEVC];
+  }
+
+  getVideoCodec(): string {
+    return `${this.config.targetVideoCodec}_rkmpp_encoder`;
+  }
+}

server/src/domain/repositories/media.repository.ts

@@ -51,6 +51,8 @@ export interface TranscodeOptions {
   inputOptions: string[];
   outputOptions: string[];
   twoPass: boolean;
+  ffmpegPath?: string;
+  ldLibraryPath?: string;
 }
 
 export interface BitrateDistribution {