immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-21 09:15:44 +03:00
Code Issues Packages Projects Releases 18 Wiki Activity

Add AV1 transcoding support (#8491)

Browse Source 
* Add AV1 transcoding support

- AV1 encoding on CPU via SVT-AV1 (libsvtav1 in ffmpeg)
- Supports CRF and optionally capped CRF (max bitrate)
- Tested playback successfully in Chrome Win+Android, Firefox Win+Linux, Android app

* AV1: Add support for encoding threads option

* Revert previous commit; specifying params multiple times is bad

We need to specify all svtav1-params at once, so putting the thread option into getThreadOptions is not possible.

* AV1: Override VAAPI getSupportedCodecs as it does not yet support AV1 unlike nvenc, qsv, amf

* Change BaseHWConfig supported codecs to only H264/HEVC

Configs that support VP9 and/or AV1 need to override getSupportedCodecs()

* Set SVT-AV1 threads with svtav1-params, remove duplicate block in NVENCConfig

* AV1Config: Fix empty svtav1-params array being added to options

* add tests

* update api

* allow crf-based two-pass mode

* formatting

* suggest 35

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
This commit is contained in:
N00MKRAD
2024-04-11 07:26:27 +02:00
committed by GitHub
parent ad5d115abe
commit f1ca1794a1
9 changed files with 215 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
server/src/entities/system-config.entity.ts
View File

@@ -153,6 +153,7 @@ export enum VideoCodec {
H264 = 'h264',
HEVC = 'hevc',
VP9 = 'vp9',
AV1 = 'av1',
}
export enum AudioCodec {

151
server/src/services/media.service.spec.ts
View File

@@ -1268,6 +1268,157 @@ describe(MediaService.name, () => {
);
});
it('should use av1 if specified', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-c:v av1',
'-c:a copy',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-v verbose',
'-vf scale=-2:720,format=yuv420p',
'-preset 12',
'-crf 23',
],
twoPass: false,
},
);
});
it('should map `veryslow` preset to 4 for av1', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 },
{ key: SystemConfigKey.FFMPEG_PRESET, value: 'veryslow' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-c:v av1',
'-c:a copy',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-v verbose',
'-vf scale=-2:720,format=yuv420p',
'-preset 4',
'-crf 23',
],
twoPass: false,
},
);
});
it('should set max bitrate for av1 if specified', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 },
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '2M' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-c:v av1',
'-c:a copy',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-v verbose',
'-vf scale=-2:720,format=yuv420p',
'-preset 12',
'-crf 23',
'-svtav1-params mbr=2M',
],
twoPass: false,
},
);
});
it('should set threads for av1 if specified', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 },
{ key: SystemConfigKey.FFMPEG_THREADS, value: 4 },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-c:v av1',
'-c:a copy',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-v verbose',
'-vf scale=-2:720,format=yuv420p',
'-preset 12',
'-crf 23',
'-svtav1-params lp=4',
],
twoPass: false,
},
);
});
it('should set both bitrate and threads for av1 if specified', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 },
{ key: SystemConfigKey.FFMPEG_THREADS, value: 4 },
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '2M' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-c:v av1',
'-c:a copy',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-v verbose',
'-vf scale=-2:720,format=yuv420p',
'-preset 12',
'-crf 23',
'-svtav1-params lp=4:mbr=2M',
],
twoPass: false,
},
);
});
it('should skip transcoding for audioless videos with optimal policy if video codec is correct', async () => {
mediaMock.probe.mockResolvedValue(probeStub.noAudioStreams);
configMock.load.mockResolvedValue([

4
server/src/services/media.service.ts
View File

@@ -32,6 +32,7 @@ import { IStorageRepository } from 'src/interfaces/storage.interface';
import { ISystemConfigRepository } from 'src/interfaces/system-config.interface';
import { ImmichLogger } from 'src/utils/logger';
import {
AV1Config,
H264Config,
HEVCConfig,
NVENCConfig,
@@ -439,6 +440,9 @@ export class MediaService {
case VideoCodec.VP9: {
return new VP9Config(config);
}
case VideoCodec.AV1: {
return new AV1Config(config);
}
default: {
throw new UnsupportedMediaTypeException(`Codec '${config.targetVideoCodec}' is unsupported`);
}

BIN
server/src/utils/.media.ts.kate-swp Normal file
View File

Binary file not shown.

50
server/src/utils/media.ts
View File

@@ -124,7 +124,7 @@ class BaseConfig implements VideoCodecSWConfig {
return false;
}
return this.isBitrateConstrained() || this.config.targetVideoCodec === VideoCodec.VP9;
return this.isBitrateConstrained();
}
getBitrateDistribution() {
@@ -265,7 +265,7 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
}
getSupportedCodecs() {
return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
return [VideoCodec.H264, VideoCodec.HEVC];
}
validateDevices(devices: string[]) {
@@ -394,6 +394,44 @@ export class VP9Config extends BaseConfig {
getThreadOptions() {
return ['-row-mt 1', ...super.getThreadOptions()];
}
eligibleForTwoPass() {
return this.config.twoPass;
}
}
export class AV1Config extends BaseConfig {
getPresetOptions() {
const speed = this.getPresetIndex() + 4; // Use 4 as slowest, giving us an effective range of 4-12 which is far more useful than 0-8
if (speed >= 0) {
return [`-preset ${speed}`];
}
return [];
}
getBitrateOptions() {
const options = [`-crf ${this.config.crf}`];
const bitrates = this.getBitrateDistribution();
const svtparams = [];
if (this.config.threads > 0) {
svtparams.push(`lp=${this.config.threads}`);
}
if (bitrates.max > 0) {
svtparams.push(`mbr=${bitrates.max}${bitrates.unit}`);
}
if (svtparams.length > 0) {
options.push(`-svtav1-params ${svtparams.join(':')}`);
}
return options;
}
getThreadOptions() {
return []; // Already set above with svtav1-params
}
eligibleForTwoPass() {
return this.config.twoPass;
}
}
export class NVENCConfig extends BaseHWConfig {
@@ -527,6 +565,10 @@ export class QSVConfig extends BaseHWConfig {
return options;
}
getSupportedCodecs() {
return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
}
// recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
getBFrames() {
if (this.config.bframes < 0) {
@@ -605,6 +647,10 @@ export class VAAPIConfig extends BaseHWConfig {
return options;
}
getSupportedCodecs() {
return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
}
useCQP() {
return this.config.cqMode !== CQMode.ICQ || this.config.targetVideoCodec === VideoCodec.VP9;
}