mirror of
https://github.com/jellyfin/jellyfin.git
synced 2025-12-16 14:03:03 +03:00
Improve dynamic HDR metadata handling (#13277)
* Add support for bitstream filter to remove dynamic hdr metadata * Add support for ffprobe's only_first_vframe for HDR10+ detection * Add BitStreamFilterOptionType for metadata removal check * Map HDR10+ metadata to VideoRangeType.cs Current implementation uses a hack that abuses the EL flag to avoid database schema changes. Should add proper field once EFCore migration is merged. * Add more Dolby Vision Range types Out of spec ones are problematic and should be marked as a dedicated invalid type and handled by the server to not crash the player. Profile 7 videos should not be treated as normal HDR10 videos at all and should remove the metadata before serving. * Remove dynamic hdr metadata when necessary * Allow direct playback of HDR10+ videos on HDR10 clients * Only use dovi codec tag when dovi metadata is not removed * Handle DV Profile 7 Videos better * Fix HDR10+ with new bitmask * Indicate the presence of HDR10+ in HLS SUPPLEMENTAL-CODECS * Fix Dovi 8.4 not labeled as HLG in HLS * Fallback to dovi_rpu bsf for av1 when possible * Fix dovi_rpu cli for av1 * Use correct EFCore db column for HDR10+ * Undo outdated migration * Add proper hdr10+ migration * Remove outdated migration * Rebase to new db code * Add migrations for Hdr10PlusPresentFlag * Directly use bsf enum * Add xmldocs for SupportsBitStreamFilterWithOption * Make `VideoRangeType.Unknown` explicitly default on api models. * Unset default for non-api model class * Use tuples for bsf dictionary for now
This commit is contained in:
gnattu
@@ -162,6 +162,13 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||
_configurationManager = configurationManager;
|
||||
}
|
||||
|
||||
private enum DynamicHdrMetadataRemovalPlan
|
||||
{
|
||||
None,
|
||||
RemoveDovi,
|
||||
RemoveHdr10Plus,
|
||||
}
|
||||
|
||||
[GeneratedRegex(@"\s+")]
|
||||
private static partial Regex WhiteSpaceRegex();
|
||||
|
||||
@@ -342,11 +349,8 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||
return isSwDecoder || isNvdecDecoder || isVaapiDecoder || isD3d11vaDecoder || isVideoToolBoxDecoder;
|
||||
}
|
||||
|
||||
return state.VideoStream.VideoRange == VideoRange.HDR
|
||||
&& (state.VideoStream.VideoRangeType == VideoRangeType.HDR10
|
||||
|| state.VideoStream.VideoRangeType == VideoRangeType.HLG
|
||||
|| state.VideoStream.VideoRangeType == VideoRangeType.DOVIWithHDR10
|
||||
|| state.VideoStream.VideoRangeType == VideoRangeType.DOVIWithHLG);
|
||||
// GPU tonemapping supports all HDR RangeTypes
|
||||
return state.VideoStream.VideoRange == VideoRange.HDR;
|
||||
}
|
||||
|
||||
private bool IsVulkanHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
|
||||
@@ -381,8 +385,7 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||
}
|
||||
|
||||
return state.VideoStream.VideoRange == VideoRange.HDR
|
||||
&& (state.VideoStream.VideoRangeType == VideoRangeType.HDR10
|
||||
|| state.VideoStream.VideoRangeType == VideoRangeType.DOVIWithHDR10);
|
||||
&& IsDoviWithHdr10Bl(state.VideoStream);
|
||||
}
|
||||
|
||||
private bool IsVideoToolboxTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
|
||||
@@ -397,7 +400,8 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||
// Certain DV profile 5 video works in Safari with direct playing, but the VideoToolBox does not produce correct mapping results with transcoding.
|
||||
// All other HDR formats working.
|
||||
return state.VideoStream.VideoRange == VideoRange.HDR
|
||||
&& state.VideoStream.VideoRangeType is VideoRangeType.HDR10 or VideoRangeType.HLG or VideoRangeType.HDR10Plus or VideoRangeType.DOVIWithHDR10 or VideoRangeType.DOVIWithHLG;
|
||||
&& (IsDoviWithHdr10Bl(state.VideoStream)
|
||||
|| state.VideoStream.VideoRangeType is VideoRangeType.HLG);
|
||||
}
|
||||
|
||||
private bool IsVideoStreamHevcRext(EncodingJobInfo state)
|
||||
@@ -1301,6 +1305,13 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||
|| codec.Contains("hevc", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public static bool IsAv1(MediaStream stream)
|
||||
{
|
||||
var codec = stream.Codec ?? string.Empty;
|
||||
|
||||
return codec.Contains("av1", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public static bool IsAAC(MediaStream stream)
|
||||
{
|
||||
var codec = stream.Codec ?? string.Empty;
|
||||
@@ -1308,8 +1319,125 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||
return codec.Contains("aac", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public static string GetBitStreamArgs(MediaStream stream)
|
||||
public static bool IsDoviWithHdr10Bl(MediaStream stream)
|
||||
{
|
||||
var rangeType = stream?.VideoRangeType;
|
||||
|
||||
return rangeType is VideoRangeType.DOVIWithHDR10
|
||||
or VideoRangeType.DOVIWithEL
|
||||
or VideoRangeType.DOVIWithHDR10Plus
|
||||
or VideoRangeType.DOVIWithELHDR10Plus
|
||||
or VideoRangeType.DOVIInvalid;
|
||||
}
|
||||
|
||||
public static bool IsDovi(MediaStream stream)
|
||||
{
|
||||
var rangeType = stream?.VideoRangeType;
|
||||
|
||||
return IsDoviWithHdr10Bl(stream)
|
||||
|| (rangeType is VideoRangeType.DOVI
|
||||
or VideoRangeType.DOVIWithHLG
|
||||
or VideoRangeType.DOVIWithSDR);
|
||||
}
|
||||
|
||||
public static bool IsHdr10Plus(MediaStream stream)
|
||||
{
|
||||
var rangeType = stream?.VideoRangeType;
|
||||
|
||||
return rangeType is VideoRangeType.HDR10Plus
|
||||
or VideoRangeType.DOVIWithHDR10Plus
|
||||
or VideoRangeType.DOVIWithELHDR10Plus;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if dynamic HDR metadata should be removed during stream copy.
|
||||
/// Please note this check assumes the range check has already been done
|
||||
/// and trivial fallbacks like HDR10+ to HDR10, DOVIWithHDR10 to HDR10 is already checked.
|
||||
/// </summary>
|
||||
private static DynamicHdrMetadataRemovalPlan ShouldRemoveDynamicHdrMetadata(EncodingJobInfo state)
|
||||
{
|
||||
var videoStream = state.VideoStream;
|
||||
if (videoStream.VideoRange is not VideoRange.HDR)
|
||||
{
|
||||
return DynamicHdrMetadataRemovalPlan.None;
|
||||
}
|
||||
|
||||
var requestedRangeTypes = state.GetRequestedRangeTypes(state.VideoStream.Codec);
|
||||
if (requestedRangeTypes.Length == 0)
|
||||
{
|
||||
return DynamicHdrMetadataRemovalPlan.None;
|
||||
}
|
||||
|
||||
var requestHasHDR10 = requestedRangeTypes.Contains(VideoRangeType.HDR10.ToString(), StringComparison.OrdinalIgnoreCase);
|
||||
var requestHasDOVI = requestedRangeTypes.Contains(VideoRangeType.DOVI.ToString(), StringComparison.OrdinalIgnoreCase);
|
||||
var requestHasDOVIwithEL = requestedRangeTypes.Contains(VideoRangeType.DOVIWithEL.ToString(), StringComparison.OrdinalIgnoreCase);
|
||||
var requestHasDOVIwithELHDR10plus = requestedRangeTypes.Contains(VideoRangeType.DOVIWithELHDR10Plus.ToString(), StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
var shouldRemoveHdr10Plus = false;
|
||||
// Case 1: Client supports HDR10, does not support DOVI with EL but EL presets
|
||||
var shouldRemoveDovi = (!requestHasDOVIwithEL && requestHasHDR10) && videoStream.VideoRangeType == VideoRangeType.DOVIWithEL;
|
||||
|
||||
// Case 2: Client supports DOVI, does not support broken DOVI config
|
||||
// Client does not report DOVI support should be allowed to copy bad data for remuxing as HDR10 players would not crash
|
||||
shouldRemoveDovi = shouldRemoveDovi || (requestHasDOVI && videoStream.VideoRangeType == VideoRangeType.DOVIInvalid);
|
||||
|
||||
// Special case: we have a video with both EL and HDR10+
|
||||
// If the client supports EL but not in the case of coexistence with HDR10+, remove HDR10+ for compatibility reasons.
|
||||
// Otherwise, remove DOVI if the client is not a DOVI player
|
||||
if (videoStream.VideoRangeType == VideoRangeType.DOVIWithELHDR10Plus)
|
||||
{
|
||||
shouldRemoveHdr10Plus = requestHasDOVIwithEL && !requestHasDOVIwithELHDR10plus;
|
||||
shouldRemoveDovi = shouldRemoveDovi || !shouldRemoveHdr10Plus;
|
||||
}
|
||||
|
||||
if (shouldRemoveDovi)
|
||||
{
|
||||
return DynamicHdrMetadataRemovalPlan.RemoveDovi;
|
||||
}
|
||||
|
||||
// If the client is a Dolby Vision Player, remove the HDR10+ metadata to avoid playback issues
|
||||
shouldRemoveHdr10Plus = shouldRemoveHdr10Plus || (requestHasDOVI && videoStream.VideoRangeType == VideoRangeType.DOVIWithHDR10Plus);
|
||||
return shouldRemoveHdr10Plus ? DynamicHdrMetadataRemovalPlan.RemoveHdr10Plus : DynamicHdrMetadataRemovalPlan.None;
|
||||
}
|
||||
|
||||
private bool CanEncoderRemoveDynamicHdrMetadata(DynamicHdrMetadataRemovalPlan plan, MediaStream videoStream)
|
||||
{
|
||||
return plan switch
|
||||
{
|
||||
DynamicHdrMetadataRemovalPlan.RemoveDovi => _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.DoviRpuStrip)
|
||||
|| (IsH265(videoStream) && _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.HevcMetadataRemoveDovi))
|
||||
|| (IsAv1(videoStream) && _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.Av1MetadataRemoveDovi)),
|
||||
DynamicHdrMetadataRemovalPlan.RemoveHdr10Plus => (IsH265(videoStream) && _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.HevcMetadataRemoveHdr10Plus))
|
||||
|| (IsAv1(videoStream) && _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.Av1MetadataRemoveHdr10Plus)),
|
||||
_ => true,
|
||||
};
|
||||
}
|
||||
|
||||
public bool IsDoviRemoved(EncodingJobInfo state)
|
||||
{
|
||||
return state?.VideoStream is not null && ShouldRemoveDynamicHdrMetadata(state) == DynamicHdrMetadataRemovalPlan.RemoveDovi
|
||||
&& CanEncoderRemoveDynamicHdrMetadata(DynamicHdrMetadataRemovalPlan.RemoveDovi, state.VideoStream);
|
||||
}
|
||||
|
||||
public bool IsHdr10PlusRemoved(EncodingJobInfo state)
|
||||
{
|
||||
return state?.VideoStream is not null && ShouldRemoveDynamicHdrMetadata(state) == DynamicHdrMetadataRemovalPlan.RemoveHdr10Plus
|
||||
&& CanEncoderRemoveDynamicHdrMetadata(DynamicHdrMetadataRemovalPlan.RemoveHdr10Plus, state.VideoStream);
|
||||
}
|
||||
|
||||
public string GetBitStreamArgs(EncodingJobInfo state, MediaStreamType streamType)
|
||||
{
|
||||
if (state is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var stream = streamType switch
|
||||
{
|
||||
MediaStreamType.Audio => state.AudioStream,
|
||||
MediaStreamType.Video => state.VideoStream,
|
||||
_ => state.VideoStream
|
||||
};
|
||||
// TODO This is auto inserted into the mpegts mux so it might not be needed.
|
||||
// https://www.ffmpeg.org/ffmpeg-bitstream-filters.html#h264_005fmp4toannexb
|
||||
if (IsH264(stream))
|
||||
@@ -1317,21 +1445,57 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||
return "-bsf:v h264_mp4toannexb";
|
||||
}
|
||||
|
||||
if (IsH265(stream))
|
||||
{
|
||||
return "-bsf:v hevc_mp4toannexb";
|
||||
}
|
||||
|
||||
if (IsAAC(stream))
|
||||
{
|
||||
// Convert adts header(mpegts) to asc header(mp4).
|
||||
return "-bsf:a aac_adtstoasc";
|
||||
}
|
||||
|
||||
if (IsH265(stream))
|
||||
{
|
||||
var filter = "-bsf:v hevc_mp4toannexb";
|
||||
|
||||
// The following checks are not complete because the copy would be rejected
|
||||
// if the encoder cannot remove required metadata.
|
||||
// And if bsf is used, we must already be using copy codec.
|
||||
switch (ShouldRemoveDynamicHdrMetadata(state))
|
||||
{
|
||||
default:
|
||||
case DynamicHdrMetadataRemovalPlan.None:
|
||||
break;
|
||||
case DynamicHdrMetadataRemovalPlan.RemoveDovi:
|
||||
filter += _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.HevcMetadataRemoveDovi)
|
||||
? ",hevc_metadata=remove_dovi=1"
|
||||
: ",dovi_rpu=strip=1";
|
||||
break;
|
||||
case DynamicHdrMetadataRemovalPlan.RemoveHdr10Plus:
|
||||
filter += ",hevc_metadata=remove_hdr10plus=1";
|
||||
break;
|
||||
}
|
||||
|
||||
return filter;
|
||||
}
|
||||
|
||||
if (IsAv1(stream))
|
||||
{
|
||||
switch (ShouldRemoveDynamicHdrMetadata(state))
|
||||
{
|
||||
default:
|
||||
case DynamicHdrMetadataRemovalPlan.None:
|
||||
return null;
|
||||
case DynamicHdrMetadataRemovalPlan.RemoveDovi:
|
||||
return _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.Av1MetadataRemoveDovi)
|
||||
? "-bsf:v av1_metadata=remove_dovi=1"
|
||||
: "-bsf:v dovi_rpu=strip=1";
|
||||
case DynamicHdrMetadataRemovalPlan.RemoveHdr10Plus:
|
||||
return "-bsf:v av1_metadata=remove_hdr10plus=1";
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public static string GetAudioBitStreamArguments(EncodingJobInfo state, string segmentContainer, string mediaSourceContainer)
|
||||
public string GetAudioBitStreamArguments(EncodingJobInfo state, string segmentContainer, string mediaSourceContainer)
|
||||
{
|
||||
var bitStreamArgs = string.Empty;
|
||||
var segmentFormat = GetSegmentFileExtension(segmentContainer).TrimStart('.');
|
||||
@@ -1342,7 +1506,7 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||
|| string.Equals(mediaSourceContainer, "aac", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(mediaSourceContainer, "hls", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
bitStreamArgs = GetBitStreamArgs(state.AudioStream);
|
||||
bitStreamArgs = GetBitStreamArgs(state, MediaStreamType.Audio);
|
||||
bitStreamArgs = string.IsNullOrEmpty(bitStreamArgs) ? string.Empty : " " + bitStreamArgs;
|
||||
}
|
||||
|
||||
@@ -2169,7 +2333,6 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||
}
|
||||
|
||||
// DOVIWithHDR10 should be compatible with HDR10 supporting players. Same goes with HLG and of course SDR. So allow copy of those formats
|
||||
|
||||
var requestHasHDR10 = requestedRangeTypes.Contains(VideoRangeType.HDR10.ToString(), StringComparison.OrdinalIgnoreCase);
|
||||
var requestHasHLG = requestedRangeTypes.Contains(VideoRangeType.HLG.ToString(), StringComparison.OrdinalIgnoreCase);
|
||||
var requestHasSDR = requestedRangeTypes.Contains(VideoRangeType.SDR.ToString(), StringComparison.OrdinalIgnoreCase);
|
||||
@@ -2177,9 +2340,17 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||
if (!requestedRangeTypes.Contains(videoStream.VideoRangeType.ToString(), StringComparison.OrdinalIgnoreCase)
|
||||
&& !((requestHasHDR10 && videoStream.VideoRangeType == VideoRangeType.DOVIWithHDR10)
|
||||
|| (requestHasHLG && videoStream.VideoRangeType == VideoRangeType.DOVIWithHLG)
|
||||
|| (requestHasSDR && videoStream.VideoRangeType == VideoRangeType.DOVIWithSDR)))
|
||||
|| (requestHasSDR && videoStream.VideoRangeType == VideoRangeType.DOVIWithSDR)
|
||||
|| (requestHasHDR10 && videoStream.VideoRangeType == VideoRangeType.HDR10Plus)))
|
||||
{
|
||||
return false;
|
||||
// Check complicated cases where we need to remove dynamic metadata
|
||||
// Conservatively refuse to copy if the encoder can't remove dynamic metadata,
|
||||
// but a removal is required for compatability reasons.
|
||||
var dynamicHdrMetadataRemovalPlan = ShouldRemoveDynamicHdrMetadata(state);
|
||||
if (!CanEncoderRemoveDynamicHdrMetadata(dynamicHdrMetadataRemovalPlan, videoStream))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7244,7 +7415,7 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||
&& string.Equals(state.OutputContainer, "ts", StringComparison.OrdinalIgnoreCase)
|
||||
&& !string.Equals(state.VideoStream.NalLengthSize, "0", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
string bitStreamArgs = GetBitStreamArgs(state.VideoStream);
|
||||
string bitStreamArgs = GetBitStreamArgs(state, MediaStreamType.Video);
|
||||
if (!string.IsNullOrEmpty(bitStreamArgs))
|
||||
{
|
||||
args += " " + bitStreamArgs;
|
||||
|
||||
Reference in New Issue
Block a user