starred/jellyfin-jellyfin-1
1
0
Fork 0
mirror of https://github.com/jellyfin/jellyfin.git synced 2025-12-23 09:14:45 +03:00
Code Issues Packages Projects Releases Wiki Activity

Clean up backup service (#15170)

Browse Source
This commit is contained in:
Cody Robibero
2025-10-24 17:57:34 -06:00
committed by GitHub
parent 7a1c1cd342
commit ac3fa3c376
1 changed files with 131 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

236
Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs
View File

@@ -199,7 +199,7 @@ public class BackupService : IBackupService
var zipEntry = zipArchive.GetEntry(NormalizePathSeparator(Path.Combine("Database", $"{entityType.Type.Name}.json"))); var zipEntry = zipArchive.GetEntry(NormalizePathSeparator(Path.Combine("Database", $"{entityType.Type.Name}.json")));
if (zipEntry is null) if (zipEntry is null)
{ {
_logger.LogInformation("No backup of expected table {Table} is present in backup. Continue anyway.", entityType.Type.Name); _logger.LogInformation("No backup of expected table {Table} is present in backup, continuing anyway", entityType.Type.Name);
continue; continue;
} }
@@ -223,7 +223,7 @@ public class BackupService : IBackupService
} }
catch (Exception ex) catch (Exception ex)
{ {
_logger.LogError(ex, "Could not store entity {Entity} continue anyway.", item); _logger.LogError(ex, "Could not store entity {Entity}, continuing anyway", item);
} }
} }
@@ -233,11 +233,11 @@ public class BackupService : IBackupService
_logger.LogInformation("Try restore Database"); _logger.LogInformation("Try restore Database");
await dbContext.SaveChangesAsync().ConfigureAwait(false); await dbContext.SaveChangesAsync().ConfigureAwait(false);
_logger.LogInformation("Restored database."); _logger.LogInformation("Restored database");
} }
} }
_logger.LogInformation("Restored Jellyfin system from {Date}.", manifest.DateCreated); _logger.LogInformation("Restored Jellyfin system from {Date}", manifest.DateCreated);
} }
} }
@@ -263,6 +263,8 @@ public class BackupService : IBackupService
Options = Map(backupOptions) Options = Map(backupOptions)
}; };
_logger.LogInformation("Running database optimization before backup");
await _jellyfinDatabaseProvider.RunScheduledOptimisation(CancellationToken.None).ConfigureAwait(false); await _jellyfinDatabaseProvider.RunScheduledOptimisation(CancellationToken.None).ConfigureAwait(false);
var backupFolder = Path.Combine(_applicationPaths.BackupPath); var backupFolder = Path.Combine(_applicationPaths.BackupPath);
@@ -281,130 +283,154 @@ public class BackupService : IBackupService
} }
var backupPath = Path.Combine(backupFolder, $"jellyfin-backup-{manifest.DateCreated.ToLocalTime():yyyyMMddHHmmss}.zip"); var backupPath = Path.Combine(backupFolder, $"jellyfin-backup-{manifest.DateCreated.ToLocalTime():yyyyMMddHHmmss}.zip");
_logger.LogInformation("Attempt to create a new backup at {BackupPath}", backupPath);
var fileStream = File.OpenWrite(backupPath); try
await using (fileStream.ConfigureAwait(false))
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, false))
{ {
_logger.LogInformation("Start backup process."); _logger.LogInformation("Attempting to create a new backup at {BackupPath}", backupPath);
var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); var fileStream = File.OpenWrite(backupPath);
await using (dbContext.ConfigureAwait(false)) await using (fileStream.ConfigureAwait(false))
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, false))
{ {
dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; _logger.LogInformation("Starting backup process");
static IAsyncEnumerable<object> GetValues(IQueryable dbSet) var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false);
await using (dbContext.ConfigureAwait(false))
{ {
var method = dbSet.GetType().GetMethod(nameof(DbSet<object>.AsAsyncEnumerable))!; dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
var enumerable = method.Invoke(dbSet, null)!;
return (IAsyncEnumerable<object>)enumerable;
}
// include the migration history as well static IAsyncEnumerable<object> GetValues(IQueryable dbSet)
var historyRepository = dbContext.GetService<IHistoryRepository>();
var migrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false);
ICollection<(Type Type, string SourceName, Func<IAsyncEnumerable<object>> ValueFactory)> entityTypes = [
.. typeof(JellyfinDbContext)
.GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance)
.Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable)))
.Select(e => (Type: e.PropertyType, dbContext.Model.FindEntityType(e.PropertyType.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!, ValueFactory: new Func<IAsyncEnumerable<object>>(() => GetValues((IQueryable)e.GetValue(dbContext)!)))),
(Type: typeof(HistoryRow), SourceName: nameof(HistoryRow), ValueFactory: () => migrations.ToAsyncEnumerable())
];
manifest.DatabaseTables = entityTypes.Select(e => e.Type.Name).ToArray();
var transaction = await dbContext.Database.BeginTransactionAsync().ConfigureAwait(false);
await using (transaction.ConfigureAwait(false))
{
_logger.LogInformation("Begin Database backup");
foreach (var entityType in entityTypes)
{ {
_logger.LogInformation("Begin backup of entity {Table}", entityType.SourceName); var method = dbSet.GetType().GetMethod(nameof(DbSet<object>.AsAsyncEnumerable))!;
var zipEntry = zipArchive.CreateEntry(NormalizePathSeparator(Path.Combine("Database", $"{entityType.SourceName}.json"))); var enumerable = method.Invoke(dbSet, null)!;
var entities = 0; return (IAsyncEnumerable<object>)enumerable;
var zipEntryStream = zipEntry.Open(); }
await using (zipEntryStream.ConfigureAwait(false))
// include the migration history as well
var historyRepository = dbContext.GetService<IHistoryRepository>();
var migrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false);
ICollection<(Type Type, string SourceName, Func<IAsyncEnumerable<object>> ValueFactory)> entityTypes =
[
.. typeof(JellyfinDbContext)
.GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance)
.Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable)))
.Select(e => (Type: e.PropertyType, dbContext.Model.FindEntityType(e.PropertyType.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!, ValueFactory: new Func<IAsyncEnumerable<object>>(() => GetValues((IQueryable)e.GetValue(dbContext)!)))),
(Type: typeof(HistoryRow), SourceName: nameof(HistoryRow), ValueFactory: () => migrations.ToAsyncEnumerable())
];
manifest.DatabaseTables = entityTypes.Select(e => e.Type.Name).ToArray();
var transaction = await dbContext.Database.BeginTransactionAsync().ConfigureAwait(false);
await using (transaction.ConfigureAwait(false))
{
_logger.LogInformation("Begin Database backup");
foreach (var entityType in entityTypes)
{ {
var jsonSerializer = new Utf8JsonWriter(zipEntryStream); _logger.LogInformation("Begin backup of entity {Table}", entityType.SourceName);
await using (jsonSerializer.ConfigureAwait(false)) var zipEntry = zipArchive.CreateEntry(NormalizePathSeparator(Path.Combine("Database", $"{entityType.SourceName}.json")));
var entities = 0;
var zipEntryStream = zipEntry.Open();
await using (zipEntryStream.ConfigureAwait(false))
{ {
jsonSerializer.WriteStartArray(); var jsonSerializer = new Utf8JsonWriter(zipEntryStream);
await using (jsonSerializer.ConfigureAwait(false))
var set = entityType.ValueFactory().ConfigureAwait(false);
await foreach (var item in set.ConfigureAwait(false))
{ {
entities++; jsonSerializer.WriteStartArray();
try
var set = entityType.ValueFactory().ConfigureAwait(false);
await foreach (var item in set.ConfigureAwait(false))
{ {
JsonSerializer.SerializeToDocument(item, _serializerSettings).WriteTo(jsonSerializer); entities++;
} try
catch (Exception ex) {
{ using var document = JsonSerializer.SerializeToDocument(item, _serializerSettings);
_logger.LogError(ex, "Could not load entity {Entity}", item); document.WriteTo(jsonSerializer);
throw; }
catch (Exception ex)
{
_logger.LogError(ex, "Could not load entity {Entity}", item);
throw;
}
} }
jsonSerializer.WriteEndArray();
} }
jsonSerializer.WriteEndArray();
} }
}
_logger.LogInformation("backup of entity {Table} with {Number} created", entityType.Type.Name, entities); _logger.LogInformation("Backup of entity {Table} with {Number} created", entityType.SourceName, entities);
}
} }
} }
}
_logger.LogInformation("Backup of folder {Table}", _applicationPaths.ConfigurationDirectoryPath); _logger.LogInformation("Backup of folder {Table}", _applicationPaths.ConfigurationDirectoryPath);
foreach (var item in Directory.EnumerateFiles(_applicationPaths.ConfigurationDirectoryPath, "*.xml", SearchOption.TopDirectoryOnly) foreach (var item in Directory.EnumerateFiles(_applicationPaths.ConfigurationDirectoryPath, "*.xml", SearchOption.TopDirectoryOnly)
.Union(Directory.EnumerateFiles(_applicationPaths.ConfigurationDirectoryPath, "*.json", SearchOption.TopDirectoryOnly))) .Union(Directory.EnumerateFiles(_applicationPaths.ConfigurationDirectoryPath, "*.json", SearchOption.TopDirectoryOnly)))
{
zipArchive.CreateEntryFromFile(item, NormalizePathSeparator(Path.Combine("Config", Path.GetFileName(item))));
}
void CopyDirectory(string source, string target, string filter = "*")
{
if (!Directory.Exists(source))
{ {
return; zipArchive.CreateEntryFromFile(item, NormalizePathSeparator(Path.Combine("Config", Path.GetFileName(item))));
} }
_logger.LogInformation("Backup of folder {Table}", source); void CopyDirectory(string source, string target, string filter = "*")
foreach (var item in Directory.EnumerateFiles(source, filter, SearchOption.AllDirectories))
{ {
zipArchive.CreateEntryFromFile(item, NormalizePathSeparator(Path.Combine(target, Path.GetRelativePath(source, item)))); if (!Directory.Exists(source))
{
return;
}
_logger.LogInformation("Backup of folder {Table}", source);
foreach (var item in Directory.EnumerateFiles(source, filter, SearchOption.AllDirectories))
{
zipArchive.CreateEntryFromFile(item, NormalizePathSeparator(Path.Combine(target, Path.GetRelativePath(source, item))));
}
}
CopyDirectory(Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "users"), Path.Combine("Config", "users"));
CopyDirectory(Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "ScheduledTasks"), Path.Combine("Config", "ScheduledTasks"));
CopyDirectory(Path.Combine(_applicationPaths.RootFolderPath), "Root");
CopyDirectory(Path.Combine(_applicationPaths.DataPath, "collections"), Path.Combine("Data", "collections"));
CopyDirectory(Path.Combine(_applicationPaths.DataPath, "playlists"), Path.Combine("Data", "playlists"));
CopyDirectory(Path.Combine(_applicationPaths.DataPath, "ScheduledTasks"), Path.Combine("Data", "ScheduledTasks"));
if (backupOptions.Subtitles)
{
CopyDirectory(Path.Combine(_applicationPaths.DataPath, "subtitles"), Path.Combine("Data", "subtitles"));
}
if (backupOptions.Trickplay)
{
CopyDirectory(Path.Combine(_applicationPaths.DataPath, "trickplay"), Path.Combine("Data", "trickplay"));
}
if (backupOptions.Metadata)
{
CopyDirectory(Path.Combine(_applicationPaths.InternalMetadataPath), Path.Combine("Data", "metadata"));
}
var manifestStream = zipArchive.CreateEntry(ManifestEntryName).Open();
await using (manifestStream.ConfigureAwait(false))
{
await JsonSerializer.SerializeAsync(manifestStream, manifest).ConfigureAwait(false);
} }
} }
CopyDirectory(Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "users"), Path.Combine("Config", "users")); _logger.LogInformation("Backup created");
CopyDirectory(Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "ScheduledTasks"), Path.Combine("Config", "ScheduledTasks")); return Map(manifest, backupPath);
CopyDirectory(Path.Combine(_applicationPaths.RootFolderPath), "Root");
CopyDirectory(Path.Combine(_applicationPaths.DataPath, "collections"), Path.Combine("Data", "collections"));
CopyDirectory(Path.Combine(_applicationPaths.DataPath, "playlists"), Path.Combine("Data", "playlists"));
CopyDirectory(Path.Combine(_applicationPaths.DataPath, "ScheduledTasks"), Path.Combine("Data", "ScheduledTasks"));
if (backupOptions.Subtitles)
{
CopyDirectory(Path.Combine(_applicationPaths.DataPath, "subtitles"), Path.Combine("Data", "subtitles"));
}
if (backupOptions.Trickplay)
{
CopyDirectory(Path.Combine(_applicationPaths.DataPath, "trickplay"), Path.Combine("Data", "trickplay"));
}
if (backupOptions.Metadata)
{
CopyDirectory(Path.Combine(_applicationPaths.InternalMetadataPath), Path.Combine("Data", "metadata"));
}
var manifestStream = zipArchive.CreateEntry(ManifestEntryName).Open();
await using (manifestStream.ConfigureAwait(false))
{
await JsonSerializer.SerializeAsync(manifestStream, manifest).ConfigureAwait(false);
}
} }
catch (Exception ex)
{
_logger.LogError(ex, "Failed to create backup, removing {BackupPath}", backupPath);
try
{
if (File.Exists(backupPath))
{
File.Delete(backupPath);
}
}
catch (Exception innerEx)
{
_logger.LogWarning(innerEx, "Unable to remove failed backup");
}
_logger.LogInformation("Backup created"); throw;
return Map(manifest, backupPath); }
} }
/// <inheritdoc/> /// <inheritdoc/>
@@ -422,7 +448,7 @@ public class BackupService : IBackupService
} }
catch (Exception ex) catch (Exception ex)
{ {
_logger.LogError(ex, "Tried to load archive from {Path} but failed.", archivePath); _logger.LogWarning(ex, "Tried to load manifest from archive {Path} but failed", archivePath);
return null; return null;
} }
@@ -459,7 +485,7 @@ public class BackupService : IBackupService
} }
catch (Exception ex) catch (Exception ex)
{ {
_logger.LogError(ex, "Could not load {BackupArchive} path.", item); _logger.LogWarning(ex, "Tried to load manifest from archive {Path} but failed", item);
} }
} }