starred/jellyfin
1
0
Fork 0
mirror of https://github.com/jellyfin/jellyfin.git synced 2026-01-23 23:20:51 +01:00
Code Issues 725 Projects Packages Wiki Activity Actions 2

Add People Dedup and multiple progress fixes (#14848)

Browse source
This commit is contained in:
JPVenson 2025-09-25 00:20:30 +03:00 • committed by GitHub
parent 897975fc57
commit 5a6d9180fe
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 257 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Emby.Server.Implementations/Data/CleanDatabaseScheduledTask.cs
View file

@ -50,6 +50,8 @@ public class CleanDatabaseScheduledTask : ILibraryPostScanTask
_logger.LogDebug("Cleaning {Number} items with dead parents", numItems);
IProgress<double> subProgress = new Progress<double>((val) => progress.Report(val / 2));
foreach (var itemId in itemIds)
{
cancellationToken.ThrowIfCancellationRequested();
@ -95,9 +97,10 @@ public class CleanDatabaseScheduledTask : ILibraryPostScanTask
numComplete++;
double percent = numComplete;
percent /= numItems;
progress.Report(percent * 100);
subProgress.Report(percent * 100);
}
subProgress = new Progress<double>((val) => progress.Report((val / 2) + 50));
var context = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
await using (context.ConfigureAwait(false))
{
@ -105,7 +108,9 @@ public class CleanDatabaseScheduledTask : ILibraryPostScanTask
await using (transaction.ConfigureAwait(false))
{
await context.ItemValues.Where(e => e.BaseItemsMap!.Count == 0).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
subProgress.Report(50);
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
subProgress.Report(100);
}
}

27
Emby.Server.Implementations/Dto/DtoService.cs
View file

@ -1051,30 +1051,15 @@ namespace Emby.Server.Implementations.Dto
// Include artists that are not in the database yet, e.g., just added via metadata editor
// var foundArtists = artistItems.Items.Select(i => i.Item1.Name).ToList();
dto.ArtistItems = hasArtist.Artists
// .Except(foundArtists, new DistinctNameComparer())
dto.ArtistItems = _libraryManager.GetArtists([.. hasArtist.Artists.Where(e => !string.IsNullOrWhiteSpace(e))])
.Where(e => e.Value.Length > 0)
.Select(i =>
{
// This should not be necessary but we're seeing some cases of it
if (string.IsNullOrEmpty(i))
return new NameGuidPair
{
return null;
}
var artist = _libraryManager.GetArtist(i, new DtoOptions(false)
{
EnableImages = false
});
if (artist is not null)
{
return new NameGuidPair
{
Name = artist.Name,
Id = artist.Id
};
}
return null;
Name = i.Key,
Id = i.Value.First().Id
};
}).Where(i => i is not null).ToArray();
}

166
Emby.Server.Implementations/Library/LibraryManager.cs
View file

@ -327,6 +327,45 @@ namespace Emby.Server.Implementations.Library
DeleteItem(item, options, parent, notifyParentItem);
}
public void DeleteItemsUnsafeFast(IEnumerable<BaseItem> items)
{
var pathMaps = items.Select(e => (Item: e, InternalPath: GetInternalMetadataPaths(e), DeletePaths: e.GetDeletePaths())).ToArray();
foreach (var (item, internalPaths, pathsToDelete) in pathMaps)
{
foreach (var metadataPath in internalPaths)
{
if (!Directory.Exists(metadataPath))
{
continue;
}
_logger.LogDebug(
"Deleting metadata path, Type: {Type}, Name: {Name}, Path: {Path}, Id: {Id}",
item.GetType().Name,
item.Name ?? "Unknown name",
metadataPath,
item.Id);
try
{
Directory.Delete(metadataPath, true);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error deleting {MetadataPath}", metadataPath);
}
}
foreach (var fileSystemInfo in pathsToDelete)
{
DeleteItemPath(item, false, fileSystemInfo);
}
}
_itemRepository.DeleteItem([.. pathMaps.Select(f => f.Item.Id)]);
}
public void DeleteItem(BaseItem item, DeleteOptions options, BaseItem parent, bool notifyParentItem)
{
ArgumentNullException.ThrowIfNull(item);
@ -403,59 +442,7 @@ namespace Emby.Server.Implementations.Library
foreach (var fileSystemInfo in item.GetDeletePaths())
{
if (Directory.Exists(fileSystemInfo.FullName) || File.Exists(fileSystemInfo.FullName))
{
try
{
_logger.LogInformation(
"Deleting item path, Type: {Type}, Name: {Name}, Path: {Path}, Id: {Id}",
item.GetType().Name,
item.Name ?? "Unknown name",
fileSystemInfo.FullName,
item.Id);
if (fileSystemInfo.IsDirectory)
{
Directory.Delete(fileSystemInfo.FullName, true);
}
else
{
File.Delete(fileSystemInfo.FullName);
}
}
catch (DirectoryNotFoundException)
{
_logger.LogInformation(
"Directory not found, only removing from database, Type: {Type}, Name: {Name}, Path: {Path}, Id: {Id}",
item.GetType().Name,
item.Name ?? "Unknown name",
fileSystemInfo.FullName,
item.Id);
}
catch (FileNotFoundException)
{
_logger.LogInformation(
"File not found, only removing from database, Type: {Type}, Name: {Name}, Path: {Path}, Id: {Id}",
item.GetType().Name,
item.Name ?? "Unknown name",
fileSystemInfo.FullName,
item.Id);
}
catch (IOException)
{
if (isRequiredForDelete)
{
throw;
}
}
catch (UnauthorizedAccessException)
{
if (isRequiredForDelete)
{
throw;
}
}
}
DeleteItemPath(item, isRequiredForDelete, fileSystemInfo);
isRequiredForDelete = false;
}
@ -463,17 +450,73 @@ namespace Emby.Server.Implementations.Library
item.SetParent(null);
_itemRepository.DeleteItem(item.Id);
_itemRepository.DeleteItem([item.Id, .. children.Select(f => f.Id)]);
_cache.TryRemove(item.Id, out _);
foreach (var child in children)
{
_itemRepository.DeleteItem(child.Id);
_cache.TryRemove(child.Id, out _);
}
ReportItemRemoved(item, parent);
}
private void DeleteItemPath(BaseItem item, bool isRequiredForDelete, FileSystemMetadata fileSystemInfo)
{
if (Directory.Exists(fileSystemInfo.FullName) || File.Exists(fileSystemInfo.FullName))
{
try
{
_logger.LogInformation(
"Deleting item path, Type: {Type}, Name: {Name}, Path: {Path}, Id: {Id}",
item.GetType().Name,
item.Name ?? "Unknown name",
fileSystemInfo.FullName,
item.Id);
if (fileSystemInfo.IsDirectory)
{
Directory.Delete(fileSystemInfo.FullName, true);
}
else
{
File.Delete(fileSystemInfo.FullName);
}
}
catch (DirectoryNotFoundException)
{
_logger.LogInformation(
"Directory not found, only removing from database, Type: {Type}, Name: {Name}, Path: {Path}, Id: {Id}",
item.GetType().Name,
item.Name ?? "Unknown name",
fileSystemInfo.FullName,
item.Id);
}
catch (FileNotFoundException)
{
_logger.LogInformation(
"File not found, only removing from database, Type: {Type}, Name: {Name}, Path: {Path}, Id: {Id}",
item.GetType().Name,
item.Name ?? "Unknown name",
fileSystemInfo.FullName,
item.Id);
}
catch (IOException)
{
if (isRequiredForDelete)
{
throw;
}
}
catch (UnauthorizedAccessException)
{
if (isRequiredForDelete)
{
throw;
}
}
}
}
private bool IsInternalItem(BaseItem item)
{
if (!item.IsFileProtocol)
@ -990,6 +1033,11 @@ namespace Emby.Server.Implementations.Library
return GetArtist(name, new DtoOptions(true));
}
public IReadOnlyDictionary<string, MusicArtist[]> GetArtists(IReadOnlyList<string> names)
{
return _itemRepository.FindArtists(names);
}
public MusicArtist GetArtist(string name, DtoOptions options)
{
return CreateItemByName<MusicArtist>(MusicArtist.GetPath, name, options);
@ -1115,18 +1163,24 @@ namespace Emby.Server.Implementations.Library
cancellationToken: cancellationToken).ConfigureAwait(false);
// Quickly scan CollectionFolders for changes
var toDelete = new List<Guid>();
foreach (var child in rootFolder.Children!.OfType<Folder>())
{
// If the user has somehow deleted the collection directory, remove the metadata from the database.
if (child is CollectionFolder collectionFolder && !Directory.Exists(collectionFolder.Path))
{
_itemRepository.DeleteItem(collectionFolder.Id);
toDelete.Add(collectionFolder.Id);
}
else
{
await child.RefreshMetadata(cancellationToken).ConfigureAwait(false);
}
}
if (toDelete.Count > 0)
{
_itemRepository.DeleteItem(toDelete.ToArray());
}
}
private async Task PerformLibraryValidation(IProgress<double> progress, CancellationToken cancellationToken)

22
Emby.Server.Implementations/Library/Validators/PeopleValidator.cs
View file

@ -1,5 +1,5 @@
using System;
using System.Globalization;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Data.Enums;
@ -55,6 +55,8 @@ public class PeopleValidator
var numPeople = people.Count;
IProgress<double> subProgress = new Progress<double>((val) => progress.Report(val / 2));
_logger.LogDebug("Will refresh {Amount} people", numPeople);
foreach (var person in people)
@ -92,7 +94,7 @@ public class PeopleValidator
double percent = numComplete;
percent /= numPeople;
progress.Report(100 * percent);
subProgress.Report(100 * percent);
}
var deadEntities = _libraryManager.GetItemList(new InternalItemsQuery
@ -102,17 +104,13 @@ public class PeopleValidator
IsLocked = false
});
foreach (var item in deadEntities)
{
_logger.LogInformation("Deleting dead {ItemType} {ItemId} {ItemName}", item.GetType().Name, item.Id.ToString("N", CultureInfo.InvariantCulture), item.Name);
subProgress = new Progress<double>((val) => progress.Report((val / 2) + 50));
_libraryManager.DeleteItem(
item,
new DeleteOptions
{
DeleteFileLocation = false
},
false);
var i = 0;
foreach (var item in deadEntities.Chunk(500))
{
_libraryManager.DeleteItemsUnsafeFast(item);
subProgress.Report(100f / deadEntities.Count * (i++ * 100));
}
progress.Report(100);

66
Emby.Server.Implementations/ScheduledTasks/Tasks/PeopleValidationTask.cs
View file

@ -1,10 +1,14 @@
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Database.Implementations;
using MediaBrowser.Controller.Library;
using MediaBrowser.Model.Globalization;
using MediaBrowser.Model.Tasks;
using Microsoft.EntityFrameworkCore;
namespace Emby.Server.Implementations.ScheduledTasks.Tasks;
@ -15,16 +19,19 @@ public class PeopleValidationTask : IScheduledTask, IConfigurableScheduledTask
{
private readonly ILibraryManager _libraryManager;
private readonly ILocalizationManager _localization;
private readonly IDbContextFactory<JellyfinDbContext> _dbContextFactory;
/// <summary>
/// Initializes a new instance of the <see cref="PeopleValidationTask" /> class.
/// </summary>
/// <param name="libraryManager">Instance of the <see cref="ILibraryManager"/> interface.</param>
/// <param name="localization">Instance of the <see cref="ILocalizationManager"/> interface.</param>
public PeopleValidationTask(ILibraryManager libraryManager, ILocalizationManager localization)
/// <param name="dbContextFactory">Instance of the <see cref="IDbContextFactory{TContext}"/> interface.</param>
public PeopleValidationTask(ILibraryManager libraryManager, ILocalizationManager localization, IDbContextFactory<JellyfinDbContext> dbContextFactory)
{
_libraryManager = libraryManager;
_localization = localization;
_dbContextFactory = dbContextFactory;
}
/// <inheritdoc />
@ -62,8 +69,61 @@ public class PeopleValidationTask : IScheduledTask, IConfigurableScheduledTask
}
/// <inheritdoc />
public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken)
public async Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken)
{
return _libraryManager.ValidatePeopleAsync(progress, cancellationToken);
IProgress<double> subProgress = new Progress<double>((val) => progress.Report(val / 2));
await _libraryManager.ValidatePeopleAsync(subProgress, cancellationToken).ConfigureAwait(false);
subProgress = new Progress<double>((val) => progress.Report((val / 2) + 50));
var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
await using (context.ConfigureAwait(false))
{
var dupQuery = context.Peoples
.GroupBy(e => new { e.Name, e.PersonType })
.Where(e => e.Count() > 1)
.Select(e => e.Select(f => f.Id).ToArray());
var total = dupQuery.Count();
const int PartitionSize = 100;
var iterator = 0;
int itemCounter;
var buffer = ArrayPool<Guid[]>.Shared.Rent(PartitionSize)!;
try
{
do
{
itemCounter = 0;
await foreach (var item in dupQuery
.Take(PartitionSize)
.AsAsyncEnumerable()
.WithCancellation(cancellationToken)
.ConfigureAwait(false))
{
buffer[itemCounter++] = item;
}
for (int i = 0; i < itemCounter; i++)
{
var item = buffer[i];
var reference = item[0];
var dups = item[1..];
await context.PeopleBaseItemMap.WhereOneOrMany(dups, e => e.PeopleId)
.ExecuteUpdateAsync(e => e.SetProperty(f => f.PeopleId, reference), cancellationToken)
.ConfigureAwait(false);
await context.Peoples.Where(e => dups.Contains(e.Id)).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
subProgress.Report(100f / total * ((iterator * PartitionSize) + i));
}
iterator++;
} while (itemCounter == PartitionSize && !cancellationToken.IsCancellationRequested);
}
finally
{
ArrayPool<Guid[]>.Shared.Return(buffer);
}
subProgress.Report(100);
}
}
}