tedkulp/Readarr
1
0
Fork 0
mirror of https://github.com/Readarr/Readarr.git synced 2026-04-19 21:44:30 -04:00
Code Issues Projects Releases Wiki Activity

Refactor ExtraFile/MetadataFile Services (#104)

Browse Source 
* Preliminary Work for Extras for Music

* DB Migration for ExtraFiles, Other Cleanup

* More Extras Work, Add Album Metadata Type

* Update Housekeeps for Music Extras

* Fix HouseKeeper and add new Tests

* Final round of Cleanup
This commit is contained in:
Qstick
2017-10-16 21:40:31 -04:00
committed by GitHub
parent 4016d359ac
commit 38cbb2114f
43 changed files with 985 additions and 841 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/NzbDrone.Core/Extras/Metadata/MetadataService.cs
+150 -95
View File
@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@@ -11,7 +11,7 @@ using NzbDrone.Core.Configuration;
using NzbDrone.Core.Extras.Files;
using NzbDrone.Core.Extras.Metadata.Files;
using NzbDrone.Core.MediaFiles;
using NzbDrone.Core.Tv;
using NzbDrone.Core.Music;
namespace NzbDrone.Core.Extras.Metadata
{
@@ -24,6 +24,7 @@ namespace NzbDrone.Core.Extras.Metadata
private readonly IHttpClient _httpClient;
private readonly IMediaFileAttributeService _mediaFileAttributeService;
private readonly IMetadataFileService _metadataFileService;
private readonly IAlbumService _albumService;
private readonly Logger _logger;
public MetadataService(IConfigService configService,
@@ -34,6 +35,7 @@ namespace NzbDrone.Core.Extras.Metadata
IHttpClient httpClient,
IMediaFileAttributeService mediaFileAttributeService,
IMetadataFileService metadataFileService,
IAlbumService albumService,
Logger logger)
: base(configService, diskProvider, diskTransferService, logger)
{
@@ -44,19 +46,20 @@ namespace NzbDrone.Core.Extras.Metadata
_httpClient = httpClient;
_mediaFileAttributeService = mediaFileAttributeService;
_metadataFileService = metadataFileService;
_albumService = albumService;
_logger = logger;
}
public override int Order => 0;
public override IEnumerable<ExtraFile> CreateAfterSeriesScan(Series series, List<EpisodeFile> episodeFiles)
public override IEnumerable<ExtraFile> CreateAfterArtistScan(Artist artist, List<Album> albums, List<TrackFile> trackFiles)
{
var metadataFiles = _metadataFileService.GetFilesBySeries(series.Id);
_cleanMetadataService.Clean(series);
var metadataFiles = _metadataFileService.GetFilesByArtist(artist.Id);
_cleanMetadataService.Clean(artist);
if (!_diskProvider.FolderExists(series.Path))
if (!_diskProvider.FolderExists(artist.Path))
{
_logger.Info("Series folder does not exist, skipping metadata creation");
_logger.Info("Artist folder does not exist, skipping metadata creation");
return Enumerable.Empty<MetadataFile>();
}
@@ -66,14 +69,20 @@ namespace NzbDrone.Core.Extras.Metadata
{
var consumerFiles = GetMetadataFilesForConsumer(consumer, metadataFiles);
files.AddIfNotNull(ProcessSeriesMetadata(consumer, series, consumerFiles));
files.AddRange(ProcessSeriesImages(consumer, series, consumerFiles));
files.AddRange(ProcessSeasonImages(consumer, series, consumerFiles));
files.AddIfNotNull(ProcessArtistMetadata(consumer, artist, consumerFiles));
files.AddRange(ProcessArtistImages(consumer, artist, consumerFiles));
files.AddRange(ProcessAlbumImages(consumer, artist, consumerFiles));
foreach (var episodeFile in episodeFiles)
foreach (var album in albums)
{
files.AddIfNotNull(ProcessEpisodeMetadata(consumer, series, episodeFile, consumerFiles));
files.AddRange(ProcessEpisodeImages(consumer, series, episodeFile, consumerFiles));
album.Artist = artist;
files.AddIfNotNull(ProcessAlbumMetadata(consumer, album, consumerFiles));
}
foreach (var trackFile in trackFiles)
{
files.AddIfNotNull(ProcessEpisodeMetadata(consumer, artist, trackFile, consumerFiles));
files.AddRange(ProcessEpisodeImages(consumer, artist, trackFile, consumerFiles));
}
}
@@ -82,15 +91,15 @@ namespace NzbDrone.Core.Extras.Metadata
return files;
}
public override IEnumerable<ExtraFile> CreateAfterEpisodeImport(Series series, EpisodeFile episodeFile)
public override IEnumerable<ExtraFile> CreateAfterTrackImport(Artist artist, TrackFile trackFile)
{
var files = new List<MetadataFile>();
foreach (var consumer in _metadataFactory.Enabled())
{
files.AddIfNotNull(ProcessEpisodeMetadata(consumer, series, episodeFile, new List<MetadataFile>()));
files.AddRange(ProcessEpisodeImages(consumer, series, episodeFile, new List<MetadataFile>()));
files.AddIfNotNull(ProcessEpisodeMetadata(consumer, artist, trackFile, new List<MetadataFile>()));
files.AddRange(ProcessEpisodeImages(consumer, artist, trackFile, new List<MetadataFile>()));
}
_metadataFileService.Upsert(files);
@@ -98,11 +107,11 @@ namespace NzbDrone.Core.Extras.Metadata
return files;
}
public override IEnumerable<ExtraFile> CreateAfterEpisodeImport(Series series, string seriesFolder, string seasonFolder)
public override IEnumerable<ExtraFile> CreateAfterTrackImport(Artist artist, string artistFolder, string albumFolder)
{
var metadataFiles = _metadataFileService.GetFilesBySeries(series.Id);
var metadataFiles = _metadataFileService.GetFilesByArtist(artist.Id);
if (seriesFolder.IsNullOrWhiteSpace() && seasonFolder.IsNullOrWhiteSpace())
if (artistFolder.IsNullOrWhiteSpace() && albumFolder.IsNullOrWhiteSpace())
{
return new List<MetadataFile>();
}
@@ -113,15 +122,15 @@ namespace NzbDrone.Core.Extras.Metadata
{
var consumerFiles = GetMetadataFilesForConsumer(consumer, metadataFiles);
if (seriesFolder.IsNotNullOrWhiteSpace())
if (artistFolder.IsNotNullOrWhiteSpace())
{
files.AddIfNotNull(ProcessSeriesMetadata(consumer, series, consumerFiles));
files.AddRange(ProcessSeriesImages(consumer, series, consumerFiles));
files.AddIfNotNull(ProcessArtistMetadata(consumer, artist, consumerFiles));
files.AddRange(ProcessArtistImages(consumer, artist, consumerFiles));
}
if (seasonFolder.IsNotNullOrWhiteSpace())
if (albumFolder.IsNotNullOrWhiteSpace())
{
files.AddRange(ProcessSeasonImages(consumer, series, consumerFiles));
files.AddRange(ProcessAlbumImages(consumer, artist, consumerFiles));
}
}
@@ -130,9 +139,9 @@ namespace NzbDrone.Core.Extras.Metadata
return files;
}
public override IEnumerable<ExtraFile> MoveFilesAfterRename(Series series, List<EpisodeFile> episodeFiles)
public override IEnumerable<ExtraFile> MoveFilesAfterRename(Artist artist, List<TrackFile> trackFiles)
{
var metadataFiles = _metadataFileService.GetFilesBySeries(series.Id);
var metadataFiles = _metadataFileService.GetFilesByArtist(artist.Id);
var movedFiles = new List<MetadataFile>();
// TODO: Move EpisodeImage and EpisodeMetadata metadata files, instead of relying on consumers to do it
@@ -140,21 +149,21 @@ namespace NzbDrone.Core.Extras.Metadata
foreach (var consumer in _metadataFactory.GetAvailableProviders())
{
foreach (var episodeFile in episodeFiles)
foreach (var trackFile in trackFiles)
{
var metadataFilesForConsumer = GetMetadataFilesForConsumer(consumer, metadataFiles).Where(m => m.EpisodeFileId == episodeFile.Id).ToList();
var metadataFilesForConsumer = GetMetadataFilesForConsumer(consumer, metadataFiles).Where(m => m.TrackFileId == trackFile.Id).ToList();
foreach (var metadataFile in metadataFilesForConsumer)
{
var newFileName = consumer.GetFilenameAfterMove(series, episodeFile, metadataFile);
var existingFileName = Path.Combine(series.Path, metadataFile.RelativePath);
var newFileName = consumer.GetFilenameAfterMove(artist, trackFile, metadataFile);
var existingFileName = Path.Combine(artist.Path, metadataFile.RelativePath);
if (newFileName.PathNotEquals(existingFileName))
{
try
{
_diskProvider.MoveFile(existingFileName, newFileName);
metadataFile.RelativePath = series.Path.GetRelativePath(newFileName);
metadataFile.RelativePath = artist.Path.GetRelativePath(newFileName);
movedFiles.Add(metadataFile);
}
catch (Exception ex)
@@ -171,40 +180,40 @@ namespace NzbDrone.Core.Extras.Metadata
return movedFiles;
}
public override ExtraFile Import(Series series, EpisodeFile episodeFile, string path, string extension, bool readOnly)
public override ExtraFile Import(Artist artist, TrackFile trackFile, string path, string extension, bool readOnly)
{
return null;
}
private List<MetadataFile> GetMetadataFilesForConsumer(IMetadata consumer, List<MetadataFile> seriesMetadata)
private List<MetadataFile> GetMetadataFilesForConsumer(IMetadata consumer, List<MetadataFile> artistMetadata)
{
return seriesMetadata.Where(c => c.Consumer == consumer.GetType().Name).ToList();
return artistMetadata.Where(c => c.Consumer == consumer.GetType().Name).ToList();
}
private MetadataFile ProcessSeriesMetadata(IMetadata consumer, Series series, List<MetadataFile> existingMetadataFiles)
private MetadataFile ProcessArtistMetadata(IMetadata consumer, Artist artist, List<MetadataFile> existingMetadataFiles)
{
var seriesMetadata = consumer.SeriesMetadata(series);
var artistMetadata = consumer.ArtistMetadata(artist);
if (seriesMetadata == null)
if (artistMetadata == null)
{
return null;
}
var hash = seriesMetadata.Contents.SHA256Hash();
var hash = artistMetadata.Contents.SHA256Hash();
var metadata = GetMetadataFile(series, existingMetadataFiles, e => e.Type == MetadataType.SeriesMetadata) ??
var metadata = GetMetadataFile(artist, existingMetadataFiles, e => e.Type == MetadataType.ArtistMetadata) ??
new MetadataFile
{
SeriesId = series.Id,
ArtistId = artist.Id,
Consumer = consumer.GetType().Name,
Type = MetadataType.SeriesMetadata
Type = MetadataType.ArtistMetadata
};
if (hash == metadata.Hash)
{
if (seriesMetadata.RelativePath != metadata.RelativePath)
if (artistMetadata.RelativePath != metadata.RelativePath)
{
metadata.RelativePath = seriesMetadata.RelativePath;
metadata.RelativePath = artistMetadata.RelativePath;
return metadata;
}
@@ -212,35 +221,79 @@ namespace NzbDrone.Core.Extras.Metadata
return null;
}
var fullPath = Path.Combine(series.Path, seriesMetadata.RelativePath);
var fullPath = Path.Combine(artist.Path, artistMetadata.RelativePath);
_logger.Debug("Writing Series Metadata to: {0}", fullPath);
SaveMetadataFile(fullPath, seriesMetadata.Contents);
_logger.Debug("Writing Artist Metadata to: {0}", fullPath);
SaveMetadataFile(fullPath, artistMetadata.Contents);
metadata.Hash = hash;
metadata.RelativePath = seriesMetadata.RelativePath;
metadata.RelativePath = artistMetadata.RelativePath;
metadata.Extension = Path.GetExtension(fullPath);
return metadata;
}
private MetadataFile ProcessEpisodeMetadata(IMetadata consumer, Series series, EpisodeFile episodeFile, List<MetadataFile> existingMetadataFiles)
private MetadataFile ProcessAlbumMetadata(IMetadata consumer, Album album, List<MetadataFile> existingMetadataFiles)
{
var episodeMetadata = consumer.EpisodeMetadata(series, episodeFile);
var albumMetadata = consumer.AlbumMetadata(album.Artist, album);
if (albumMetadata == null)
{
return null;
}
var hash = albumMetadata.Contents.SHA256Hash();
var metadata = GetMetadataFile(album.Artist, existingMetadataFiles, e => e.Type == MetadataType.AlbumMetadata && e.AlbumId == album.Id) ??
new MetadataFile
{
ArtistId = album.ArtistId,
AlbumId = album.Id,
Consumer = consumer.GetType().Name,
Type = MetadataType.AlbumMetadata
};
if (hash == metadata.Hash)
{
if (albumMetadata.RelativePath != metadata.RelativePath)
{
metadata.RelativePath = albumMetadata.RelativePath;
return metadata;
}
return null;
}
var fullPath = Path.Combine(album.Path, albumMetadata.RelativePath);
_logger.Debug("Writing Album Metadata to: {0}", fullPath);
SaveMetadataFile(fullPath, albumMetadata.Contents);
metadata.Hash = hash;
metadata.RelativePath = albumMetadata.RelativePath;
metadata.Extension = Path.GetExtension(fullPath);
return metadata;
}
private MetadataFile ProcessEpisodeMetadata(IMetadata consumer, Artist artist, TrackFile trackFile, List<MetadataFile> existingMetadataFiles)
{
var episodeMetadata = consumer.TrackMetadata(artist, trackFile);
if (episodeMetadata == null)
{
return null;
}
var fullPath = Path.Combine(series.Path, episodeMetadata.RelativePath);
var fullPath = Path.Combine(artist.Path, episodeMetadata.RelativePath);
var existingMetadata = GetMetadataFile(series, existingMetadataFiles, c => c.Type == MetadataType.EpisodeMetadata &&
c.EpisodeFileId == episodeFile.Id);
var existingMetadata = GetMetadataFile(artist, existingMetadataFiles, c => c.Type == MetadataType.TrackMetadata &&
c.TrackFileId == trackFile.Id);
if (existingMetadata != null)
{
var existingFullPath = Path.Combine(series.Path, existingMetadata.RelativePath);
var existingFullPath = Path.Combine(artist.Path, existingMetadata.RelativePath);
if (fullPath.PathNotEquals(existingFullPath))
{
_diskTransferService.TransferFile(existingFullPath, fullPath, TransferMode.Move);
@@ -253,11 +306,11 @@ namespace NzbDrone.Core.Extras.Metadata
var metadata = existingMetadata ??
new MetadataFile
{
SeriesId = series.Id,
SeasonNumber = episodeFile.SeasonNumber,
EpisodeFileId = episodeFile.Id,
ArtistId = artist.Id,
AlbumId = trackFile.AlbumId,
TrackFileId = trackFile.Id,
Consumer = consumer.GetType().Name,
Type = MetadataType.EpisodeMetadata,
Type = MetadataType.TrackMetadata,
RelativePath = episodeMetadata.RelativePath,
Extension = Path.GetExtension(fullPath)
};
@@ -267,7 +320,7 @@ namespace NzbDrone.Core.Extras.Metadata
return null;
}
_logger.Debug("Writing Episode Metadata to: {0}", fullPath);
_logger.Debug("Writing Track Metadata to: {0}", fullPath);
SaveMetadataFile(fullPath, episodeMetadata.Contents);
metadata.Hash = hash;
@@ -275,32 +328,32 @@ namespace NzbDrone.Core.Extras.Metadata
return metadata;
}
private List<MetadataFile> ProcessSeriesImages(IMetadata consumer, Series series, List<MetadataFile> existingMetadataFiles)
private List<MetadataFile> ProcessArtistImages(IMetadata consumer, Artist artist, List<MetadataFile> existingMetadataFiles)
{
var result = new List<MetadataFile>();
foreach (var image in consumer.SeriesImages(series))
foreach (var image in consumer.ArtistImages(artist))
{
var fullPath = Path.Combine(series.Path, image.RelativePath);
var fullPath = Path.Combine(artist.Path, image.RelativePath);
if (_diskProvider.FileExists(fullPath))
{
_logger.Debug("Series image already exists: {0}", fullPath);
_logger.Debug("Artist image already exists: {0}", fullPath);
continue;
}
var metadata = GetMetadataFile(series, existingMetadataFiles, c => c.Type == MetadataType.SeriesImage &&
var metadata = GetMetadataFile(artist, existingMetadataFiles, c => c.Type == MetadataType.ArtistImage &&
c.RelativePath == image.RelativePath) ??
new MetadataFile
{
SeriesId = series.Id,
ArtistId = artist.Id,
Consumer = consumer.GetType().Name,
Type = MetadataType.SeriesImage,
Type = MetadataType.ArtistImage,
RelativePath = image.RelativePath,
Extension = Path.GetExtension(fullPath)
};
DownloadImage(series, image);
DownloadImage(artist, image);
result.Add(metadata);
}
@@ -308,36 +361,38 @@ namespace NzbDrone.Core.Extras.Metadata
return result;
}
private List<MetadataFile> ProcessSeasonImages(IMetadata consumer, Series series, List<MetadataFile> existingMetadataFiles)
private List<MetadataFile> ProcessAlbumImages(IMetadata consumer, Artist artist, List<MetadataFile> existingMetadataFiles)
{
var result = new List<MetadataFile>();
foreach (var season in series.Seasons)
var albums = _albumService.GetAlbumsByArtist(artist.Id);
foreach (var album in albums)
{
foreach (var image in consumer.SeasonImages(series, season))
foreach (var image in consumer.AlbumImages(artist, album))
{
var fullPath = Path.Combine(series.Path, image.RelativePath);
var fullPath = Path.Combine(artist.Path, image.RelativePath);
if (_diskProvider.FileExists(fullPath))
{
_logger.Debug("Season image already exists: {0}", fullPath);
_logger.Debug("Album image already exists: {0}", fullPath);
continue;
}
var metadata = GetMetadataFile(series, existingMetadataFiles, c => c.Type == MetadataType.SeasonImage &&
c.SeasonNumber == season.SeasonNumber &&
var metadata = GetMetadataFile(artist, existingMetadataFiles, c => c.Type == MetadataType.AlbumImage &&
c.AlbumId == album.Id &&
c.RelativePath == image.RelativePath) ??
new MetadataFile
{
SeriesId = series.Id,
SeasonNumber = season.SeasonNumber,
ArtistId = artist.Id,
AlbumId = album.Id,
Consumer = consumer.GetType().Name,
Type = MetadataType.SeasonImage,
Type = MetadataType.AlbumImage,
RelativePath = image.RelativePath,
Extension = Path.GetExtension(fullPath)
};
DownloadImage(series, image);
DownloadImage(artist, image);
result.Add(metadata);
}
@@ -346,26 +401,26 @@ namespace NzbDrone.Core.Extras.Metadata
return result;
}
private List<MetadataFile> ProcessEpisodeImages(IMetadata consumer, Series series, EpisodeFile episodeFile, List<MetadataFile> existingMetadataFiles)
private List<MetadataFile> ProcessEpisodeImages(IMetadata consumer, Artist artist, TrackFile trackFile, List<MetadataFile> existingMetadataFiles)
{
var result = new List<MetadataFile>();
foreach (var image in consumer.EpisodeImages(series, episodeFile))
foreach (var image in consumer.TrackImages(artist, trackFile))
{
var fullPath = Path.Combine(series.Path, image.RelativePath);
var fullPath = Path.Combine(artist.Path, image.RelativePath);
if (_diskProvider.FileExists(fullPath))
{
_logger.Debug("Episode image already exists: {0}", fullPath);
_logger.Debug("Track image already exists: {0}", fullPath);
continue;
}
var existingMetadata = GetMetadataFile(series, existingMetadataFiles, c => c.Type == MetadataType.EpisodeImage &&
c.EpisodeFileId == episodeFile.Id);
var existingMetadata = GetMetadataFile(artist, existingMetadataFiles, c => c.Type == MetadataType.TrackImage &&
c.TrackFileId == trackFile.Id);
if (existingMetadata != null)
{
var existingFullPath = Path.Combine(series.Path, existingMetadata.RelativePath);
var existingFullPath = Path.Combine(artist.Path, existingMetadata.RelativePath);
if (fullPath.PathNotEquals(existingFullPath))
{
_diskTransferService.TransferFile(existingFullPath, fullPath, TransferMode.Move);
@@ -378,16 +433,16 @@ namespace NzbDrone.Core.Extras.Metadata
var metadata = existingMetadata ??
new MetadataFile
{
SeriesId = series.Id,
SeasonNumber = episodeFile.SeasonNumber,
EpisodeFileId = episodeFile.Id,
ArtistId = artist.Id,
AlbumId = trackFile.AlbumId,
TrackFileId = trackFile.Id,
Consumer = consumer.GetType().Name,
Type = MetadataType.EpisodeImage,
Type = MetadataType.TrackImage,
RelativePath = image.RelativePath,
Extension = Path.GetExtension(fullPath)
};
DownloadImage(series, image);
DownloadImage(artist, image);
result.Add(metadata);
}
@@ -395,9 +450,9 @@ namespace NzbDrone.Core.Extras.Metadata
return result;
}
private void DownloadImage(Series series, ImageFileResult image)
private void DownloadImage(Artist artist, ImageFileResult image)
{
var fullPath = Path.Combine(series.Path, image.RelativePath);
var fullPath = Path.Combine(artist.Path, image.RelativePath);
try
{
@@ -413,11 +468,11 @@ namespace NzbDrone.Core.Extras.Metadata
}
catch (WebException ex)
{
_logger.Warn(ex, "Couldn't download image {0} for {1}. {2}", image.Url, series, ex.Message);
_logger.Warn(ex, "Couldn't download image {0} for {1}. {2}", image.Url, artist, ex.Message);
}
catch (Exception ex)
{
_logger.Error(ex, "Couldn't download image {0} for {1}. {2}", image.Url, series, ex.Message);
_logger.Error(ex, "Couldn't download image {0} for {1}. {2}", image.Url, artist, ex.Message);
}
}
@@ -427,7 +482,7 @@ namespace NzbDrone.Core.Extras.Metadata
_mediaFileAttributeService.SetFilePermissions(path);
}
private MetadataFile GetMetadataFile(Series series, List<MetadataFile> existingMetadataFiles, Func<MetadataFile, bool> predicate)
private MetadataFile GetMetadataFile(Artist artist, List<MetadataFile> existingMetadataFiles, Func<MetadataFile, bool> predicate)
{
var matchingMetadataFiles = existingMetadataFiles.Where(predicate).ToList();
@@ -439,7 +494,7 @@ namespace NzbDrone.Core.Extras.Metadata
//Remove duplicate metadata files from DB and disk
foreach (var file in matchingMetadataFiles.Skip(1))
{
var path = Path.Combine(series.Path, file.RelativePath);
var path = Path.Combine(artist.Path, file.RelativePath);
_logger.Debug("Removing duplicate Metadata file: {0}", path);