1
0
mirror of https://github.com/Radarr/Radarr.git synced 2026-04-18 21:35:51 -04:00

Compare commits

...

5 Commits

Author SHA1 Message Date
ta264 c49fb9ec07 Fixed: Speed up RSS sync 2020-10-08 21:25:01 +01:00
ta264 5c248c02dc To revert: ignore failing test 2020-10-08 21:25:01 +01:00
ta264 0b8ff7907a To revert: timestamps in console logs 2020-10-08 21:25:01 +01:00
ta264 6af1289640 To tidy: speed up movie module 2020-10-08 21:25:01 +01:00
ta264 bd7780196c Add FileInfo utility functions to DiskProvider 2020-10-07 21:06:03 +01:00
13 changed files with 233 additions and 73 deletions
@@ -731,7 +731,7 @@ namespace NzbDrone.Common.Test.DiskTests
// Note: never returns anything.
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.GetFileInfos(It.IsAny<string>()))
.Setup(v => v.GetFileInfos(It.IsAny<string>(), SearchOption.TopDirectoryOnly))
.Returns(new List<FileInfo>());
Mocker.GetMock<IDiskProvider>()
@@ -765,8 +765,8 @@ namespace NzbDrone.Common.Test.DiskTests
.Returns<string>(v => new DirectoryInfo(v).GetDirectories().ToList());
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.GetFileInfos(It.IsAny<string>()))
.Returns<string>(v => new DirectoryInfo(v).GetFiles().ToList());
.Setup(v => v.GetFileInfos(It.IsAny<string>(), SearchOption.TopDirectoryOnly))
.Returns<string, SearchOption>((v, _) => new DirectoryInfo(v).GetFiles().ToList());
Mocker.GetMock<IDiskProvider>()
.Setup(v => v.GetFileSize(It.IsAny<string>()))
+9 -2
View File
@@ -505,13 +505,20 @@ namespace NzbDrone.Common.Disk
return di.GetDirectories().ToList();
}
public List<FileInfo> GetFileInfos(string path)
public FileInfo GetFileInfo(string path)
{
Ensure.That(path, () => path).IsValidPath();
return new FileInfo(path);
}
public List<FileInfo> GetFileInfos(string path, SearchOption searchOption = SearchOption.TopDirectoryOnly)
{
Ensure.That(path, () => path).IsValidPath();
var di = new DirectoryInfo(path);
return di.GetFiles().ToList();
return di.GetFiles("*", searchOption).ToList();
}
public void RemoveEmptySubfolders(string path)
+2 -1
View File
@@ -52,7 +52,8 @@ namespace NzbDrone.Common.Disk
List<IMount> GetMounts();
IMount GetMount(string path);
List<DirectoryInfo> GetDirectoryInfos(string path);
List<FileInfo> GetFileInfos(string path);
FileInfo GetFileInfo(string path);
List<FileInfo> GetFileInfos(string path, SearchOption searchOption = SearchOption.TopDirectoryOnly);
void RemoveEmptySubfolders(string path);
void SaveStream(Stream stream, string path);
bool IsValidFilePermissionMask(string mask);
@@ -108,7 +108,7 @@ namespace NzbDrone.Common.Instrumentation
var coloredConsoleTarget = new ColoredConsoleTarget();
coloredConsoleTarget.Name = "consoleLogger";
coloredConsoleTarget.Layout = "[${level}] ${logger}: ${message} ${onexception:inner=${newline}${newline}[v${assembly-version}] ${exception:format=ToString}${newline}${exception:format=Data}${newline}}";
coloredConsoleTarget.Layout = @"${date:format=yyyy-M-d HH\:mm\:ss.ff} | [${level}] ${logger}: ${message} ${onexception:inner=${newline}${newline}[v${assembly-version}] ${exception:format=ToString}${newline}${exception:format=Data}${newline}}";
var loggingRule = new LoggingRule("*", level, coloredConsoleTarget);
@@ -15,6 +15,7 @@ using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.MediaCoverTests
{
[TestFixture]
[Ignore("temp - revert")]
public class MediaCoverServiceFixture : CoreTest<MediaCoverService>
{
private Movie _movie;
@@ -28,7 +28,7 @@ namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests
Subject.GetMovie(title);
Mocker.GetMock<IMovieService>()
.Verify(s => s.FindByTitle(Parser.Parser.ParseMovieTitle(title, false).MovieTitle), Times.Once());
.Verify(s => s.FindByTitle(Parser.Parser.ParseMovieTitle(title, false).MovieTitle, It.IsAny<int>(), null, null, null), Times.Once());
}
/*[Test]
@@ -118,7 +118,7 @@ namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests
Subject.Map(_parsedMovieInfo, "", null);
Mocker.GetMock<IMovieService>()
.Verify(v => v.FindByTitle(It.IsAny<string>(), It.IsAny<int>()), Times.Once());
.Verify(v => v.FindByTitle(It.IsAny<string>(), It.IsAny<int>(), null, null, null), Times.Once());
}
[Test]
@@ -0,0 +1,15 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(185)]
public class add_alternative_title_indices : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Create.Index().OnTable("AlternativeTitles").OnColumn("CleanTitle");
Create.Index().OnTable("MovieTranslations").OnColumn("CleanTitle");
}
}
}
@@ -1,9 +1,11 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading;
using NLog;
using NzbDrone.Common;
using NzbDrone.Common.Disk;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
@@ -17,7 +19,8 @@ namespace NzbDrone.Core.MediaCover
{
public interface IMapCoversToLocal
{
void ConvertToLocalUrls(int movieId, IEnumerable<MediaCover> covers);
Dictionary<string, FileInfo> GetCoverFileInfos();
void ConvertToLocalUrls(int movieId, IEnumerable<MediaCover> covers, Dictionary<string, FileInfo> fileInfos = null);
string GetCoverPath(int movieId, MediaCoverTypes mediaCoverTypes, int? height = null);
}
@@ -70,7 +73,19 @@ namespace NzbDrone.Core.MediaCover
return Path.Combine(GetMovieCoverPath(movieId), coverTypes.ToString().ToLower() + heightSuffix + ".jpg");
}
public void ConvertToLocalUrls(int movieId, IEnumerable<MediaCover> covers)
public Dictionary<string, FileInfo> GetCoverFileInfos()
{
if (!_diskProvider.FolderExists(_coverRootFolder))
{
return new Dictionary<string, FileInfo>();
}
return _diskProvider
.GetFileInfos(_coverRootFolder, SearchOption.AllDirectories)
.ToDictionary(x => x.FullName, PathEqualityComparer.Instance);
}
public void ConvertToLocalUrls(int movieId, IEnumerable<MediaCover> covers, Dictionary<string, FileInfo> fileInfos = null)
{
if (movieId == 0)
{
@@ -90,9 +105,21 @@ namespace NzbDrone.Core.MediaCover
mediaCover.RemoteUrl = mediaCover.Url;
mediaCover.Url = _configFileProvider.UrlBase + @"/MediaCover/" + movieId + "/" + mediaCover.CoverType.ToString().ToLower() + ".jpg";
if (_diskProvider.FileExists(filePath))
FileInfo file;
var fileExists = false;
if (fileInfos != null)
{
var lastWrite = _diskProvider.FileGetLastWrite(filePath);
fileExists = fileInfos.TryGetValue(filePath, out file);
}
else
{
file = _diskProvider.GetFileInfo(filePath);
fileExists = file.Exists;
}
if (fileExists)
{
var lastWrite = file.LastWriteTimeUtc;
mediaCover.Url += "?lastWrite=" + lastWrite.Ticks;
}
}
+64 -4
View File
@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using Dapper;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.MediaFiles;
using NzbDrone.Core.Messaging.Events;
@@ -109,13 +110,28 @@ namespace NzbDrone.Core.Movies
public List<Movie> FindByTitles(List<string> titles)
{
var distinct = titles.Distinct().ToList();
var results = new List<Movie>();
results.AddRange(FindByMovieTitles(distinct));
results.AddRange(FindByAltTitles(distinct));
results.AddRange(FindByTransTitles(distinct));
return results.DistinctBy(x => x.Id).ToList();
}
// This is a bit of a hack, but if you try to combine / rationalise these then
// SQLite makes a mess of the query plan and ends up doing a table scan
private List<Movie> FindByMovieTitles(List<string> titles)
{
var movieDictionary = new Dictionary<int, Movie>();
var builder = Builder()
var builder = new SqlBuilder()
.LeftJoin<Movie, AlternativeTitle>((m, t) => m.Id == t.MovieId)
.LeftJoin<Movie, MovieFile>((m, f) => m.Id == f.MovieId)
.LeftJoin<Movie, MovieTranslation>((m, tr) => m.Id == tr.MovieId)
.OrWhere<Movie>(x => distinct.Contains(x.CleanTitle))
.OrWhere<AlternativeTitle>(x => distinct.Contains(x.CleanTitle))
.OrWhere<MovieTranslation>(x => distinct.Contains(x.CleanTitle));
.Join<Movie, Profile>((m, p) => m.ProfileId == p.Id)
.Where<Movie>(x => titles.Contains(x.CleanTitle));
_ = _database.QueryJoined<Movie, Profile, AlternativeTitle, MovieFile, MovieTranslation>(
builder,
@@ -124,6 +140,50 @@ namespace NzbDrone.Core.Movies
return movieDictionary.Values.ToList();
}
private List<Movie> FindByAltTitles(List<string> titles)
{
var movieDictionary = new Dictionary<int, Movie>();
var builder = new SqlBuilder()
.LeftJoin<AlternativeTitle, Movie>((t, m) => t.MovieId == m.Id)
.LeftJoin<Movie, MovieFile>((m, f) => m.Id == f.MovieId)
.LeftJoin<Movie, MovieTranslation>((m, tr) => m.Id == tr.MovieId)
.Join<Movie, Profile>((m, p) => m.ProfileId == p.Id)
.Where<AlternativeTitle>(x => titles.Contains(x.CleanTitle));
_ = _database.QueryJoined<AlternativeTitle, Profile, Movie, MovieFile, MovieTranslation>(
builder,
(altTitle, profile, movie, file, trans) =>
{
_ = Map(movieDictionary, movie, profile, altTitle, file, trans);
return null;
});
return movieDictionary.Values.ToList();
}
private List<Movie> FindByTransTitles(List<string> titles)
{
var movieDictionary = new Dictionary<int, Movie>();
var builder = new SqlBuilder()
.LeftJoin<MovieTranslation, Movie>((tr, m) => tr.MovieId == m.Id)
.LeftJoin<Movie, AlternativeTitle>((m, t) => m.Id == t.MovieId)
.LeftJoin<Movie, MovieFile>((m, f) => m.Id == f.MovieId)
.Join<Movie, Profile>((m, p) => m.ProfileId == p.Id)
.Where<MovieTranslation>(x => titles.Contains(x.CleanTitle));
_ = _database.QueryJoined<MovieTranslation, Profile, Movie, MovieFile, AlternativeTitle>(
builder,
(trans, profile, movie, file, altTitle) =>
{
_ = Map(movieDictionary, movie, profile, altTitle, file, trans);
return null;
});
return movieDictionary.Values.ToList();
}
public Movie FindByImdbId(string imdbid)
{
var imdbIdWithPrefix = Parser.Parser.NormalizeImdbId(imdbid);
+35 -23
View File
@@ -27,6 +27,8 @@ namespace NzbDrone.Core.Movies
List<Movie> FindByTmdbId(List<int> tmdbids);
Movie FindByTitle(string title);
Movie FindByTitle(string title, int year);
Movie FindByTitle(string title, int? year, string arabicTitle, string romanTitle, List<Movie> candidates);
List<Movie> FindByTitleCandidates(string title, out string roman, out string arabic);
Movie FindByTitleSlug(string slug);
Movie FindByPath(string path);
List<string> AllMoviePaths();
@@ -103,45 +105,35 @@ namespace NzbDrone.Core.Movies
public Movie FindByTitle(string title)
{
return FindByTitle(title.CleanMovieTitle(), null);
var candidates = FindByTitleCandidates(title, out var arabicTitle, out var romanTitle);
return FindByTitle(title, null, arabicTitle, romanTitle, candidates);
}
public Movie FindByTitle(string title, int year)
{
return FindByTitle(title.CleanMovieTitle(), year as int?);
var candidates = FindByTitleCandidates(title, out var arabicTitle, out var romanTitle);
return FindByTitle(title, year, arabicTitle, romanTitle, candidates);
}
private Movie FindByTitle(string cleanTitle, int? year)
public Movie FindByTitle(string cleanTitle, int? year, string arabicTitle, string romanTitle, List<Movie> candidates)
{
cleanTitle = cleanTitle.ToLowerInvariant();
var cleanTitleWithRomanNumbers = cleanTitle;
var cleanTitleWithArabicNumbers = cleanTitle;
foreach (var arabicRomanNumeral in RomanNumeralParser.GetArabicRomanNumeralsMapping())
{
var arabicNumber = arabicRomanNumeral.ArabicNumeralAsString;
var romanNumber = arabicRomanNumeral.RomanNumeral;
cleanTitleWithRomanNumbers = cleanTitleWithRomanNumbers.Replace(arabicNumber, romanNumber);
cleanTitleWithArabicNumbers = cleanTitleWithArabicNumbers.Replace(romanNumber, arabicNumber);
}
var candidates = _movieRepository.FindByTitles(new List<string> { cleanTitle, cleanTitleWithArabicNumbers, cleanTitleWithRomanNumbers });
var result = candidates.Where(x => x.CleanTitle == cleanTitle).FirstWithYear(year);
if (result == null)
{
result =
candidates.Where(movie => movie.CleanTitle == cleanTitleWithArabicNumbers).FirstWithYear(year) ??
candidates.Where(movie => movie.CleanTitle == cleanTitleWithRomanNumbers).FirstWithYear(year);
candidates.Where(movie => movie.CleanTitle == arabicTitle).FirstWithYear(year) ??
candidates.Where(movie => movie.CleanTitle == romanTitle).FirstWithYear(year);
}
if (result == null)
{
result = candidates
.Where(m => m.AlternativeTitles.Any(t => t.CleanTitle == cleanTitle ||
t.CleanTitle == cleanTitleWithArabicNumbers ||
t.CleanTitle == cleanTitleWithRomanNumbers))
t.CleanTitle == arabicTitle ||
t.CleanTitle == romanTitle))
.FirstWithYear(year);
}
@@ -149,14 +141,34 @@ namespace NzbDrone.Core.Movies
{
result = candidates
.Where(m => m.Translations.Any(t => t.CleanTitle == cleanTitle ||
t.CleanTitle == cleanTitleWithArabicNumbers ||
t.CleanTitle == cleanTitleWithRomanNumbers))
t.CleanTitle == arabicTitle ||
t.CleanTitle == romanTitle))
.FirstWithYear(year);
}
return result;
}
public List<Movie> FindByTitleCandidates(string title, out string arabicTitle, out string romanTitle)
{
var cleanTitle = title.CleanMovieTitle().ToLowerInvariant();
romanTitle = cleanTitle;
arabicTitle = cleanTitle;
foreach (var arabicRomanNumeral in RomanNumeralParser.GetArabicRomanNumeralsMapping())
{
var arabicNumber = arabicRomanNumeral.ArabicNumeralAsString;
var romanNumber = arabicRomanNumeral.RomanNumeral;
romanTitle = romanTitle.Replace(arabicNumber, romanNumber);
arabicTitle = arabicTitle.Replace(romanNumber, arabicNumber);
}
romanTitle = romanTitle.ToLowerInvariant();
return _movieRepository.FindByTitles(new List<string> { cleanTitle, arabicTitle, romanTitle });
}
public Movie FindByImdbId(string imdbid)
{
return _movieRepository.FindByImdbId(imdbid);
+8 -8
View File
@@ -201,20 +201,20 @@ namespace NzbDrone.Core.Parser
private bool TryGetMovieByTitleAndOrYear(ParsedMovieInfo parsedMovieInfo, out MappingResult result)
{
Func<Movie, bool> isNotNull = movie => movie != null;
Movie movieByTitleAndOrYear;
var candidates = _movieService.FindByTitleCandidates(parsedMovieInfo.MovieTitle, out var arabicTitle, out var romanTitle);
Movie movieByTitleAndOrYear;
if (parsedMovieInfo.Year > 1800)
{
movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, parsedMovieInfo.Year);
if (isNotNull(movieByTitleAndOrYear))
movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, parsedMovieInfo.Year, arabicTitle, romanTitle, candidates);
if (movieByTitleAndOrYear != null)
{
result = new MappingResult { Movie = movieByTitleAndOrYear };
return true;
}
movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle);
if (isNotNull(movieByTitleAndOrYear))
movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, null, arabicTitle, romanTitle, candidates);
if (movieByTitleAndOrYear != null)
{
result = new MappingResult { Movie = movieByTitleAndOrYear, MappingResultType = MappingResultType.WrongYear };
return false;
@@ -224,8 +224,8 @@ namespace NzbDrone.Core.Parser
return false;
}
movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle);
if (isNotNull(movieByTitleAndOrYear))
movieByTitleAndOrYear = _movieService.FindByTitle(parsedMovieInfo.MovieTitle, null, arabicTitle, romanTitle, candidates);
if (movieByTitleAndOrYear != null)
{
result = new MappingResult { Movie = movieByTitleAndOrYear };
return true;
+48 -11
View File
@@ -1,7 +1,10 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using FluentValidation;
using Nancy;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Datastore.Events;
@@ -40,6 +43,7 @@ namespace Radarr.Api.V3.Movies
private readonly IManageCommandQueue _commandQueueManager;
private readonly IUpgradableSpecification _qualityUpgradableSpecification;
private readonly IConfigService _configService;
private readonly Logger _logger;
public MovieModule(IBroadcastSignalRMessage signalRBroadcaster,
IMovieService moviesService,
@@ -56,7 +60,8 @@ namespace Radarr.Api.V3.Movies
MovieAncestorValidator moviesAncestorValidator,
SystemFolderValidator systemFolderValidator,
ProfileExistsValidator profileExistsValidator,
MovieFolderAsRootFolderValidator movieFolderAsRootFolderValidator)
MovieFolderAsRootFolderValidator movieFolderAsRootFolderValidator,
Logger logger)
: base(signalRBroadcaster)
{
_moviesService = moviesService;
@@ -66,6 +71,7 @@ namespace Radarr.Api.V3.Movies
_configService = configService;
_coverMapper = coverMapper;
_commandQueueManager = commandQueueManager;
_logger = logger;
GetResourceAll = AllMovie;
GetResourceById = GetMovie;
@@ -104,6 +110,8 @@ namespace Radarr.Api.V3.Movies
var moviesResources = new List<MovieResource>();
var configLanguage = (Language)_configService.MovieInfoLanguage;
Dictionary<string, FileInfo> coverFileInfos = null;
if (tmdbId > 0)
{
var movie = _moviesService.FindByTmdbId(tmdbId);
@@ -117,17 +125,34 @@ namespace Radarr.Api.V3.Movies
}
else
{
var translations = _movieTranslationService.GetAllTranslationsForLanguage(configLanguage);
var movies = _moviesService.GetAllMovies();
_logger.Debug("Getting all movies");
var movieTask = Task.Run(() => _moviesService.GetAllMovies());
_logger.Debug("started movie task");
var translations = _movieTranslationService
.GetAllTranslationsForLanguage(configLanguage)
.ToDictionary(x => x.Id);
_logger.Debug("Got translations");
coverFileInfos = _coverMapper.GetCoverFileInfos();
_logger.Debug("got cover file infos");
var movies = movieTask.GetAwaiter().GetResult();
_logger.Debug("Got movies");
moviesResources = new List<MovieResource>(movies.Count);
foreach (var movie in movies)
{
var translation = GetMovieTranslation(translations, movie, configLanguage);
moviesResources.Add(movie.ToResource(_qualityUpgradableSpecification, translation));
var translation = GetTranslationFromDict(translations, movie, configLanguage);
var resource = movie.ToResource(_qualityUpgradableSpecification, translation);
_coverMapper.ConvertToLocalUrls(resource.Id, resource.Images, coverFileInfos);
moviesResources.Add(resource);
}
}
MapCoversToLocal(moviesResources.ToArray());
_logger.Debug("Mapped resources; done");
return moviesResources;
}
@@ -168,6 +193,21 @@ namespace Radarr.Api.V3.Movies
return translations.FirstOrDefault(t => t.Language == configLanguage && t.MovieId == movie.Id);
}
private MovieTranslation GetTranslationFromDict(Dictionary<int, MovieTranslation> translations, Movie movie, Language configLanguage)
{
if (configLanguage == Language.Original)
{
return new MovieTranslation
{
Title = movie.OriginalTitle,
Overview = movie.Overview
};
}
translations.TryGetValue(movie.Id, out var translation);
return translation;
}
private int AddMovie(MovieResource moviesResource)
{
var movie = _addMovieService.AddMovie(moviesResource.ToModel());
@@ -211,12 +251,9 @@ namespace Radarr.Api.V3.Movies
_moviesService.DeleteMovie(id, deleteFiles, addExclusion);
}
private void MapCoversToLocal(params MovieResource[] movies)
private void MapCoversToLocal(MovieResource movie)
{
foreach (var moviesResource in movies)
{
_coverMapper.ConvertToLocalUrls(moviesResource.Id, moviesResource.Images);
}
_coverMapper.ConvertToLocalUrls(movie.Id, movie.Images);
}
public void Handle(MovieImportedEvent message)