Whole album matching and fingerprinting (#592)

* Cache result of GetAllArtists

* Fixed: Manual import not respecting album import notifications

* Fixed: partial album imports stay in queue, prompting manual import

* Fixed: Allow release if tracks are missing

* Fixed: Be tolerant of missing/extra "The" at start of artist name

* Improve manual import UI

* Omit video tracks from DB entirely

* Revert "faster test packaging in build.sh"

This reverts commit 2723e2a7b8.

-u and -T are not supported on macOS

* Fix tests on linux and macOS

* Actually lint on linux

On linux yarn runs scripts with sh not bash so ** doesn't recursively glob

* Match whole albums

* Option to disable fingerprinting

* Rip out MediaInfo

* Don't split up things that have the same album selected in manual import

* Try to speed up IndentificationService

* More speedups

* Some fixes and increase power of recording id

* Fix NRE when no tags

* Fix NRE when some (but not all) files in a directory have missing tags

* Bump taglib, tidy up tag parsing

* Add a health check

* Remove media info setting

* Tags -> audioTags

* Add some tests where tags are null

* Rename history events

* Add missing method to interface

* Reinstate MediaInfo tags and update info with artist scan

Also adds migration to remove old format media info

* This file no longer exists

* Don't penalise year if missing from tags

* Formatting improvements

* Use correct system newline

* Switch to the netstandard2.0 library to support net 461

* TagLib.File is IDisposable so should be in a using

* Improve filename matching and add tests

* Neater logging of parsed tags

* Fix disk scan tests for new media info update

* Fix quality detection source

* Fix Inexact Artist/Album match

* Add button to clear track mapping

* Fix warning

* Pacify eslint

* Use \ not /

* Fix UI updates

* Fix media covers

Prevent localizing URL propaging back to the metadata object

* Reduce database overhead broadcasting UI updates

* Relax timings a bit to make test pass

* Remove irrelevant tests

* Test framework for identification service

* Fix PreferMissingToBadMatch test case

* Make fingerprinting more robust

* More logging

* Penalize unknown media format and country

* Prefer USA to UK

* Allow Data CD

* Fix exception if fingerprinting fails for all files

* Fix tests

* Fix NRE

* Allow apostrophes and remove accents in filename aggregation

* Address codacy issues

* Cope with old versions of fpcalc and suggest upgrade

* fpcalc health check passes if fingerprinting disabled

* Get the Artist meta with the artist

* Fix the mapper so that lazy loaded lists will be populated on Join

And therefore we can join TrackFiles on Tracks by default and avoid an
extra query

* Rename subtitle -> lyric

* Tidy up MediaInfoFormatter
This commit is contained in:
ta264
2019-02-16 14:49:24 +00:00
committed by Qstick
parent 8bf364945f
commit bb02d73c42
174 changed files with 11577 additions and 3490 deletions
@@ -44,7 +44,7 @@ namespace NzbDrone.Core.Test.MediaFiles
.Returns(true);
Mocker.GetMock<IImportApprovedTracks>()
.Setup(s => s.Import(It.IsAny<List<ImportDecision>>(), true, null, ImportMode.Auto))
.Setup(s => s.Import(It.IsAny<List<ImportDecision<LocalTrack>>>(), true, null, ImportMode.Auto))
.Returns(new List<ImportResult>());
var downloadItem = Builder<DownloadClientItem>.CreateNew()
@@ -76,15 +76,15 @@ namespace NzbDrone.Core.Test.MediaFiles
{
var localTrack = new LocalTrack();
var imported = new List<ImportDecision>();
imported.Add(new ImportDecision(localTrack));
var imported = new List<ImportDecision<LocalTrack>>();
imported.Add(new ImportDecision<LocalTrack>(localTrack));
Mocker.GetMock<IMakeImportDecision>()
.Setup(s => s.GetImportDecisions(It.IsAny<List<string>>(), It.IsAny<Artist>(), null))
.Returns(imported);
Mocker.GetMock<IImportApprovedTracks>()
.Setup(s => s.Import(It.IsAny<List<ImportDecision>>(), It.IsAny<bool>(), It.IsAny<DownloadClientItem>(), It.IsAny<ImportMode>()))
.Setup(s => s.Import(It.IsAny<List<ImportDecision<LocalTrack>>>(), It.IsAny<bool>(), It.IsAny<DownloadClientItem>(), It.IsAny<ImportMode>()))
.Returns(imported.Select(i => new ImportResult(i)).ToList())
.Callback(() => WasImportedResponse());
}
@@ -155,7 +155,7 @@ namespace NzbDrone.Core.Test.MediaFiles
public void should_not_delete_folder_if_no_files_were_imported()
{
Mocker.GetMock<IImportApprovedTracks>()
.Setup(s => s.Import(It.IsAny<List<ImportDecision>>(), false, null, ImportMode.Auto))
.Setup(s => s.Import(It.IsAny<List<ImportDecision<LocalTrack>>>(), false, null, ImportMode.Auto))
.Returns(new List<ImportResult>());
Subject.ProcessRootFolder(new DirectoryInfo(_droneFactory));
@@ -171,15 +171,15 @@ namespace NzbDrone.Core.Test.MediaFiles
var localTrack = new LocalTrack();
var imported = new List<ImportDecision>();
imported.Add(new ImportDecision(localTrack));
var imported = new List<ImportDecision<LocalTrack>>();
imported.Add(new ImportDecision<LocalTrack>(localTrack));
Mocker.GetMock<IMakeImportDecision>()
.Setup(s => s.GetImportDecisions(It.IsAny<List<string>>(), It.IsAny<Artist>(), null))
.Returns(imported);
Mocker.GetMock<IImportApprovedTracks>()
.Setup(s => s.Import(It.IsAny<List<ImportDecision>>(), true, null, ImportMode.Auto))
.Setup(s => s.Import(It.IsAny<List<ImportDecision<LocalTrack>>>(), true, null, ImportMode.Auto))
.Returns(imported.Select(i => new ImportResult(i)).ToList());
Subject.ProcessRootFolder(new DirectoryInfo(_droneFactory));
@@ -225,8 +225,8 @@ namespace NzbDrone.Core.Test.MediaFiles
result.Should().HaveCount(1);
result.First().ImportDecision.Should().NotBeNull();
result.First().ImportDecision.LocalTrack.Should().NotBeNull();
result.First().ImportDecision.LocalTrack.Path.Should().Be(fileName);
result.First().ImportDecision.Item.Should().NotBeNull();
result.First().ImportDecision.Item.Path.Should().Be(fileName);
result.First().Result.Should().Be(ImportResultType.Rejected);
}
@@ -237,15 +237,15 @@ namespace NzbDrone.Core.Test.MediaFiles
var localTrack = new LocalTrack();
var imported = new List<ImportDecision>();
imported.Add(new ImportDecision(localTrack));
var imported = new List<ImportDecision<LocalTrack>>();
imported.Add(new ImportDecision<LocalTrack>(localTrack));
Mocker.GetMock<IMakeImportDecision>()
.Setup(s => s.GetImportDecisions(It.IsAny<List<string>>(), It.IsAny<Artist>(), null))
.Returns(imported);
Mocker.GetMock<IImportApprovedTracks>()
.Setup(s => s.Import(It.IsAny<List<ImportDecision>>(), true, null, ImportMode.Auto))
.Setup(s => s.Import(It.IsAny<List<ImportDecision<LocalTrack>>>(), true, null, ImportMode.Auto))
.Returns(imported.Select(i => new ImportResult(i)).ToList());
//Mocker.GetMock<IDetectSample>()
@@ -272,56 +272,6 @@ namespace NzbDrone.Core.Test.MediaFiles
ExceptionVerification.ExpectedWarns(1);
}
[Test]
public void should_use_folder_if_folder_import()
{
GivenValidArtist();
var folderName = @"C:\media\ba09030e-1234-1234-1234-123456789abc\[HorribleSubs] Maria the Virgin Witch - 09 [720p]".AsOsAgnostic();
var fileName = @"C:\media\ba09030e-1234-1234-1234-123456789abc\[HorribleSubs] Maria the Virgin Witch - 09 [720p]\[HorribleSubs] Maria the Virgin Witch - 09 [720p].mkv".AsOsAgnostic();
Mocker.GetMock<IDiskProvider>().Setup(c => c.FolderExists(folderName))
.Returns(true);
Mocker.GetMock<IDiskProvider>().Setup(c => c.GetFiles(folderName, SearchOption.TopDirectoryOnly))
.Returns(new[] { fileName });
var localTrack = new LocalTrack();
var imported = new List<ImportDecision>();
imported.Add(new ImportDecision(localTrack));
Subject.ProcessPath(fileName);
Mocker.GetMock<IMakeImportDecision>()
.Verify(s => s.GetImportDecisions(It.IsAny<List<string>>(), It.IsAny<Artist>(), It.Is<ParsedTrackInfo>(v => v.TrackNumbers.First() == 9)), Times.Once());
}
[Test]
public void should_not_use_folder_if_file_import()
{
GivenValidArtist();
var fileName = @"C:\media\ba09030e-1234-1234-1234-123456789abc\Torrents\[HorribleSubs] Maria the Virgin Witch - 09 [720p].mkv".AsOsAgnostic();
Mocker.GetMock<IDiskProvider>().Setup(c => c.FolderExists(fileName))
.Returns(false);
Mocker.GetMock<IDiskProvider>().Setup(c => c.FileExists(fileName))
.Returns(true);
var localTrack = new LocalTrack();
var imported = new List<ImportDecision>();
imported.Add(new ImportDecision(localTrack));
var result = Subject.ProcessPath(fileName);
Mocker.GetMock<IMakeImportDecision>()
.Verify(s => s.GetImportDecisions(It.IsAny<List<string>>(), It.IsAny<Artist>(), null), Times.Once());
}
[Test]
public void should_not_process_if_file_and_folder_do_not_exist()
{
@@ -348,15 +298,15 @@ namespace NzbDrone.Core.Test.MediaFiles
var localTrack = new LocalTrack();
var imported = new List<ImportDecision>();
imported.Add(new ImportDecision(localTrack));
var imported = new List<ImportDecision<LocalTrack>>();
imported.Add(new ImportDecision<LocalTrack>(localTrack));
Mocker.GetMock<IMakeImportDecision>()
.Setup(s => s.GetImportDecisions(It.IsAny<List<string>>(), It.IsAny<Artist>(), null))
.Returns(imported);
Mocker.GetMock<IImportApprovedTracks>()
.Setup(s => s.Import(It.IsAny<List<ImportDecision>>(), true, null, ImportMode.Auto))
.Setup(s => s.Import(It.IsAny<List<ImportDecision<LocalTrack>>>(), true, null, ImportMode.Auto))
.Returns(new List<ImportResult>());
//Mocker.GetMock<IDetectSample>()
@@ -424,13 +374,13 @@ namespace NzbDrone.Core.Test.MediaFiles
private void VerifyNoImport()
{
Mocker.GetMock<IImportApprovedTracks>().Verify(c => c.Import(It.IsAny<List<ImportDecision>>(), true, null, ImportMode.Auto),
Mocker.GetMock<IImportApprovedTracks>().Verify(c => c.Import(It.IsAny<List<ImportDecision<LocalTrack>>>(), true, null, ImportMode.Auto),
Times.Never());
}
private void VerifyImport()
{
Mocker.GetMock<IImportApprovedTracks>().Verify(c => c.Import(It.IsAny<List<ImportDecision>>(), true, null, ImportMode.Auto),
Mocker.GetMock<IImportApprovedTracks>().Verify(c => c.Import(It.IsAny<List<ImportDecision<LocalTrack>>>(), true, null, ImportMode.Auto),
Times.Once());
}
}