tedkulp/Readarr
1
0
Fork 0
mirror of https://github.com/Readarr/Readarr.git synced 2026-04-17 21:25:39 -04:00
Code Issues Projects Releases Wiki Activity

New: Search for new editions from goodreads when identifying

Browse Source
This commit is contained in:
ta264
2021-03-31 21:20:32 +01:00
parent c1f2ea6c8a
commit 41f5f0f2d4
19 changed files with 149 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/NzbDrone.Core.Test/MediaFiles/TrackImport/Identification/CandidateServiceFixture.cs
View File

@@ -35,7 +35,7 @@ namespace NzbDrone.Core.Test.MediaFiles.BookImport.Identification
}
};
Subject.GetRemoteCandidates(edition).Should().BeEmpty();
Subject.GetRemoteCandidates(edition, null).Should().BeEmpty();
}
}
}

2
src/NzbDrone.Core/Books/Model/Book.cs
View File

@@ -1,7 +1,7 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
using Equ;
using Newtonsoft.Json;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.MediaFiles;

52
src/NzbDrone.Core/MediaFiles/BookImport/Identification/CandidateService.cs
View File

@@ -13,7 +13,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
public interface ICandidateService
{
List<CandidateEdition> GetDbCandidatesFromTags(LocalEdition localEdition, IdentificationOverrides idOverrides, bool includeExisting);
IEnumerable<CandidateEdition> GetRemoteCandidates(LocalEdition localEdition);
IEnumerable<CandidateEdition> GetRemoteCandidates(LocalEdition localEdition, IdentificationOverrides idOverrides);
}
public class CandidateService : ICandidateService
@@ -184,8 +184,10 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
return candidateReleases;
}
public IEnumerable<CandidateEdition> GetRemoteCandidates(LocalEdition localEdition)
public IEnumerable<CandidateEdition> GetRemoteCandidates(LocalEdition localEdition, IdentificationOverrides idOverrides)
{
// TODO handle edition override
// Gets candidate book releases from the metadata server.
// Will eventually need adding locally if we find a match
List<Book> remoteBooks;
@@ -210,7 +212,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
remoteBooks = new List<Book>();
}
foreach (var candidate in ToCandidates(remoteBooks, seenCandidates))
foreach (var candidate in ToCandidates(remoteBooks, seenCandidates, idOverrides))
{
yield return candidate;
}
@@ -232,7 +234,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
remoteBooks = new List<Book>();
}
foreach (var candidate in ToCandidates(remoteBooks, seenCandidates))
foreach (var candidate in ToCandidates(remoteBooks, seenCandidates, idOverrides))
{
yield return candidate;
}
@@ -255,15 +257,18 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
remoteBooks = new List<Book>();
}
foreach (var candidate in ToCandidates(remoteBooks, seenCandidates))
foreach (var candidate in ToCandidates(remoteBooks, seenCandidates, idOverrides))
{
yield return candidate;
}
}
}
// If we got an id result, stop
if (seenCandidates.Any())
// If we got an id result, or any overrides are set, stop
if (seenCandidates.Any() ||
idOverrides?.Edition != null ||
idOverrides?.Book != null ||
idOverrides?.Author != null)
{
yield break;
}
@@ -301,7 +306,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
remoteBooks = new List<Book>();
}
foreach (var candidate in ToCandidates(remoteBooks, seenCandidates))
foreach (var candidate in ToCandidates(remoteBooks, seenCandidates, idOverrides))
{
yield return candidate;
}
@@ -324,7 +329,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
remoteBooks = new List<Book>();
}
foreach (var candidate in ToCandidates(remoteBooks, seenCandidates))
foreach (var candidate in ToCandidates(remoteBooks, seenCandidates, idOverrides))
{
yield return candidate;
}
@@ -342,14 +347,14 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
remoteBooks = new List<Book>();
}
foreach (var candidate in ToCandidates(remoteBooks, seenCandidates))
foreach (var candidate in ToCandidates(remoteBooks, seenCandidates, idOverrides))
{
yield return candidate;
}
}
}
private List<CandidateEdition> ToCandidates(IEnumerable<Book> books, HashSet<string> seenCandidates)
private List<CandidateEdition> ToCandidates(IEnumerable<Book> books, HashSet<string> seenCandidates, IdentificationOverrides idOverrides)
{
var candidates = new List<CandidateEdition>();
@@ -359,10 +364,11 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
// by a database lazy load
foreach (var edition in book.Editions.Value)
{
if (!seenCandidates.Contains(edition.ForeignEditionId))
edition.Book = book;
if (!seenCandidates.Contains(edition.ForeignEditionId) && SatisfiesOverride(edition, idOverrides))
{
seenCandidates.Add(edition.ForeignEditionId);
edition.Book = book;
candidates.Add(new CandidateEdition
{
Edition = edition,
@@ -374,5 +380,25 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
return candidates;
}
private bool SatisfiesOverride(Edition edition, IdentificationOverrides idOverride)
{
if (idOverride?.Edition != null)
{
return edition.ForeignEditionId == idOverride.Edition.ForeignEditionId;
}
if (idOverride?.Book != null)
{
return edition.Book.Value.ForeignBookId == idOverride.Book.ForeignBookId;
}
if (idOverride?.Author != null)
{
return edition.Book.Value.Author.Value.ForeignAuthorId == idOverride.Author.ForeignAuthorId;
}
return true;
}
}
}

28
src/NzbDrone.Core/MediaFiles/BookImport/Identification/IdentificationService.cs
View File

@@ -115,6 +115,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
private void IdentifyRelease(LocalEdition localBookRelease, IdentificationOverrides idOverrides, ImportDecisionMakerConfig config)
{
var watch = System.Diagnostics.Stopwatch.StartNew();
bool usedRemote = false;
IEnumerable<CandidateEdition> candidateReleases = _candidateService.GetDbCandidatesFromTags(localBookRelease, idOverrides, config.IncludeExisting);
@@ -126,14 +127,20 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
_logger.Debug($"Retrieved {allLocalTracks.Count} possible tracks in {watch.ElapsedMilliseconds}ms");
if (!candidateReleases.Any() && config.AddNewAuthors)
if (!candidateReleases.Any())
{
candidateReleases = _candidateService.GetRemoteCandidates(localBookRelease);
candidateReleases = _candidateService.GetRemoteCandidates(localBookRelease, idOverrides);
if (!config.AddNewAuthors)
{
candidateReleases = candidateReleases.Where(x => x.Edition.Book.Value.Id > 0);
}
usedRemote = true;
}
if (!candidateReleases.Any())
{
// can't find any candidates even after fingerprinting
// can't find any candidates even after using remote search
// populate the overrides and return
foreach (var localTrack in localBookRelease.LocalBooks)
{
@@ -147,6 +154,21 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
GetBestRelease(localBookRelease, candidateReleases, allLocalTracks);
// If the result isn't great and we haven't tried remote candidates, try looking for remote candidates
// Goodreads may have a better edition of a local book
if (localBookRelease.Distance.NormalizedDistance() > 0.15 && !usedRemote)
{
_logger.Debug("Match not good enough, trying remote candidates");
candidateReleases = _candidateService.GetRemoteCandidates(localBookRelease, idOverrides);
if (!config.AddNewAuthors)
{
candidateReleases = candidateReleases.Where(x => x.Edition.Book.Value.Id > 0);
}
GetBestRelease(localBookRelease, candidateReleases, allLocalTracks);
}
_logger.Debug($"Best release found in {watch.ElapsedMilliseconds}ms");
localBookRelease.PopulateMatch();

2
src/NzbDrone.Core/MediaFiles/BookImport/Manual/ManualImportFile.cs
View File

@@ -9,7 +9,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Manual
public string Path { get; set; }
public int AuthorId { get; set; }
public int BookId { get; set; }
public int EditionId { get; set; }
public string ForeignEditionId { get; set; }
public QualityModel Quality { get; set; }
public string DownloadId { get; set; }
public bool DisableReleaseSwitching { get; set; }

27
src/NzbDrone.Core/MediaFiles/BookImport/Manual/ManualImportService.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.IO;
using System.IO.Abstractions;
using System.Linq;
@@ -15,6 +16,7 @@ using NzbDrone.Core.Download;
using NzbDrone.Core.Download.TrackedDownloads;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.RootFolders;
@@ -37,6 +39,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Manual
private readonly IAuthorService _authorService;
private readonly IBookService _bookService;
private readonly IEditionService _editionService;
private readonly IProvideBookInfo _bookInfo;
private readonly IAudioTagService _audioTagService;
private readonly IImportApprovedBooks _importApprovedBooks;
private readonly ITrackedDownloadService _trackedDownloadService;
@@ -53,6 +56,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Manual
IAuthorService authorService,
IBookService bookService,
IEditionService editionService,
IProvideBookInfo bookInfo,
IAudioTagService audioTagService,
IImportApprovedBooks importApprovedBooks,
ITrackedDownloadService trackedDownloadService,
@@ -69,6 +73,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Manual
_authorService = authorService;
_bookService = bookService;
_editionService = editionService;
_bookInfo = bookInfo;
_audioTagService = audioTagService;
_importApprovedBooks = importApprovedBooks;
_trackedDownloadService = trackedDownloadService;
@@ -299,7 +304,14 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Manual
var author = _authorService.GetAuthor(file.AuthorId);
var book = _bookService.GetBook(file.BookId);
var edition = _editionService.GetEdition(file.EditionId);
var edition = _editionService.GetEditionByForeignEditionId(file.ForeignEditionId);
if (edition == null)
{
var tuple = _bookInfo.GetBookInfo(file.ForeignEditionId);
edition = tuple.Item2.Editions.Value.SingleOrDefault(x => x.ForeignEditionId == file.ForeignEditionId);
}
var fileTrackInfo = _audioTagService.ReadTags(file.Path) ?? new ParsedTrackInfo();
var fileInfo = _diskProvider.GetFileInfo(file.Path);
@@ -355,20 +367,23 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Manual
foreach (var groupedTrackedDownload in importedTrackedDownload.GroupBy(i => i.TrackedDownload.DownloadItem.DownloadId).ToList())
{
var trackedDownload = groupedTrackedDownload.First().TrackedDownload;
var outputPath = trackedDownload.ImportItem.OutputPath.FullPath;
if (_diskProvider.FolderExists(outputPath))
{
if (_downloadedTracksImportService.ShouldDeleteFolder(
_diskProvider.GetDirectoryInfo(outputPath),
trackedDownload.RemoteBook.Author) && trackedDownload.DownloadItem.CanMoveFiles)
if (_downloadedTracksImportService.ShouldDeleteFolder(_diskProvider.GetDirectoryInfo(outputPath)) &&
trackedDownload.DownloadItem.CanMoveFiles)
{
_diskProvider.DeleteFolder(outputPath, true);
}
}
if (groupedTrackedDownload.Select(c => c.ImportResult).Count(c => c.Result == ImportResultType.Imported) >= Math.Max(1, trackedDownload.RemoteBook.Books.Count))
var importedCount = groupedTrackedDownload.Select(c => c.ImportResult)
.Count(c => c.Result == ImportResultType.Imported);
var downloadItemCount = Math.Max(1, trackedDownload.RemoteBook?.Books.Count ?? 1);
var allItemsImported = importedCount >= downloadItemCount;
if (allItemsImported)
{
trackedDownload.State = TrackedDownloadState.Imported;
_eventAggregator.PublishEvent(new DownloadCompletedEvent(trackedDownload));

6
src/NzbDrone.Core/MediaFiles/DownloadedBooksImportService.cs
View File

@@ -21,7 +21,7 @@ namespace NzbDrone.Core.MediaFiles
{
List<ImportResult> ProcessRootFolder(IDirectoryInfo directoryInfo);
List<ImportResult> ProcessPath(string path, ImportMode importMode = ImportMode.Auto, Author author = null, DownloadClientItem downloadClientItem = null);
bool ShouldDeleteFolder(IDirectoryInfo directoryInfo, Author author);
bool ShouldDeleteFolder(IDirectoryInfo directoryInfo);
}
public class DownloadedBooksImportService : IDownloadedBooksImportService
@@ -110,7 +110,7 @@ namespace NzbDrone.Core.MediaFiles
return new List<ImportResult>();
}
public bool ShouldDeleteFolder(IDirectoryInfo directoryInfo, Author author)
public bool ShouldDeleteFolder(IDirectoryInfo directoryInfo)
{
try
{
@@ -238,7 +238,7 @@ namespace NzbDrone.Core.MediaFiles
if (importMode == ImportMode.Move &&
importResults.Any(i => i.Result == ImportResultType.Imported) &&
ShouldDeleteFolder(directoryInfo, author))
ShouldDeleteFolder(directoryInfo))
{
_logger.Debug("Deleting folder after importing valid files");
_diskProvider.DeleteFolder(directoryInfo.FullName, true);

49
src/NzbDrone.Core/MetadataSource/Goodreads/GoodreadsProxy.cs
View File

@@ -421,30 +421,12 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
public List<Book> SearchByIsbn(string isbn)
{
var result = SearchByField("isbn", isbn);
// we don't get isbn back in search result, but if only one result assume the query was correct
// and add in the searched isbn
if (result.Count == 1 && result[0].Editions.Value.Count == 1)
{
result[0].Editions.Value[0].Isbn13 = isbn;
}
return result;
return SearchByField("isbn", isbn, e => e.Isbn13 = isbn);
}
public List<Book> SearchByAsin(string asin)
{
var result = SearchByField("asin", asin);
// we don't get isbn back in search result, but if only one result assume the query was correct
// and add in the searched isbn
if (result.Count == 1 && result[0].Editions.Value.Count == 1)
{
result[0].Editions.Value[0].Asin = asin;
}
return result;
return SearchByField("asin", asin, e => e.Asin = asin);
}
public List<Book> SearchByGoodreadsId(int id)
@@ -492,7 +474,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
}
}
public List<Book> SearchByField(string field, string query)
public List<Book> SearchByField(string field, string query, Action<Edition> applyData = null)
{
try
{
@@ -500,9 +482,9 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
.AddQueryParam("q", query)
.Build();
var result = _cachedHttpClient.Get<List<SearchJsonResource>>(httpRequest, true, TimeSpan.FromDays(5));
var response = _cachedHttpClient.Get<List<SearchJsonResource>>(httpRequest, true, TimeSpan.FromDays(5));
return result.Resource.SelectList(MapJsonSearchResult);
return response.Resource.SelectList(x => MapJsonSearchResult(x, response.Resource.Count == 1 ? applyData : null));
}
catch (HttpException)
{
@@ -735,7 +717,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
return book;
}
private Book MapJsonSearchResult(SearchJsonResource resource)
private Book MapJsonSearchResult(SearchJsonResource resource, Action<Edition> applyData = null)
{
var book = _bookService.FindById(resource.WorkId.ToString());
var edition = _editionService.GetEditionByForeignEditionId(resource.BookId.ToString());
@@ -751,6 +733,11 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
PageCount = resource.PageCount,
Overview = resource.Description?.Html ?? string.Empty
};
if (applyData != null)
{
applyData(edition);
}
}
edition.Monitored = true;
@@ -777,7 +764,19 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
};
}
book.Editions = new List<Edition> { edition };
if (book.Editions != null)
{
if (book.Editions.Value.Any())
{
edition.Monitored = false;
}
book.Editions.Value.Add(edition);
}
else
{
book.Editions = new List<Edition> { edition };
}
var authorId = resource.Author.Id.ToString();
var author = _authorService.FindById(authorId);

8
src/NzbDrone.Integration.Test/ApiTests/WantedFixture.cs
View File

@@ -64,7 +64,7 @@ namespace NzbDrone.Integration.Test.ApiTests
{
EnsureProfileCutoff(1, Quality.AZW3);
var author = EnsureAuthor("14586394", "43765115", "Andrew Hunter Murray", true);
EnsureBookFile(author, 1, 1, Quality.MOBI);
EnsureBookFile(author, 1, "43765115", Quality.MOBI);
var result = WantedCutoffUnmet.GetPaged(0, 15, "releaseDate", "desc");
@@ -88,7 +88,7 @@ namespace NzbDrone.Integration.Test.ApiTests
{
EnsureProfileCutoff(1, Quality.AZW3);
var author = EnsureAuthor("14586394", "43765115", "Andrew Hunter Murray", false);
EnsureBookFile(author, 1, 1, Quality.MOBI);
EnsureBookFile(author, 1, "43765115", Quality.MOBI);
var result = WantedCutoffUnmet.GetPaged(0, 15, "releaseDate", "desc");
@@ -101,7 +101,7 @@ namespace NzbDrone.Integration.Test.ApiTests
{
EnsureProfileCutoff(1, Quality.AZW3);
var author = EnsureAuthor("14586394", "43765115", "Andrew Hunter Murray", true);
EnsureBookFile(author, 1, 1, Quality.MOBI);
EnsureBookFile(author, 1, "43765115", Quality.MOBI);
var result = WantedCutoffUnmet.GetPaged(0, 15, "releaseDate", "desc");
@@ -126,7 +126,7 @@ namespace NzbDrone.Integration.Test.ApiTests
{
EnsureProfileCutoff(1, Quality.AZW3);
var author = EnsureAuthor("14586394", "43765115", "Andrew Hunter Murray", false);
EnsureBookFile(author, 1, 1, Quality.MOBI);
EnsureBookFile(author, 1, "43765115", Quality.MOBI);
var result = WantedCutoffUnmet.GetPaged(0, 15, "releaseDate", "desc", "monitored", "false");

12
src/NzbDrone.Integration.Test/IntegrationTestBase.cs
View File

@@ -234,13 +234,13 @@ namespace NzbDrone.Integration.Test
Assert.Fail("Timed on wait");
}
public AuthorResource EnsureAuthor(string authorId, string goodreadsBookId, string authorName, bool? monitored = null)
public AuthorResource EnsureAuthor(string authorId, string goodreadsEditionId, string authorName, bool? monitored = null)
{
var result = Author.All().FirstOrDefault(v => v.ForeignAuthorId == authorId);
if (result == null)
{
var lookup = Author.Lookup("readarr:" + goodreadsBookId);
var lookup = Author.Lookup("readarr:" + goodreadsEditionId);
var author = lookup.First();
author.QualityProfileId = 1;
author.MetadataProfileId = 1;
@@ -291,9 +291,9 @@ namespace NzbDrone.Integration.Test
}
}
public void EnsureBookFile(AuthorResource author, int bookId, int editionId, Quality quality)
public void EnsureBookFile(AuthorResource author, int bookId, string foreignEditionId, Quality quality)
{
var result = Books.GetBooksInAuthor(author.Id).Single(v => v.Id == editionId);
var result = Books.GetBooksInAuthor(author.Id).Single(v => v.Id == bookId);
// if (result.BookFile == null)
if (true)
@@ -312,14 +312,14 @@ namespace NzbDrone.Integration.Test
Path = path,
AuthorId = author.Id,
BookId = bookId,
EditionId = editionId,
ForeignEditionId = foreignEditionId,
Quality = new QualityModel(quality)
}
}
});
Commands.WaitAll();
var track = Books.GetBooksInAuthor(author.Id).Single(x => x.Id == editionId);
var track = Books.GetBooksInAuthor(author.Id).Single(x => x.Id == bookId);
// track.BookFileId.Should().NotBe(0);
}

2
src/Readarr.Api.V1/ManualImport/ManualImportController.cs
View File

@@ -78,7 +78,7 @@ namespace Readarr.Api.V1.ManualImport
Size = resource.Size,
Author = resource.Author == null ? null : _authorService.GetAuthor(resource.Author.Id),
Book = resource.Book == null ? null : _bookService.GetBook(resource.Book.Id),
Edition = resource.EditionId == 0 ? null : _editionService.GetEdition(resource.EditionId),
Edition = resource.ForeignEditionId == null ? null : _editionService.GetEditionByForeignEditionId(resource.ForeignEditionId),
Quality = resource.Quality,
DownloadId = resource.DownloadId,
AdditionalFile = resource.AdditionalFile,

4
src/Readarr.Api.V1/ManualImport/ManualImportResource.cs
View File

@@ -17,7 +17,7 @@ namespace Readarr.Api.V1.ManualImport
public long Size { get; set; }
public AuthorResource Author { get; set; }
public BookResource Book { get; set; }
public int EditionId { get; set; }
public string ForeignEditionId { get; set; }
public QualityModel Quality { get; set; }
public int QualityWeight { get; set; }
public string DownloadId { get; set; }
@@ -45,7 +45,7 @@ namespace Readarr.Api.V1.ManualImport
Size = model.Size,
Author = model.Author.ToResource(),
Book = model.Book.ToResource(),
EditionId = model.Edition?.Id ?? 0,
ForeignEditionId = model.Edition?.ForeignEditionId,
Quality = model.Quality,
//QualityWeight