using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Net; using System.Text.Json; using System.Threading; using LazyCache; using LazyCache.Providers; using Microsoft.Extensions.Caching.Memory; using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; using NzbDrone.Core.Books; using NzbDrone.Core.Exceptions; using NzbDrone.Core.Http; using NzbDrone.Core.MediaCover; using NzbDrone.Core.MetadataSource.Goodreads; namespace NzbDrone.Core.MetadataSource.BookInfo { public class BookInfoProxy : IProvideAuthorInfo, IProvideBookInfo, ISearchForNewBook, ISearchForNewAuthor, ISearchForNewEntity { private static readonly JsonSerializerOptions SerializerSettings = new JsonSerializerOptions { PropertyNameCaseInsensitive = false, Converters = { new STJUtcConverter() } }; private readonly IHttpClient _httpClient; private readonly ICachedHttpResponseService _cachedHttpClient; private readonly IGoodreadsSearchProxy _goodreadsSearchProxy; private readonly IAuthorService _authorService; private readonly IBookService _bookService; private readonly IEditionService _editionService; private readonly Logger _logger; private readonly IMetadataRequestBuilder _requestBuilder; private readonly ICached> _cache; private readonly CachingService _authorCache; public BookInfoProxy(IHttpClient httpClient, ICachedHttpResponseService cachedHttpClient, IGoodreadsSearchProxy goodreadsSearchProxy, IAuthorService authorService, IBookService bookService, IEditionService editionService, IMetadataRequestBuilder requestBuilder, Logger logger, ICacheManager cacheManager) { _httpClient = httpClient; _cachedHttpClient = cachedHttpClient; _goodreadsSearchProxy = goodreadsSearchProxy; _authorService = authorService; _bookService = bookService; _editionService = editionService; _requestBuilder = requestBuilder; _cache = cacheManager.GetCache>(GetType()); _logger = logger; _authorCache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions { SizeLimit = 10 }))); _authorCache.DefaultCachePolicy = new CacheDefaults { DefaultCacheDurationSeconds = 60 }; } public HashSet GetChangedAuthors(DateTime startTime) { var httpRequest = _requestBuilder.GetRequestBuilder().Create() .SetSegment("route", "author/changed") .AddQueryParam("since", startTime.ToString("o")) .Build(); httpRequest.SuppressHttpError = true; var httpResponse = _httpClient.Get(httpRequest); if (httpResponse.Resource.Limited) { return null; } return new HashSet(httpResponse.Resource.Ids.Select(x => x.ToString())); } public Author GetAuthorInfo(string foreignAuthorId, bool useCache = true) { _logger.Debug("Getting Author details GoodreadsId of {0}", foreignAuthorId); try { if (useCache) { return PollAuthor(foreignAuthorId); } return PollAuthorUncached(foreignAuthorId); } catch (BookInfoException e) { _logger.Warn(e, "Unexpected error getting author info"); throw new AuthorNotFoundException(foreignAuthorId); } } public HashSet GetChangedBooks(DateTime startTime) { return _cache.Get("ChangedBooks", () => GetChangedBooksUncached(startTime), TimeSpan.FromMinutes(30)); } private HashSet GetChangedBooksUncached(DateTime startTime) { return null; } public Tuple> GetBookInfo(string foreignBookId) { try { return PollBook(foreignBookId); } catch (BookInfoException e) { _logger.Warn(e, "Unexpected error getting book info"); throw new AuthorNotFoundException(foreignBookId); } } public List SearchForNewEntity(string title) { var books = SearchForNewBook(title, null, false); var result = new List(); foreach (var book in books) { var author = book.Author.Value; if (!result.Contains(author)) { result.Add(author); } result.Add(book); } return result; } public List SearchForNewAuthor(string title) { var books = SearchForNewBook(title, null); return books .Select(x => x.Author.Value) .DistinctBy(x => x.ForeignAuthorId) .ToList(); } public List SearchForNewBook(string title, string author, bool getAllEditions = true) { var q = title.ToLower().Trim(); if (author != null) { q += " " + author; } try { var lowerTitle = title.ToLowerInvariant(); var split = lowerTitle.Split(':'); var prefix = split[0]; if (split.Length == 2 && new[] { "author", "work", "edition", "isbn", "asin" }.Contains(prefix)) { var slug = split[1].Trim(); if (slug.IsNullOrWhiteSpace() || slug.Any(char.IsWhiteSpace)) { return new List(); } if (prefix == "author" || prefix == "work" || prefix == "edition") { var isValid = int.TryParse(slug, out var searchId); if (!isValid) { return new List(); } if (prefix == "author") { return SearchByGoodreadsAuthorId(searchId); } if (prefix == "work") { return SearchByGoodreadsWorkId(searchId); } if (prefix == "edition") { return SearchByGoodreadsBookId(searchId, getAllEditions); } } // to handle isbn / asin q = slug; } return Search(q, getAllEditions); } catch (HttpException ex) { _logger.Warn(ex, ex.Message); throw new GoodreadsException("Search for '{0}' failed. Unable to communicate with Goodreads.", title); } catch (Exception ex) { _logger.Warn(ex, ex.Message); throw new GoodreadsException("Search for '{0}' failed. Invalid response received from Goodreads.", title); } } public List SearchByIsbn(string isbn) { return Search(isbn, true); } public List SearchByAsin(string asin) { return Search(asin, true); } private List Search(string query, bool getAllEditions) { List result; try { result = _goodreadsSearchProxy.Search(query); } catch (Exception e) { _logger.Warn(e, "Error searching for {0}", query); return new List(); } var books = new List(); if (getAllEditions) { // Slower but more exhaustive, less intensive on metadata API var bookIds = result.Select(x => x.WorkId).ToList(); var idMap = result.Select(x => new { AuthorId = x.Author.Id, BookId = x.WorkId }) .GroupBy(x => x.AuthorId) .ToDictionary(x => x.Key, x => x.Select(i => i.BookId.ToString()).ToList()); List authorBooks; foreach (var author in idMap.Keys) { authorBooks = SearchByGoodreadsAuthorId(author); books.AddRange(authorBooks.Where(b => idMap[author].Contains(b.ForeignBookId))); } var missingBooks = bookIds.ExceptBy(x => x.ToString(), books, x => x.ForeignBookId, StringComparer.Ordinal).ToList(); foreach (var book in missingBooks) { books.AddRange(SearchByGoodreadsWorkId(book)); } return books; } else { // Use sparingly, hits metadata API quite hard var ids = result.Select(x => x.BookId).ToList(); if (ids.Count == 0) { return new List(); } if (ids.Count == 1) { return SearchByGoodreadsBookId(ids[0], false); } try { return MapSearchResult(ids); } catch (Exception e) { _logger.Warn(e, "Error mapping search results"); return new List(); } } } private List SearchByGoodreadsAuthorId(int id) { try { var authorId = id.ToString(); var result = GetAuthorInfo(authorId); var books = result.Books.Value; var authors = new Dictionary { { authorId, result.Metadata.Value } }; foreach (var book in books) { AddDbIds(authorId, book, authors); } return books; } catch (AuthorNotFoundException) { return new List(); } catch (BookInfoException e) { _logger.Warn(e, "Error searching by author id"); return new List(); } } public List SearchByGoodreadsWorkId(int id) { try { var tuple = GetBookInfo(id.ToString()); AddDbIds(tuple.Item1, tuple.Item2, tuple.Item3.ToDictionary(x => x.ForeignAuthorId)); return new List { tuple.Item2 }; } catch (BookNotFoundException) { return new List(); } catch (BookInfoException e) { _logger.Warn(e, "Error searching by work id"); return new List(); } } public List SearchByGoodreadsBookId(int id, bool getAllEditions) { try { var book = GetEditionInfo(id, getAllEditions); return new List { book }; } catch (AuthorNotFoundException) { return new List(); } catch (BookNotFoundException) { return new List(); } catch (EditionNotFoundException) { return new List(); } catch (BookInfoException e) { _logger.Warn(e, "Error searching by book id"); return new List(); } } private Book GetEditionInfo(int id, bool getAllEditions) { var httpRequest = _requestBuilder.GetRequestBuilder().Create() .SetSegment("route", $"book/{id}") .Build(); httpRequest.SuppressHttpError = true; // we expect a redirect var httpResponse = _httpClient.Get(httpRequest); if (httpResponse.StatusCode == HttpStatusCode.NotFound) { throw new EditionNotFoundException(id.ToString()); } if (!httpResponse.HasHttpRedirect) { throw new BookInfoException($"Unexpected response from {httpRequest.Url}"); } var location = httpResponse.Headers.GetSingleValue("Location"); var split = location.Split('/'); var type = split[0]; var newId = split[1]; Book book; List authors; if (type == "author") { var author = PollAuthor(newId); book = author.Books.Value.Where(b => b.Editions.Value.Any(e => e.ForeignEditionId == id.ToString())).FirstOrDefault(); authors = new List { author.Metadata.Value }; } else if (type == "work") { var tuple = PollBook(newId); book = tuple.Item2; authors = tuple.Item3; } else { throw new NotImplementedException($"Unexpected response from {httpResponse.Request.Url}"); } if (book == null || book.Editions.Value.All(e => e.ForeignEditionId != id.ToString())) { throw new EditionNotFoundException(id.ToString()); } if (!getAllEditions) { var trimmed = new Book(); trimmed.UseMetadataFrom(book); trimmed.Author.Value.Metadata = book.AuthorMetadata.Value; trimmed.AuthorMetadata = book.AuthorMetadata.Value; trimmed.SeriesLinks = book.SeriesLinks; var edition = book.Editions.Value.SingleOrDefault(e => e.ForeignEditionId == id.ToString()); trimmed.Editions = new List { edition }; return trimmed; } var authorDict = authors.ToDictionary(x => x.ForeignAuthorId); AddDbIds(book.AuthorMetadata.Value.ForeignAuthorId, book, authorDict); return book; } private List MapSearchResult(List ids) { var httpRequest = _requestBuilder.GetRequestBuilder().Create() .SetSegment("route", $"book/bulk") .SetHeader("Content-Type", "application/json") .Build(); httpRequest.SetContent(ids.ToJson()); httpRequest.AllowAutoRedirect = true; var httpResponse = _httpClient.Post(httpRequest); var mapped = MapBulkBook(httpResponse.Resource); var idStr = ids.Select(x => x.ToString()).ToList(); return mapped.OrderBy(b => idStr.IndexOf(b.Editions.Value.First().ForeignEditionId)).ToList(); } private List MapBulkBook(BulkBookResource resource) { var authors = resource.Authors.Select(MapAuthorMetadata).ToDictionary(x => x.ForeignAuthorId, x => x); var series = resource.Series.Select(MapSeries).ToList(); var books = new List(); foreach (var work in resource.Works) { var book = MapBook(work); var authorId = work.Books.OrderByDescending(b => b.AverageRating * b.RatingCount).First().Contributors.First().ForeignId.ToString(); AddDbIds(authorId, book, authors); books.Add(book); } MapSeriesLinks(series, books, resource.Series); return books; } private void AddDbIds(string authorId, Book book, Dictionary authors) { var dbBook = _bookService.FindById(book.ForeignBookId); if (dbBook != null) { book.UseDbFieldsFrom(dbBook); var editions = _editionService.GetEditionsByBook(dbBook.Id).ToDictionary(x => x.ForeignEditionId); // If we have any database editions, exactly one will be monitored. // So unmonitor all the found editions and let the UseDbFieldsFrom set // the monitored status foreach (var edition in book.Editions.Value) { edition.Monitored = false; if (editions.TryGetValue(edition.ForeignEditionId, out var dbEdition)) { edition.UseDbFieldsFrom(dbEdition); } } // Double check at least one edition is monitored if (book.Editions.Value.Any() && !book.Editions.Value.Any(x => x.Monitored)) { var mostPopular = book.Editions.Value.OrderByDescending(x => x.Ratings.Popularity).First(); mostPopular.Monitored = true; } } var author = _authorService.FindById(authorId); if (author == null) { if (!authors.TryGetValue(authorId, out var metadata)) { throw new BookInfoException(string.Format("Expected author metadata for id [{0}] in book data {1}", authorId, book)); } author = new Author { CleanName = Parser.Parser.CleanAuthorName(metadata.Name), Metadata = metadata }; } book.Author = author; book.AuthorMetadata = author.Metadata.Value; book.AuthorMetadataId = author.AuthorMetadataId; } private Author PollAuthor(string foreignAuthorId) { return _authorCache.GetOrAdd(foreignAuthorId, () => PollAuthorUncached(foreignAuthorId), new LazyCacheEntryOptions { AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(10), ImmediateAbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(10), Size = 1, SlidingExpiration = TimeSpan.FromMinutes(1), ExpirationMode = ExpirationMode.ImmediateEviction }.RegisterPostEvictionCallback((key, value, reason, state) => _logger.Debug($"Clearing cache for {key} due to {reason}"))); } private Author PollAuthorUncached(string foreignAuthorId) { AuthorResource resource = null; var useCache = true; for (var i = 0; i < 60; i++) { var httpRequest = _requestBuilder.GetRequestBuilder().Create() .SetSegment("route", $"author/{foreignAuthorId}") .Build(); httpRequest.AllowAutoRedirect = true; httpRequest.SuppressHttpError = true; var httpResponse = _cachedHttpClient.Get(httpRequest, useCache, TimeSpan.FromMinutes(30)); if (httpResponse.HasHttpError) { if (httpResponse.StatusCode == HttpStatusCode.NotFound) { throw new AuthorNotFoundException(foreignAuthorId); } else if (httpResponse.StatusCode == HttpStatusCode.BadRequest) { throw new BadRequestException(foreignAuthorId); } else { throw new BookInfoException("Unexpected error fetching author data"); } } resource = JsonSerializer.Deserialize(httpResponse.Content, SerializerSettings); if (resource.Works != null) { resource.Works ??= new List(); resource.Series ??= new List(); break; } useCache = false; Thread.Sleep(2000); } if (resource?.Works == null) { throw new BookInfoException($"Failed to get works for {foreignAuthorId}"); } return MapAuthor(resource); } private Tuple> PollBook(string foreignBookId) { WorkResource resource = null; for (var i = 0; i < 60; i++) { var httpRequest = _requestBuilder.GetRequestBuilder().Create() .SetSegment("route", $"work/{foreignBookId}") .Build(); httpRequest.SuppressHttpError = true; // this may redirect to an author var httpResponse = _httpClient.Get(httpRequest); if (httpResponse.StatusCode == HttpStatusCode.NotFound) { throw new BookNotFoundException(foreignBookId); } if (httpResponse.HasHttpRedirect) { var location = httpResponse.Headers.GetSingleValue("Location"); var split = location.Split('/'); var type = split[0]; var newId = split[1]; if (type == "author") { var author = PollAuthor(newId); var authorBook = author.Books.Value.SingleOrDefault(x => x.ForeignBookId == foreignBookId); if (authorBook == null) { throw new BookNotFoundException(foreignBookId); } var authorMetadata = new List { author.Metadata.Value }; return Tuple.Create(author.ForeignAuthorId, authorBook, authorMetadata); } else { throw new NotImplementedException($"Unexpected response from {httpResponse.Request.Url}"); } } if (httpResponse.HasHttpError) { if (httpResponse.StatusCode == HttpStatusCode.BadRequest) { throw new BadRequestException(foreignBookId); } else { throw new BookInfoException("Unexpected response fetching book data"); } } resource = JsonSerializer.Deserialize(httpResponse.Content, SerializerSettings); if (resource.Books != null) { break; } Thread.Sleep(2000); } if (resource?.Books == null || resource?.Authors == null || (!resource?.Authors?.Any() ?? false)) { throw new BookInfoException($"Failed to get books for {foreignBookId}"); } var book = MapBook(resource); var authorId = GetAuthorId(resource).ToString(); var metadata = resource.Authors.Select(MapAuthorMetadata).ToList(); var series = resource.Series.Select(MapSeries).ToList(); MapSeriesLinks(series, new List { book }, resource.Series); return Tuple.Create(authorId, book, metadata); } private static AuthorMetadata MapAuthorMetadata(AuthorResource resource) { var metadata = new AuthorMetadata { ForeignAuthorId = resource.ForeignId.ToString(), TitleSlug = resource.ForeignId.ToString(), Name = resource.Name.CleanSpaces(), Overview = resource.Description, Ratings = new Ratings { Votes = resource.RatingCount, Value = (decimal)resource.AverageRating }, Status = AuthorStatusType.Continuing }; metadata.SortName = metadata.Name.ToLower(); metadata.NameLastFirst = metadata.Name.ToLastFirst(); metadata.SortNameLastFirst = metadata.NameLastFirst.ToLower(); if (resource.ImageUrl.IsNotNullOrWhiteSpace()) { metadata.Images.Add(new MediaCover.MediaCover { Url = resource.ImageUrl, CoverType = MediaCoverTypes.Poster }); } if (resource.Url.IsNotNullOrWhiteSpace()) { metadata.Links.Add(new Links { Url = resource.Url, Name = "Goodreads" }); } return metadata; } private static Author MapAuthor(AuthorResource resource) { var metadata = MapAuthorMetadata(resource); var books = resource.Works .Where(x => x.ForeignId > 0 && GetAuthorId(x) == resource.ForeignId) .Select(MapBook) .ToList(); books.ForEach(x => x.AuthorMetadata = metadata); var series = resource.Series.Select(MapSeries).ToList(); MapSeriesLinks(series, books, resource.Series); var result = new Author { Metadata = metadata, CleanName = Parser.Parser.CleanAuthorName(metadata.Name), Books = books, Series = series }; return result; } private static void MapSeriesLinks(List series, List books, List resource) { var bookDict = books.ToDictionary(x => x.ForeignBookId); var seriesDict = series.ToDictionary(x => x.ForeignSeriesId); foreach (var book in books) { book.SeriesLinks = new List(); } // only take series where there are some works foreach (var s in resource.Where(x => x.LinkItems.Any())) { if (seriesDict.TryGetValue(s.ForeignId.ToString(), out var curr)) { curr.LinkItems = s.LinkItems.Where(x => x.ForeignWorkId != 0 && bookDict.ContainsKey(x.ForeignWorkId.ToString())).Select(l => new SeriesBookLink { Book = bookDict[l.ForeignWorkId.ToString()], Series = curr, IsPrimary = l.Primary, Position = l.PositionInSeries, SeriesPosition = l.SeriesPosition }).ToList(); foreach (var l in curr.LinkItems.Value) { l.Book.Value.SeriesLinks.Value.Add(l); } } } } private static Series MapSeries(SeriesResource resource) { var series = new Series { ForeignSeriesId = resource.ForeignId.ToString(), Title = resource.Title, Description = resource.Description }; return series; } private static Book MapBook(WorkResource resource) { var book = new Book { ForeignBookId = resource.ForeignId.ToString(), Title = resource.Title, TitleSlug = resource.ForeignId.ToString(), CleanTitle = Parser.Parser.CleanAuthorName(resource.Title), ReleaseDate = resource.ReleaseDate, Genres = resource.Genres, RelatedBooks = resource.RelatedWorks }; book.Links.Add(new Links { Url = resource.Url, Name = "Goodreads Editions" }); if (resource.Books != null) { book.Editions = resource.Books.Select(x => MapEdition(x)).ToList(); // monitor the most popular release var mostPopular = book.Editions.Value.OrderByDescending(x => x.Ratings.Popularity).FirstOrDefault(); if (mostPopular != null) { mostPopular.Monitored = true; // fix work title if missing if (book.Title.IsNullOrWhiteSpace()) { book.Title = mostPopular.Title; } } } else { book.Editions = new List(); } // If we are missing the book release date, set as the earliest edition release date if (!book.ReleaseDate.HasValue) { var editionReleases = book.Editions.Value .Where(x => x.ReleaseDate.HasValue && x.ReleaseDate.Value.Month != 1 && x.ReleaseDate.Value.Day != 1) .ToList(); if (editionReleases.Any()) { book.ReleaseDate = editionReleases.Min(x => x.ReleaseDate.Value); } else { editionReleases = book.Editions.Value.Where(x => x.ReleaseDate.HasValue).ToList(); if (editionReleases.Any()) { book.ReleaseDate = editionReleases.Min(x => x.ReleaseDate.Value); } } } Debug.Assert(!book.Editions.Value.Any() || book.Editions.Value.Count(x => x.Monitored) == 1, "one edition monitored"); book.AnyEditionOk = true; var ratingCount = book.Editions.Value.Sum(x => x.Ratings.Votes); if (ratingCount > 0) { book.Ratings = new Ratings { Votes = ratingCount, Value = book.Editions.Value.Sum(x => x.Ratings.Votes * x.Ratings.Value) / ratingCount }; } else { book.Ratings = new Ratings { Votes = 0, Value = 0 }; } return book; } private static Edition MapEdition(BookResource resource) { var edition = new Edition { ForeignEditionId = resource.ForeignId.ToString(), TitleSlug = resource.ForeignId.ToString(), Isbn13 = resource.Isbn13, Asin = resource.Asin, Title = resource.Title.CleanSpaces(), Language = resource.Language, Overview = resource.Description, Format = resource.Format, IsEbook = resource.IsEbook, Disambiguation = resource.EditionInformation, Publisher = resource.Publisher, PageCount = resource.NumPages ?? 0, ReleaseDate = resource.ReleaseDate, Ratings = new Ratings { Votes = resource.RatingCount, Value = (decimal)resource.AverageRating } }; if (resource.ImageUrl.IsNotNullOrWhiteSpace()) { edition.Images.Add(new MediaCover.MediaCover { Url = resource.ImageUrl, CoverType = MediaCoverTypes.Cover }); } edition.Links.Add(new Links { Url = resource.Url, Name = "Goodreads Book" }); return edition; } private static int GetAuthorId(WorkResource b) { return b.Books.OrderByDescending(x => x.RatingCount * x.AverageRating).FirstOrDefault(x => x.Contributors.Any())?.Contributors.First().ForeignId ?? 0; } } }