diff --git a/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs b/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs index 9c96a4dff1..8595889da1 100644 --- a/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs +++ b/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs @@ -30,6 +30,8 @@ namespace NzbDrone.Core.ImportLists public virtual int PageSize => 0; public virtual TimeSpan RateLimit => TimeSpan.FromSeconds(2); + protected virtual bool UsePreGeneratedPages => false; + public abstract IImportListRequestGenerator GetRequestGenerator(); public abstract IParseImportListResponse GetParser(); @@ -79,7 +81,7 @@ namespace NzbDrone.Core.ImportLists break; } - if (!IsFullPage(page)) + if (!UsePreGeneratedPages && !IsFullPage(page)) { break; } @@ -210,7 +212,26 @@ namespace NzbDrone.Core.ImportLists { var parser = GetParser(); var generator = GetRequestGenerator(); - var releases = FetchPage(generator.GetMovies().GetAllTiers().First().First(), parser); + var pageableRequests = generator.GetMovies(); + + var allTiers = pageableRequests.GetAllTiers(); + if (!allTiers.Any()) + { + return new NzbDroneValidationFailure(string.Empty, + "No pages were returned from your import list, please check your settings and the log for details.") + { IsWarning = true }; + } + + var firstTier = allTiers.First(); + if (!firstTier.Any()) + { + return new NzbDroneValidationFailure(string.Empty, + "No data could be retrieved from your import list, please check your settings.") + { IsWarning = true }; + } + + var firstRequest = firstTier.First(); + var releases = FetchPage(firstRequest, parser); if (releases.Empty()) { diff --git a/src/NzbDrone.Core/ImportLists/TMDb/List/TMDbListRequestGenerator.cs b/src/NzbDrone.Core/ImportLists/TMDb/List/TMDbListRequestGenerator.cs index 42490d110f..58487d746c 100644 --- a/src/NzbDrone.Core/ImportLists/TMDb/List/TMDbListRequestGenerator.cs +++ b/src/NzbDrone.Core/ImportLists/TMDb/List/TMDbListRequestGenerator.cs @@ -24,7 +24,7 @@ namespace NzbDrone.Core.ImportLists.TMDb.List private IEnumerable GetMoviesRequest() { - Logger.Info("Importing TMDb movies from list: {0}", Settings.ListId); + Logger.Info("TMDb List {0}: Importing movies", Settings.ListId); var requestBuilder = RequestBuilder.Create() .SetSegment("api", "4") @@ -32,19 +32,25 @@ namespace NzbDrone.Core.ImportLists.TMDb.List .SetSegment("id", Settings.ListId) .SetSegment("secondaryRoute", ""); - Logger.Debug("Getting total pages that TMDb List: {0} consists of", Settings.ListId); + Logger.Trace("TMDb List {0}: Getting total pages", Settings.ListId); var jsonResponse = JsonConvert.DeserializeObject(HttpClient.Execute(requestBuilder.Build()).Content); MaxPages = jsonResponse.TotalPages; + if (jsonResponse.TotalPages > 1) + { + Logger.Debug("TMDb List {0}: processing {1} pages", Settings.ListId, MaxPages); + } + for (var pageNumber = 1; pageNumber <= MaxPages; pageNumber++) { requestBuilder.AddQueryParam("page", pageNumber, true); var request = requestBuilder.Build(); - Logger.Debug("Importing TMDb movies from: {0}", request.Url); + Logger.Debug("TMDb List {0}: Processing page {1} of {2}", Settings.ListId, pageNumber, MaxPages); + Logger.Trace("TMDb List {0}: Request URL: {1}", Settings.ListId, request.Url); yield return new ImportListRequest(request); } diff --git a/src/NzbDrone.Core/ImportLists/TMDb/Popular/TMDbPopularRequestGenerator.cs b/src/NzbDrone.Core/ImportLists/TMDb/Popular/TMDbPopularRequestGenerator.cs index f36cc63752..df81348414 100644 --- a/src/NzbDrone.Core/ImportLists/TMDb/Popular/TMDbPopularRequestGenerator.cs +++ b/src/NzbDrone.Core/ImportLists/TMDb/Popular/TMDbPopularRequestGenerator.cs @@ -119,7 +119,8 @@ namespace NzbDrone.Core.ImportLists.TMDb.Popular var request = requestBuilder.Build(); - Logger.Debug("Importing TMDb movies from: {0}", request.Url); + Logger.Debug("TMDb Popular: Processing page {0} of {1}", pageNumber, MaxPages); + Logger.Trace("TMDb Popular: Request URL: {0}", request.Url); yield return new ImportListRequest(request); } diff --git a/src/NzbDrone.Core/ImportLists/TMDb/TMDbImportBase.cs b/src/NzbDrone.Core/ImportLists/TMDb/TMDbImportBase.cs index 48262edf93..0073b5d0a9 100644 --- a/src/NzbDrone.Core/ImportLists/TMDb/TMDbImportBase.cs +++ b/src/NzbDrone.Core/ImportLists/TMDb/TMDbImportBase.cs @@ -14,6 +14,7 @@ namespace NzbDrone.Core.ImportLists.TMDb public override ImportListType ListType => ImportListType.TMDB; public override TimeSpan MinRefreshInterval => TimeSpan.FromHours(12); public override int PageSize => 20; + protected override bool UsePreGeneratedPages => true; public readonly ISearchForNewMovie _skyhookProxy; public readonly IHttpRequestBuilderFactory _requestBuilder; diff --git a/src/NzbDrone.Core/ImportLists/TMDb/User/TMDbUserRequestGenerator.cs b/src/NzbDrone.Core/ImportLists/TMDb/User/TMDbUserRequestGenerator.cs index b0ea10234b..9964112c0d 100644 --- a/src/NzbDrone.Core/ImportLists/TMDb/User/TMDbUserRequestGenerator.cs +++ b/src/NzbDrone.Core/ImportLists/TMDb/User/TMDbUserRequestGenerator.cs @@ -52,17 +52,25 @@ namespace NzbDrone.Core.ImportLists.TMDb.User requestBuilder.Method = HttpMethod.Get; + Logger.Trace("TMDb User {0}: Getting total pages", (TMDbUserListType)Settings.ListType); + var jsonResponse = JsonConvert.DeserializeObject(HttpClient.Execute(requestBuilder.Build()).Content); MaxPages = jsonResponse.TotalPages; + if (jsonResponse.TotalPages > 1) + { + Logger.Debug("TMDb User {0}: processing {1} pages", (TMDbUserListType)Settings.ListType, MaxPages); + } + for (var pageNumber = 1; pageNumber <= MaxPages; pageNumber++) { requestBuilder.AddQueryParam("page", pageNumber, true); var request = requestBuilder.Build(); - Logger.Debug("Importing TMDb movies from: {0}", request.Url); + Logger.Debug("TMDb User {0}: Processing page {1} of {2}", (TMDbUserListType)Settings.ListType, pageNumber, MaxPages); + Logger.Trace("TMDb User {0}: Request URL: {1}", (TMDbUserListType)Settings.ListType, request.Url); yield return new ImportListRequest(request); }