Feature/Add TMDb Functionality (#739)

* Inital TMDb List, needs paging support and user lists, private or public

* Clean up Base

* TMDb grabs upto 5 pages for import, update validation, added minimum vote average

* Added logic for MovieLinksTemplate

* Clean up a bit

* Add Public Lists

src/NzbDrone.Core/NetImport/HttpNetImportBase.cs

@@ -27,9 +27,9 @@ namespace NzbDrone.Core.NetImport
 
         public override bool Enabled => true;
 
-        public bool SupportsPaging => PageSize > 0;
+        public bool SupportsPaging => PageSize > 20;
 
-        public virtual int PageSize => 0;
+        public virtual int PageSize => 20;
 
         public virtual TimeSpan RateLimit => TimeSpan.FromSeconds(2);
 
@@ -58,13 +58,6 @@ namespace NzbDrone.Core.NetImport
 
             try
             {
-                var fullyUpdated = false;
-                Movie lastMovie = null;
-                if (isRecent)
-                {
-                    //lastReleaseInfo = _indexerStatusService.GetLastRssSyncReleaseInfo(Definition.Id);
-                }
-
                 for (int i = 0; i < pageableRequestChain.Tiers; i++)
                 {
                     var pageableRequests = pageableRequestChain.GetTier(i);
@@ -80,37 +73,6 @@ namespace NzbDrone.Core.NetImport
                             var page = FetchPage(request, parser);
 
                             pagedReleases.AddRange(page);
-
-                            if (isRecent && page.Any())
-                            {
-                                if (lastMovie == null)
-                                {
-                                    fullyUpdated = true;
-                                    break;
-                                }/*
-                                var oldestReleaseDate = page.Select(v => v.PublishDate).Min();
-                                if (oldestReleaseDate < lastReleaseInfo.PublishDate || page.Any(v => v.DownloadUrl == lastReleaseInfo.DownloadUrl))
-                                {
-                                    fullyUpdated = true;
-                                    break;
-                                }
-
-                                if (pagedReleases.Count >= MaxNumResultsPerQuery &&
-                                    oldestReleaseDate < DateTime.UtcNow - TimeSpan.FromHours(24))
-                                {
-                                    fullyUpdated = false;
-                                    break;
-                                }*///update later
-                            }
-                            else if (pagedReleases.Count >= MaxNumResultsPerQuery)
-                            {
-                                break;
-                            }
-
-                            if (!IsFullPage(page))
-                            {
-                                break;
-                            }
                         }
 
                         movies.AddRange(pagedReleases);
@@ -121,29 +83,9 @@ namespace NzbDrone.Core.NetImport
                         break;
                     }
                 }
-
-                if (isRecent && !movies.Empty())
-                {
-                    var ordered = movies.OrderByDescending(v => v.Title).ToList();
-
-                    lastMovie = ordered.First();
-                    //_indexerStatusService.UpdateRssSyncStatus(Definition.Id, lastReleaseInfo);
-                }
-
-                //_indexerStatusService.RecordSuccess(Definition.Id);
             }
             catch (WebException webException)
             {
-                if (webException.Status == WebExceptionStatus.NameResolutionFailure ||
-                    webException.Status == WebExceptionStatus.ConnectFailure)
-                {
-                    //_indexerStatusService.RecordConnectionFailure(Definition.Id);
-                }
-                else
-                {
-                    //_indexerStatusService.RecordFailure(Definition.Id);
-                }
-
                 if (webException.Message.Contains("502") || webException.Message.Contains("503") ||
                     webException.Message.Contains("timed out"))
                 {
@@ -158,28 +100,23 @@ namespace NzbDrone.Core.NetImport
             {
                 if ((int)httpException.Response.StatusCode == 429)
                 {
-                    //_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
                     _logger.Warn("API Request Limit reached for {0}", this);
                 }
                 else
                 {
-                    //_indexerStatusService.RecordFailure(Definition.Id);
                     _logger.Warn("{0} {1}", this, httpException.Message);
                 }
             }
             catch (RequestLimitReachedException)
             {
-                //_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
                 _logger.Warn("API Request Limit reached for {0}", this);
             }
             catch (ApiKeyException)
             {
-                //_indexerStatusService.RecordFailure(Definition.Id);
                 _logger.Warn("Invalid API Key for {0} {1}", this, url);
             }
             catch (CloudFlareCaptchaException ex)
             {
-                //_indexerStatusService.RecordFailure(Definition.Id);
                 if (ex.IsExpired)
                 {
                     _logger.Error(ex, "Expired CAPTCHA token for {0}, please refresh in indexer settings.", this);
@@ -191,13 +128,11 @@ namespace NzbDrone.Core.NetImport
             }
             catch (IndexerException ex)
             {
-                //_indexerStatusService.RecordFailure(Definition.Id);
                 var message = string.Format("{0} - {1}", ex.Message, url);
                 _logger.Warn(ex, message);
             }
             catch (Exception feedEx)
             {
-                //_indexerStatusService.RecordFailure(Definition.Id);
                 feedEx.Data.Add("FeedUrl", url);
                 _logger.Error(feedEx, "An error occurred while processing feed. " + url);
             }
@@ -205,11 +140,6 @@ namespace NzbDrone.Core.NetImport
             return movies;
         }
 
-        protected virtual bool IsFullPage(IList<Movie> page)
-        {
-            return PageSize != 0 && page.Count >= PageSize;
-        }
-
         protected virtual IList<Movie> FetchPage(NetImportRequest request, IParseNetImportResponse parser)
         {
             var response = FetchIndexerResponse(request);

src/NzbDrone.Core/NetImport/TMDb/TMDbImport.cs

@@ -0,0 +1,40 @@
+using NLog;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Parser;
+
+
+namespace NzbDrone.Core.NetImport.TMDb
+{
+    public class TMDbImport : HttpNetImportBase<TMDbSettings>
+    {
+        public override string Name => "TMDb Lists";
+        public override bool Enabled => true;
+        public override bool EnableAuto => false;
+        private readonly IHttpClient _httpClient;
+        private readonly Logger _logger;
+
+        public TMDbImport(IHttpClient httpClient, IConfigService configService, IParsingService parsingService,
+            Logger logger)
+            : base(httpClient, configService, parsingService, logger)
+        {
+            _logger = logger;
+            _httpClient = httpClient;
+        }
+
+        public override INetImportRequestGenerator GetRequestGenerator()
+        {
+            return new TMDbRequestGenerator()
+            {
+                Settings = Settings,
+                Logger = _logger,
+                HttpClient = _httpClient
+            };
+        }
+
+        public override IParseNetImportResponse GetParser()
+        {
+            return new TMDbParser(Settings);
+        }
+    }
+}

src/NzbDrone.Core/NetImport/TMDb/TMDbLanguageCodes.cs

@@ -0,0 +1,28 @@
+using System.Runtime.Serialization;
+
+namespace NzbDrone.Core.NetImport.TMDb
+{
+    public enum TMDbLanguageCodes
+    {
+        da,
+        nl,
+        en,
+        fi,
+        fr,
+        de,
+        el,
+        hu,
+        it,
+        ja,
+        ko,
+        no,
+        pl,
+        pt,
+        ru,
+        es,
+        sv,
+        tr,
+        vi,
+        zh
+    }
+}

src/NzbDrone.Core/NetImport/TMDb/TMDbListType.cs

@@ -0,0 +1,18 @@
+using System.Runtime.Serialization;
+
+namespace NzbDrone.Core.NetImport.TMDb
+{
+    public enum TMDbListType
+    {
+        [EnumMember(Value = "List")]
+        List = 0,
+        [EnumMember(Value = "In Theaters")]
+        Theaters = 1,
+        [EnumMember(Value = "Popular")]
+        Popular = 2,
+        [EnumMember(Value = "Top Rated")]
+        Top = 3,
+        [EnumMember(Value = "Upcoming")]
+        Upcoming = 4
+    }
+}

src/NzbDrone.Core/NetImport/TMDb/TMDbParser.cs

@@ -0,0 +1,110 @@
+using Newtonsoft.Json;
+using NzbDrone.Core.NetImport.Exceptions;
+using System;
+using System.Collections.Generic;
+using System.Net;
+using NLog;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.MetadataSource.SkyHook.Resource;
+
+namespace NzbDrone.Core.NetImport.TMDb
+{
+    public class TMDbParser : IParseNetImportResponse
+    {
+        private readonly TMDbSettings _settings;
+        private NetImportResponse _importResponse;
+        private readonly Logger _logger;
+
+        public TMDbParser(TMDbSettings settings)
+        {
+            _settings = settings;
+        }
+
+        public IList<Tv.Movie> ParseResponse(NetImportResponse importResponse)
+        {
+            _importResponse = importResponse;
+
+            var movies = new List<Tv.Movie>();
+
+            if (!PreProcess(_importResponse))
+            {
+                return movies;
+            }
+
+            if (_settings.ListType != (int) TMDbListType.List)
+            {
+                var jsonResponse = JsonConvert.DeserializeObject<MovieSearchRoot>(_importResponse.Content);
+
+                // no movies were return
+                if (jsonResponse == null)
+                {
+                    return movies;
+                }
+
+                foreach (var movie in jsonResponse.results)
+                {
+                    if (movie.vote_average >= double.Parse(_settings.MinVoteAverage))
+                    {
+                        movies.AddIfNotNull(new Tv.Movie()
+                        {
+                            Title = movie.title,
+                            TmdbId = movie.id,
+                            ImdbId = null,
+                            Year = DateTime.Parse(movie.release_date).Year
+                        });
+                    }
+                }
+            }
+            else
+            {
+                var jsonResponse = JsonConvert.DeserializeObject<ListResponseRoot>(_importResponse.Content);
+
+                // no movies were return
+                if (jsonResponse == null)
+                {
+                    return movies;
+                }
+
+                foreach (var movie in jsonResponse.items)
+                {
+                    // Skip non-movie things
+                    if (movie.media_type != "movie")
+                    {
+                        continue;
+                    }
+
+                    if (movie.vote_average >= double.Parse(_settings.MinVoteAverage))
+                    {
+                        movies.AddIfNotNull(new Tv.Movie()
+                        {
+                            Title = movie.title,
+                            TmdbId = movie.id,
+                            ImdbId = null,
+                            Year = DateTime.Parse(movie.release_date).Year
+                        });
+                    }
+                }
+            }
+
+
+            return movies;
+        }
+
+        protected virtual bool PreProcess(NetImportResponse indexerResponse)
+        {
+            if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
+            {
+                throw new NetImportException(indexerResponse, "Indexer API call resulted in an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
+            }
+
+            if (indexerResponse.HttpResponse.Headers.ContentType != null && indexerResponse.HttpResponse.Headers.ContentType.Contains("text/json") &&
+                indexerResponse.HttpRequest.Headers.Accept != null && !indexerResponse.HttpRequest.Headers.Accept.Contains("text/json"))
+            {
+                throw new NetImportException(indexerResponse, "Indexer responded with html content. Site is likely blocked or unavailable.");
+            }
+
+            return true;
+        }
+
+    }
+}

src/NzbDrone.Core/NetImport/TMDb/TMDbRequestGenerator.cs

@@ -0,0 +1,104 @@
+using System;
+using NzbDrone.Common.Http;
+using System.Collections.Generic;
+using NLog;
+using NzbDrone.Common.Serializer;
+using NzbDrone.Core.MetadataSource.SkyHook.Resource;
+
+namespace NzbDrone.Core.NetImport.TMDb
+{
+    public class TMDbRequestGenerator : INetImportRequestGenerator
+    {
+        public TMDbSettings Settings { get; set; }
+        public IHttpClient HttpClient { get; set; }
+        public Logger Logger { get; set; }
+
+        public int MaxPages { get; set; }
+
+        public TMDbRequestGenerator()
+        {
+            MaxPages = 3;
+        }
+
+        public virtual NetImportPageableRequestChain GetMovies()
+        {
+            var searchType = "";
+
+            switch (Settings.ListType)
+            {
+                case (int)TMDbListType.List:
+                    searchType = $"/3/list/{Settings.ListId}";
+                    break;
+                case (int)TMDbListType.Theaters:
+                    searchType = "/3/movie/now_playing";
+                    break;
+                case (int)TMDbListType.Popular:
+                    searchType = "/3/movie/popular";
+                    break;
+                case (int)TMDbListType.Top:
+                    searchType = "/3/movie/top_rated";
+                    break;
+                case (int)TMDbListType.Upcoming:
+                    searchType = "/3/movie/upcoming";
+                    break;
+            }
+
+            var pageableRequests = new NetImportPageableRequestChain();
+            if (Settings.ListType != (int) TMDbListType.List)
+            {
+                // First query to get the total_Pages
+                var requestBuilder = new HttpRequestBuilder($"{Settings.Link.Trim()}")
+                {
+                    LogResponseContent = true
+                };
+
+                requestBuilder.Method = HttpMethod.GET;
+                requestBuilder.Resource(searchType);
+
+                var request = requestBuilder
+                    .AddQueryParam("api_key", "1a7373301961d03f97f853a876dd1212")
+                    .Accept(HttpAccept.Json)
+                    .Build();
+
+                var response = HttpClient.Execute(request);
+                var result = Json.Deserialize<MovieSearchRoot>(response.Content);
+
+                // @TODO Prolly some error handling to do here
+                pageableRequests.Add(GetPagedRequests(searchType, result.total_pages));
+                return pageableRequests;
+            }
+            else
+            {
+                pageableRequests.Add(GetPagedRequests(searchType, 0));
+                return pageableRequests;
+            }
+        } 
+        
+        private IEnumerable<NetImportRequest> GetPagedRequests(string searchType, int totalPages)
+        {
+            var baseUrl = $"{Settings.Link.Trim()}{searchType}?api_key=1a7373301961d03f97f853a876dd1212";
+            if (Settings.ListType != (int) TMDbListType.List)
+            {
+                for (var pageNumber = 1; pageNumber <= totalPages; pageNumber++)
+                {
+                    // Limit the amount of pages
+                    if (pageNumber >= MaxPages + 1)
+                    {
+                        Logger.Info(
+                            $"Found more than {MaxPages} pages, skipping the {totalPages - (MaxPages + 1)} remaining pages");
+                        break;
+                    }
+
+                    Logger.Trace($"Importing TMDb movies from: {baseUrl}&page={pageNumber}");
+                    yield return new NetImportRequest($"{baseUrl}&page={pageNumber}", HttpAccept.Json);
+                }
+            }
+            else
+            {
+                Logger.Trace($"Importing TMDb movies from: {baseUrl}");
+                yield return new NetImportRequest($"{baseUrl}", HttpAccept.Json);
+            }
+
+        }
+    }
+}

src/NzbDrone.Core/NetImport/TMDb/TMDbSettings.cs

@@ -0,0 +1,55 @@
+using System.Collections.Generic;
+using System.Globalization;
+using FluentValidation;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.NetImport.TMDb
+{
+
+    public class TMDbSettingsValidator : AbstractValidator<TMDbSettings>
+    {
+        public TMDbSettingsValidator()
+        {
+            RuleFor(c => c.Link).ValidRootUrl();
+            RuleFor(c => double.Parse(c.MinVoteAverage)).ExclusiveBetween(0, 10);
+        }
+    }
+
+    public class TMDbSettings : NetImportBaseSettings
+    {
+        private static readonly TMDbSettingsValidator Validator = new TMDbSettingsValidator();
+
+        public TMDbSettings()
+        {
+            Link = "https://api.themoviedb.org";
+            MinVoteAverage = "5.5";
+            // Language = (int) TMDbLanguageCodes.en;
+        }
+
+        [FieldDefinition(0, Label = "TMDb API URL", HelpText = "Link to to TMDb API URL, do not change unless you know what you are doing.")]
+        public new string Link { get; set; }
+
+        [FieldDefinition(1, Label = "List Type", Type = FieldType.Select, SelectOptions = typeof(TMDbListType), HelpText = "Type of list your seeking to import from")]
+        public int ListType { get; set; }
+
+        //[FieldDefinition(2, Label = "Language", Type = FieldType.Select, SelectOptions = typeof(TMDbLanguageCodes), HelpText = "Filter movies by Language")]
+        //public int Language { get; set; }
+
+        [FieldDefinition(2, Label = "Minimum Vote Average", HelpText = "Filter movies by rating (0.0-10.0)")]
+        public string MinVoteAverage { get; set; }
+
+        [FieldDefinition(3, Label = "Public List ID", HelpText = "Required for List")]
+        public string ListId { get; set; }
+
+        public new NzbDroneValidationResult Validate()
+        {
+            return new NzbDroneValidationResult(Validator.Validate(this));
+        }
+    }
+
+    
+
+
+
+}