Big Abstraction for IMDBWatchlist -> RSSImport (With a test)

2026-04-21 22:05:43 -04:00 · 2017-01-21 20:29:31 +01:00
parent 734a36de06
commit a98b69859c
17 changed files with 2104 additions and 118 deletions
@@ -0,0 +1,40 @@
+using System.Collections.Generic;
+using System.Xml.Serialization;
+
+namespace NzbDrone.Core.NetImport.RSSImport
+{
+    class IMDbWatchListAPI
+    {
+        [XmlRoot(ElementName = "item")]
+        public class Movie
+        {
+            [XmlElement(ElementName = "pubDate")]
+            public string PublishDate { get; set; }
+            [XmlElement(ElementName = "title")]
+            public string Title { get; set; }
+            [XmlElement(ElementName = "link")]
+            public string Link { get; set; }
+            [XmlElement(ElementName = "guid")]
+            public string Guid { get; set; }
+            [XmlElement(ElementName = "description")]
+            public string Description { get; set; }
+        }
+
+        [XmlRoot(ElementName = "channel")]
+        public class Channel
+        {
+            [XmlElement(ElementName = "title")]
+            public string Title { get; set; }
+            [XmlElement(ElementName = "link")]
+            public string Link { get; set; }
+            [XmlElement(ElementName = "description")]
+            public string Description { get; set; }
+            [XmlElement(ElementName = "pubDate")]
+            public string PublishDate { get; set; }
+            [XmlElement(ElementName = "lastBuildDate")]
+            public string LastBuildDate { get; set; }
+            [XmlElement(ElementName = "item")]
+            public List<Movie> Movie { get; set; }
+        }
+    }
+}
@@ -0,0 +1,79 @@
+using System;
+using System.Linq;
+using System.Net;
+using System.Xml.Linq;
+using FluentValidation.Results;
+using NLog;
+using NzbDrone.Core.Exceptions;
+using RestSharp;
+using NzbDrone.Core.Rest;
+
+namespace NzbDrone.Core.NetImport.RSSImport
+{
+    public interface IIMDbWatchListProxy
+    {
+        void ImportMovies(string url);
+        ValidationFailure Test(RSSImportSettings settings);
+    }
+
+    public class IMDbWatchListProxy : IIMDbWatchListProxy
+    {
+        private readonly Logger _logger;
+        private const string URL = "http://rss.imdb.com";
+
+        public IMDbWatchListProxy(Logger logger)
+        {
+            _logger = logger;
+        }
+
+        public void ImportMovies(string id)
+        {
+            var client = RestClientFactory.BuildClient(URL);
+            var request = new RestRequest("/list/{id}", Method.GET);
+            request.RequestFormat = DataFormat.Xml;
+            request.AddParameter("id", id, ParameterType.UrlSegment);
+
+            var response = client.ExecuteAndValidate(request);
+            ValidateResponse(response);
+        }
+
+        private void Verify(string id)
+        {
+            var client = RestClientFactory.BuildClient(URL);
+            var request = new RestRequest("/list/{id}", Method.GET);
+            request.RequestFormat = DataFormat.Xml;
+            request.AddParameter("id", id, ParameterType.UrlSegment);
+
+            var response = client.ExecuteAndValidate(request);
+            ValidateResponse(response);
+        }
+
+        private void ValidateResponse(IRestResponse response)
+        {
+            var xDoc = XDocument.Parse(response.Content);
+            var nma = xDoc.Descendants("nma").Single();
+            var error = nma.Descendants("error").SingleOrDefault();
+
+            if (error != null)
+            {
+                ((HttpStatusCode)Convert.ToInt32(error.Attribute("code").Value)).VerifyStatusCode(error.Value);
+            }
+        }
+
+        public ValidationFailure Test(RSSImportSettings settings)
+        {
+            try
+            {
+                Verify(settings.Link);
+                ImportMovies(settings.Link);
+            }
+            catch (Exception ex)
+            {
+                _logger.Error(ex, "Unable to import movies: " + ex.Message);
+                return new ValidationFailure("IMDbWatchListId", "Unable to import movies");
+            }
+
+            return null;
+        }
+    }
+}
@@ -0,0 +1,52 @@
+using System;
+using System.Collections.Generic;
+using System.Xml.Serialization;
+using FluentValidation.Results;
+using NLog;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.PassThePopcorn;
+using NzbDrone.Core.Parser;
+using NzbDrone.Core.ThingiProvider;
+using NzbDrone.Core.Tv;
+
+namespace NzbDrone.Core.NetImport.RSSImport
+{
+    public class RSSImport : HttpNetImportBase<RSSImportSettings>
+    {
+        public override string Name => "RSSList";
+        public override bool Enabled => true;
+
+        public RSSImport(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
+            : base(httpClient, configService, parsingService, logger)
+        { }
+
+        public new virtual IEnumerable<ProviderDefinition> DefaultDefinitions
+        {
+            get
+            {
+                var config = (RSSImportSettings)new RSSImportSettings();
+                config.Link = "https://rss.imdb.com/list/YOURLISTID";
+
+                yield return new NetImportDefinition
+                {
+                    Name = GetType().Name,
+                    Enabled = config.Validate().IsValid && Enabled,
+                    Implementation = GetType().Name,
+                    Settings = config
+                };
+            }
+        }
+
+        public override INetImportRequestGenerator GetRequestGenerator()
+        {
+            return new RSSImportRequestGenerator() { Settings = Settings };
+        }
+
+        public override IParseNetImportResponse GetParser()
+        {
+            return new RSSImportParser(Settings);
+        }
+    }
+}
@@ -0,0 +1,236 @@
+using Newtonsoft.Json;
+using NzbDrone.Core.NetImport.Exceptions;
+using NzbDrone.Core.Tv;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Xml;
+using System.Xml.Linq;
+using NLog;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.NetImport.RSSImport
+{
+    public class RSSImportParser : IParseNetImportResponse
+    {
+        private readonly RSSImportSettings _settings;
+        private NetImportResponse _importResponse;
+        private readonly Logger _logger;
+
+        private static readonly Regex ReplaceEntities = new Regex("&[a-z]+;", RegexOptions.Compiled | RegexOptions.IgnoreCase);
+
+        public RSSImportParser(RSSImportSettings settings)
+        {
+            _settings = settings;
+        }
+
+        public virtual IList<Movie> ParseResponse(NetImportResponse importResponse)
+        {
+            _importResponse = importResponse;
+
+            var movies = new List<Movie>();
+
+            if (!PreProcess(importResponse))
+            {
+                return movies;
+            }
+
+            var document = LoadXmlDocument(importResponse);
+            var items = GetItems(document);
+
+            foreach (var item in items)
+            {
+                try
+                {
+                    var reportInfo = ProcessItem(item);
+
+                    movies.AddIfNotNull(reportInfo);
+                }
+                catch (Exception itemEx)
+                {
+                    //itemEx.Data.Add("Item", item.Title());
+                    _logger.Error(itemEx, "An error occurred while processing feed item from " + importResponse.Request.Url);
+                }
+            }
+
+            return movies;
+        }
+
+        protected virtual XDocument LoadXmlDocument(NetImportResponse indexerResponse)
+        {
+            try
+            {
+                var content = indexerResponse.Content;
+                content = ReplaceEntities.Replace(content, ReplaceEntity);
+
+                using (var xmlTextReader = XmlReader.Create(new StringReader(content), new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore, IgnoreComments = true }))
+                {
+                    return XDocument.Load(xmlTextReader);
+                }
+            }
+            catch (XmlException ex)
+            {
+                var contentSample = indexerResponse.Content.Substring(0, Math.Min(indexerResponse.Content.Length, 512));
+                _logger.Debug("Truncated response content (originally {0} characters): {1}", indexerResponse.Content.Length, contentSample);
+
+                ex.Data.Add("ContentLength", indexerResponse.Content.Length);
+                ex.Data.Add("ContentSample", contentSample);
+
+                throw;
+            }
+        }
+
+        protected virtual string ReplaceEntity(Match match)
+        {
+            try
+            {
+                var character = WebUtility.HtmlDecode(match.Value);
+                return string.Concat("&#", (int)character[0], ";");
+            }
+            catch
+            {
+                return match.Value;
+            }
+        }
+
+        protected virtual Movie CreateNewMovie()
+        {
+            return new Movie();
+        }
+
+        protected virtual bool PreProcess(NetImportResponse indexerResponse)
+        {
+            if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
+            {
+                throw new NetImportException(indexerResponse, "Indexer API call resulted in an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
+            }
+
+            if (indexerResponse.HttpResponse.Headers.ContentType != null && indexerResponse.HttpResponse.Headers.ContentType.Contains("text/html") &&
+                indexerResponse.HttpRequest.Headers.Accept != null && !indexerResponse.HttpRequest.Headers.Accept.Contains("text/html"))
+            {
+                throw new NetImportException(indexerResponse, "Indexer responded with html content. Site is likely blocked or unavailable.");
+            }
+
+            return true;
+        }
+
+        protected Movie ProcessItem(XElement item)
+        {
+            var releaseInfo = CreateNewMovie();
+
+            releaseInfo = ProcessItem(item, releaseInfo);
+
+            //_logger.Trace("Parsed: {0}", releaseInfo.Title);
+
+            return PostProcess(item, releaseInfo);
+        }
+
+        protected virtual Movie ProcessItem(XElement item, Movie releaseInfo)
+        {
+            var result = Parser.Parser.ParseMovieTitle(GetTitle(item));
+
+            releaseInfo.Title = GetTitle(item);
+
+            if (result != null)
+            {
+                releaseInfo.Title = result.MovieTitle;
+                releaseInfo.Year = result.Year;
+                releaseInfo.ImdbId = result.ImdbId;
+            }
+
+            try
+            {
+                if (releaseInfo.ImdbId.IsNullOrWhiteSpace())
+                {
+                    releaseInfo.ImdbId = GetImdbId(item);
+                }
+
+            }
+            catch (Exception)
+            {
+                _logger.Debug("Unable to extract Imdb Id :(.");
+            }
+
+            return releaseInfo;
+        }
+
+        protected virtual Movie PostProcess(XElement item, Movie releaseInfo)
+        {
+            return releaseInfo;
+        }
+
+        protected virtual string GetTitle(XElement item)
+        {
+            return item.TryGetValue("title", "Unknown");
+        }
+
+        protected virtual DateTime GetPublishDate(XElement item)
+        {
+            var dateString = item.TryGetValue("pubDate");
+
+            if (dateString.IsNullOrWhiteSpace())
+            {
+                throw new UnsupportedFeedException("Rss feed must have a pubDate element with a valid publish date.");
+            }
+
+            return XElementExtensions.ParseDate(dateString);
+        }
+
+        protected virtual string GetImdbId(XElement item)
+        {
+            var url = item.TryGetValue("link");
+            if (url.IsNullOrWhiteSpace())
+            {
+                return "";
+            }
+            return Parser.Parser.ParseImdbId(url);
+        }
+
+        protected IEnumerable<XElement> GetItems(XDocument document)
+        {
+            var root = document.Root;
+
+            if (root == null)
+            {
+                return Enumerable.Empty<XElement>();
+            }
+
+            var channel = root.Element("channel");
+
+            if (channel == null)
+            {
+                return Enumerable.Empty<XElement>();
+            }
+
+            return channel.Elements("item");
+        }
+
+        protected virtual string ParseUrl(string value)
+        {
+            if (value.IsNullOrWhiteSpace())
+            {
+                return null;
+            }
+
+            try
+            {
+                var url = _importResponse.HttpRequest.Url + new HttpUri(value);
+
+                return url.FullUri;
+            }
+            catch (Exception ex)
+            {
+                _logger.Debug(ex, string.Format("Failed to parse Url {0}, ignoring.", value));
+                return null;
+            }
+        }
+    }
+}
@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using NzbDrone.Common.Http;
+using NzbDrone.Common.Serializer;
+using NzbDrone.Core.IndexerSearch.Definitions;
+
+namespace NzbDrone.Core.NetImport.RSSImport
+{
+    public class RSSImportRequestGenerator : INetImportRequestGenerator
+    {
+        public RSSImportSettings Settings { get; set; }
+
+        public virtual NetImportPageableRequestChain GetMovies()
+        {
+            var pageableRequests = new NetImportPageableRequestChain();
+
+            pageableRequests.Add(GetMovies(null));
+
+            return pageableRequests;
+        }
+
+        public NetImportPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
+        {
+            return new NetImportPageableRequestChain();
+        }
+
+        private IEnumerable<NetImportRequest> GetMovies(string searchParameters)
+        {
+            var request = new NetImportRequest($"{Settings.Link.Trim()}", HttpAccept.Rss);
+            yield return request;
+        }
+    }
+}
@@ -0,0 +1,23 @@
+using FluentValidation;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Profiles;
+using NzbDrone.Core.ThingiProvider;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.NetImport.RSSImport
+{
+
+    public class RSSImportSettings : NetImportBaseSettings
+    {
+        //private const string helpLink = "https://imdb.com";
+
+        public RSSImportSettings()
+        {
+            Link = "http://rss.yoursite.com";
+            ProfileId = 1;
+        }
+
+        [FieldDefinition(0, Label = "RSS Link", HelpText = "Link to the rss feed of movies.")]
+        public new string Link { get; set; }
+    }
+}