Cleanup Search UI, Newznab Caps API

2026-04-25 22:59:10 -04:00 · 2020-10-20 13:46:58 -04:00
parent f290afa68c
commit 84cbfe870f
105 changed files with 562 additions and 791 deletions
@@ -0,0 +1,34 @@
+using NLog;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+
+namespace NzbDrone.Core.Indexers.TorrentRss
+{
+    public class TorrentRssIndexer : HttpIndexerBase<TorrentRssIndexerSettings>
+    {
+        public override string Name => "Torrent RSS Feed";
+
+        public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
+        public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
+        public override bool SupportsSearch => false;
+        public override int PageSize => 0;
+
+        private readonly ITorrentRssParserFactory _torrentRssParserFactory;
+
+        public TorrentRssIndexer(ITorrentRssParserFactory torrentRssParserFactory, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
+            : base(httpClient, indexerStatusService, configService, logger)
+        {
+            _torrentRssParserFactory = torrentRssParserFactory;
+        }
+
+        public override IIndexerRequestGenerator GetRequestGenerator()
+        {
+            return new TorrentRssIndexerRequestGenerator { Settings = Settings };
+        }
+
+        public override IParseIndexerResponse GetParser()
+        {
+            return _torrentRssParserFactory.GetParser(Settings);
+        }
+    }
+}
@@ -0,0 +1,12 @@
+namespace NzbDrone.Core.Indexers.TorrentRss
+{
+    public class TorrentRssIndexerParserSettings
+    {
+        public bool UseEZTVFormat { get; set; }
+        public bool ParseSeedersInDescription { get; set; }
+        public bool UseEnclosureUrl { get; set; }
+        public bool UseEnclosureLength { get; set; }
+        public bool ParseSizeInDescription { get; set; }
+        public string SizeElementName { get; set; }
+    }
+}
@@ -0,0 +1,45 @@
+using System;
+using System.Collections.Generic;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.IndexerSearch.Definitions;
+
+namespace NzbDrone.Core.Indexers.TorrentRss
+{
+    public class TorrentRssIndexerRequestGenerator : IIndexerRequestGenerator
+    {
+        public TorrentRssIndexerSettings Settings { get; set; }
+
+        public virtual IndexerPageableRequestChain GetRecentRequests()
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            pageableRequests.Add(GetRssRequests(null));
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
+        {
+            return new IndexerPageableRequestChain();
+        }
+
+        private IEnumerable<IndexerRequest> GetRssRequests(string searchParameters)
+        {
+            var request = new IndexerRequest(Settings.BaseUrl.Trim().TrimEnd('/'), HttpAccept.Rss);
+
+            if (Settings.Cookie.IsNotNullOrWhiteSpace())
+            {
+                foreach (var cookie in HttpHeader.ParseCookies(Settings.Cookie))
+                {
+                    request.HttpRequest.Cookies[cookie.Key] = cookie.Value;
+                }
+            }
+
+            yield return request;
+        }
+
+        public Func<IDictionary<string, string>> GetCookies { get; set; }
+        public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
+    }
+}
@@ -0,0 +1,54 @@
+using System.Collections.Generic;
+using FluentValidation;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Languages;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Indexers.TorrentRss
+{
+    public class TorrentRssIndexerSettingsValidator : AbstractValidator<TorrentRssIndexerSettings>
+    {
+        public TorrentRssIndexerSettingsValidator()
+        {
+            RuleFor(c => c.BaseUrl).ValidRootUrl();
+        }
+    }
+
+    public class TorrentRssIndexerSettings : ITorrentIndexerSettings
+    {
+        private static readonly TorrentRssIndexerSettingsValidator Validator = new TorrentRssIndexerSettingsValidator();
+
+        public TorrentRssIndexerSettings()
+        {
+            BaseUrl = string.Empty;
+            AllowZeroSize = false;
+            MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
+            MultiLanguages = new List<int>();
+            RequiredFlags = new List<int>();
+        }
+
+        [FieldDefinition(0, Label = "Full RSS Feed URL")]
+        public string BaseUrl { get; set; }
+
+        [FieldDefinition(1, Label = "Cookie", HelpText = "If you site requires a login cookie to access the rss, you'll have to retrieve it via a browser.")]
+        public string Cookie { get; set; }
+
+        [FieldDefinition(2, Type = FieldType.Checkbox, Label = "Allow Zero Size", HelpText = "Enabling this will allow you to use feeds that don't specify release size, but be careful, size related checks will not be performed.")]
+        public bool AllowZeroSize { get; set; }
+
+        [FieldDefinition(3, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
+        public IEnumerable<int> MultiLanguages { get; set; }
+
+        [FieldDefinition(4, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
+        public int MinimumSeeders { get; set; }
+
+        [FieldDefinition(6, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", HelpLink = "https://github.com/Prowlarr/Prowlarr/wiki/Indexer-Flags#1-required-flags", Advanced = true)]
+        public IEnumerable<int> RequiredFlags { get; set; }
+
+        public NzbDroneValidationResult Validate()
+        {
+            return new NzbDroneValidationResult(Validator.Validate(this));
+        }
+    }
+}
@@ -0,0 +1,65 @@
+using System;
+using NLog;
+using NzbDrone.Common.Cache;
+using NzbDrone.Common.Serializer;
+using NzbDrone.Core.Indexers.Exceptions;
+
+namespace NzbDrone.Core.Indexers.TorrentRss
+{
+    public interface ITorrentRssParserFactory
+    {
+        TorrentRssParser GetParser(TorrentRssIndexerSettings settings);
+    }
+
+    public class TorrentRssParserFactory : ITorrentRssParserFactory
+    {
+        protected readonly Logger _logger;
+
+        private readonly ICached<TorrentRssIndexerParserSettings> _settingsCache;
+
+        private readonly ITorrentRssSettingsDetector _torrentRssSettingsDetector;
+
+        public TorrentRssParserFactory(ICacheManager cacheManager, ITorrentRssSettingsDetector torrentRssSettingsDetector, Logger logger)
+        {
+            _settingsCache = cacheManager.GetCache<TorrentRssIndexerParserSettings>(GetType());
+            _torrentRssSettingsDetector = torrentRssSettingsDetector;
+            _logger = logger;
+        }
+
+        public TorrentRssParser GetParser(TorrentRssIndexerSettings indexerSettings)
+        {
+            var key = indexerSettings.ToJson();
+            var parserSettings = _settingsCache.Get(key, () => DetectParserSettings(indexerSettings), TimeSpan.FromDays(7));
+
+            if (parserSettings.UseEZTVFormat)
+            {
+                return new EzrssTorrentRssParser();
+            }
+            else
+            {
+                return new TorrentRssParser
+                {
+                    UseGuidInfoUrl = false,
+                    ParseSeedersInDescription = parserSettings.ParseSeedersInDescription,
+
+                    UseEnclosureUrl = parserSettings.UseEnclosureUrl,
+                    UseEnclosureLength = parserSettings.UseEnclosureLength,
+                    ParseSizeInDescription = parserSettings.ParseSizeInDescription,
+                    SizeElementName = parserSettings.SizeElementName
+                };
+            }
+        }
+
+        private TorrentRssIndexerParserSettings DetectParserSettings(TorrentRssIndexerSettings indexerSettings)
+        {
+            var settings = _torrentRssSettingsDetector.Detect(indexerSettings);
+
+            if (settings == null)
+            {
+                throw new UnsupportedFeedException("Could not parse feed from {0}", indexerSettings.BaseUrl);
+            }
+
+            return settings;
+        }
+    }
+}
@@ -0,0 +1,307 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Xml;
+using System.Xml.Linq;
+using NLog;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.Indexers.TorrentRss
+{
+    public interface ITorrentRssSettingsDetector
+    {
+        TorrentRssIndexerParserSettings Detect(TorrentRssIndexerSettings settings);
+    }
+
+    public class TorrentRssSettingsDetector : ITorrentRssSettingsDetector
+    {
+        private const long ValidSizeThreshold = 2 * 1024 * 1024;
+
+        protected readonly Logger _logger;
+
+        private readonly IHttpClient _httpClient;
+
+        public TorrentRssSettingsDetector(IHttpClient httpClient, Logger logger)
+        {
+            _httpClient = httpClient;
+            _logger = logger;
+        }
+
+        /// <summary>
+        /// Detect settings for Parser, based on URL
+        /// </summary>
+        /// <param name="settings">Indexer Settings to use for Parser</param>
+        /// <returns>Parsed Settings or <c>null</c></returns>
+        public TorrentRssIndexerParserSettings Detect(TorrentRssIndexerSettings indexerSettings)
+        {
+            _logger.Debug("Evaluating TorrentRss feed '{0}'", indexerSettings.BaseUrl);
+
+            var requestGenerator = new TorrentRssIndexerRequestGenerator { Settings = indexerSettings };
+            var request = requestGenerator.GetRecentRequests().GetAllTiers().First().First();
+
+            HttpResponse httpResponse = null;
+            try
+            {
+                httpResponse = _httpClient.Execute(request.HttpRequest);
+            }
+            catch (Exception ex)
+            {
+                _logger.Warn(ex, string.Format("Unable to connect to indexer {0}: {1}", request.Url, ex.Message));
+                return null;
+            }
+
+            var indexerResponse = new IndexerResponse(request, httpResponse);
+            return GetParserSettings(indexerResponse, indexerSettings);
+        }
+
+        private TorrentRssIndexerParserSettings GetParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)
+        {
+            var settings = GetEzrssParserSettings(response, indexerSettings);
+            if (settings != null)
+            {
+                return settings;
+            }
+
+            settings = GetGenericTorrentRssParserSettings(response, indexerSettings);
+            if (settings != null)
+            {
+                return settings;
+            }
+
+            return null;
+        }
+
+        private TorrentRssIndexerParserSettings GetEzrssParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)
+        {
+            if (!IsEZTVFeed(response))
+            {
+                return null;
+            }
+
+            _logger.Trace("Feed has Ezrss schema");
+
+            var parser = new EzrssTorrentRssParser();
+            var releases = ParseResponse(parser, response);
+
+            try
+            {
+                ValidateReleases(releases, indexerSettings);
+                ValidateReleaseSize(releases, indexerSettings);
+
+                _logger.Debug("Feed was parseable by Ezrss Parser");
+                return new TorrentRssIndexerParserSettings
+                {
+                    UseEZTVFormat = true
+                };
+            }
+            catch (Exception ex)
+            {
+                _logger.Trace(ex, "Feed wasn't parsable by Ezrss Parser");
+                return null;
+            }
+        }
+
+        private TorrentRssIndexerParserSettings GetGenericTorrentRssParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)
+        {
+            var parser = new TorrentRssParser
+            {
+                UseEnclosureUrl = true,
+                UseEnclosureLength = true,
+                ParseSeedersInDescription = true
+            };
+
+            var item = parser.GetItems(response).FirstOrDefault();
+            if (item == null)
+            {
+                throw new UnsupportedFeedException("Empty feed, cannot check if feed is parsable.");
+            }
+
+            var settings = new TorrentRssIndexerParserSettings()
+            {
+                UseEnclosureUrl = true,
+                UseEnclosureLength = true,
+                ParseSeedersInDescription = true
+            };
+
+            if (item.Element("enclosure") == null)
+            {
+                parser.UseEnclosureUrl = settings.UseEnclosureUrl = false;
+            }
+
+            var releases = ParseResponse(parser, response);
+            ValidateReleases(releases, indexerSettings);
+
+            if (!releases.Any(v => v.Seeders.HasValue))
+            {
+                _logger.Trace("Feed doesn't have Seeders in Description, disabling option.");
+                parser.ParseSeedersInDescription = settings.ParseSeedersInDescription = false;
+            }
+
+            if (!releases.Any(r => r.Size < ValidSizeThreshold))
+            {
+                _logger.Trace("Feed has valid size in enclosure.");
+                return settings;
+            }
+
+            parser.UseEnclosureLength = settings.UseEnclosureLength = false;
+
+            foreach (var sizeElementName in new[] { "size", "Size" })
+            {
+                parser.SizeElementName = settings.SizeElementName = sizeElementName;
+
+                releases = ParseResponse(parser, response);
+                ValidateReleases(releases, indexerSettings);
+
+                if (!releases.Any(r => r.Size < ValidSizeThreshold))
+                {
+                    _logger.Trace("Feed has valid size in Size element.");
+                    return settings;
+                }
+            }
+
+            parser.SizeElementName = settings.SizeElementName = null;
+            parser.ParseSizeInDescription = settings.ParseSizeInDescription = true;
+
+            releases = ParseResponse(parser, response);
+            ValidateReleases(releases, indexerSettings);
+
+            if (releases.Count(r => r.Size >= ValidSizeThreshold) > releases.Length / 2)
+            {
+                if (releases.Any(r => r.Size < ValidSizeThreshold))
+                {
+                    _logger.Debug("Feed {0} contains very small releases.", response.Request.Url);
+                }
+
+                _logger.Trace("Feed has valid size in description.");
+                return settings;
+            }
+
+            parser.ParseSizeInDescription = settings.ParseSizeInDescription = false;
+
+            _logger.Debug("Feed doesn't have release size.");
+
+            releases = ParseResponse(parser, response);
+            ValidateReleases(releases, indexerSettings);
+            ValidateReleaseSize(releases, indexerSettings);
+
+            return settings;
+        }
+
+        private bool IsEZTVFeed(IndexerResponse response)
+        {
+            var content = XmlCleaner.ReplaceEntities(response.Content);
+            content = XmlCleaner.ReplaceUnicode(content);
+
+            using (var xmlTextReader = XmlReader.Create(new StringReader(content), new XmlReaderSettings { DtdProcessing = DtdProcessing.Parse, ValidationType = ValidationType.None, IgnoreComments = true, XmlResolver = null }))
+            {
+                var document = XDocument.Load(xmlTextReader);
+
+                // Check Namespace
+                if (document.Root == null)
+                {
+                    throw new InvalidDataException("Could not parse IndexerResponse into XML.");
+                }
+
+                var ns = document.Root.GetNamespaceOfPrefix("torrent");
+                if (ns == "http://xmlns.ezrss.it/0.1/")
+                {
+                    _logger.Trace("Identified feed as EZTV compatible by EZTV Namespace");
+                    return true;
+                }
+
+                // Check DTD in DocType
+                if (document.DocumentType != null && document.DocumentType.SystemId == "http://xmlns.ezrss.it/0.1/dtd/")
+                {
+                    _logger.Trace("Identified feed as EZTV compatible by EZTV DTD");
+                    return true;
+                }
+
+                // Check namespaces
+                if (document.Descendants().Any(v => v.GetDefaultNamespace().NamespaceName == "http://xmlns.ezrss.it/0.1/"))
+                {
+                    _logger.Trace("Identified feed as EZTV compatible by EZTV Namespace");
+                    return true;
+                }
+
+                return false;
+            }
+        }
+
+        private TorrentInfo[] ParseResponse(IParseIndexerResponse parser, IndexerResponse response)
+        {
+            try
+            {
+                var releases = parser.ParseResponse(response).Cast<TorrentInfo>().ToArray();
+                return releases;
+            }
+            catch (Exception ex)
+            {
+                _logger.Debug(ex, "Unable to parse indexer feed: " + ex.Message);
+                throw new UnsupportedFeedException("Unable to parse indexer: " + ex.Message);
+            }
+        }
+
+        private void ValidateReleases(TorrentInfo[] releases, TorrentRssIndexerSettings indexerSettings)
+        {
+            if (releases == null || releases.Empty())
+            {
+                throw new UnsupportedFeedException("Empty feed, cannot check if feed is parsable.");
+            }
+
+            var torrentInfo = releases.First();
+
+            _logger.Trace("TorrentInfo: \n{0}", torrentInfo.ToString("L"));
+
+            if (releases.Any(r => r.Title.IsNullOrWhiteSpace()))
+            {
+                throw new UnsupportedFeedException("Feed contains releases without title.");
+            }
+
+            if (releases.Any(r => !IsValidDownloadUrl(r.DownloadUrl)))
+            {
+                throw new UnsupportedFeedException("Failed to find a valid download url in the feed.");
+            }
+
+            var total = releases.Where(v => v.Guid != null).Select(v => v.Guid).ToArray();
+            var distinct = total.Distinct().ToArray();
+
+            if (distinct.Length != total.Length)
+            {
+                throw new UnsupportedFeedException("Feed contains releases with same guid, rejecting malformed rss feed.");
+            }
+        }
+
+        private void ValidateReleaseSize(TorrentInfo[] releases, TorrentRssIndexerSettings indexerSettings)
+        {
+            if (!indexerSettings.AllowZeroSize && releases.Any(r => r.Size == 0))
+            {
+                throw new UnsupportedFeedException("Feed doesn't contain the release content size.");
+            }
+
+            if (releases.Any(r => r.Size != 0 && r.Size < ValidSizeThreshold))
+            {
+                throw new UnsupportedFeedException("Size of one more releases lower than {0}, feed must contain release content size.", ValidSizeThreshold.SizeSuffix());
+            }
+        }
+
+        private static bool IsValidDownloadUrl(string url)
+        {
+            if (url.IsNullOrWhiteSpace())
+            {
+                return false;
+            }
+
+            if (url.StartsWith("magnet:") ||
+                url.StartsWith("http:") ||
+                url.StartsWith("https:"))
+            {
+                return true;
+            }
+
+            return false;
+        }
+    }
+}