Cleanup Search UI, Newznab Caps API

2026-04-23 22:45:06 -04:00 · 2020-10-20 13:46:58 -04:00
parent f290afa68c
commit 84cbfe870f
105 changed files with 562 additions and 791 deletions
@@ -0,0 +1,152 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using FluentValidation.Results;
+using NLog;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Indexers.Newznab;
+using NzbDrone.Core.ThingiProvider;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Indexers.Torznab
+{
+    public class Torznab : HttpIndexerBase<TorznabSettings>
+    {
+        private readonly INewznabCapabilitiesProvider _capabilitiesProvider;
+
+        public override string Name => "Torznab";
+
+        public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
+        public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
+        public override int PageSize => _capabilitiesProvider.GetCapabilities(Settings).DefaultPageSize;
+
+        public override IIndexerRequestGenerator GetRequestGenerator()
+        {
+            return new NewznabRequestGenerator(_capabilitiesProvider)
+            {
+                PageSize = PageSize,
+                Settings = Settings
+            };
+        }
+
+        public override IParseIndexerResponse GetParser()
+        {
+            return new TorznabRssParser();
+        }
+
+        public override IEnumerable<ProviderDefinition> DefaultDefinitions
+        {
+            get
+            {
+                yield return GetDefinition("Jackett", GetSettings("http://localhost:9117/api/v2.0/indexers/YOURINDEXER/results/torznab/"));
+                yield return GetDefinition("HD4Free.xyz", GetSettings("http://hd4free.xyz"));
+            }
+        }
+
+        public Torznab(INewznabCapabilitiesProvider capabilitiesProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
+            : base(httpClient, indexerStatusService, configService, logger)
+        {
+            _capabilitiesProvider = capabilitiesProvider;
+        }
+
+        private IndexerDefinition GetDefinition(string name, TorznabSettings settings)
+        {
+            return new IndexerDefinition
+            {
+                EnableRss = false,
+                EnableAutomaticSearch = false,
+                EnableInteractiveSearch = false,
+                Name = name,
+                Implementation = GetType().Name,
+                Settings = settings,
+                Protocol = DownloadProtocol.Usenet,
+                SupportsRss = SupportsRss,
+                SupportsSearch = SupportsSearch
+            };
+        }
+
+        private TorznabSettings GetSettings(string url, string apiPath = null, int[] categories = null)
+        {
+            var settings = new TorznabSettings { BaseUrl = url };
+
+            if (categories != null)
+            {
+                settings.Categories = categories;
+            }
+
+            if (apiPath.IsNotNullOrWhiteSpace())
+            {
+                settings.ApiPath = apiPath;
+            }
+
+            return settings;
+        }
+
+        protected override void Test(List<ValidationFailure> failures)
+        {
+            base.Test(failures);
+            if (failures.HasErrors())
+            {
+                return;
+            }
+
+            failures.AddIfNotNull(TestCapabilities());
+        }
+
+        protected static List<int> CategoryIds(List<NewznabCategory> categories)
+        {
+            var l = categories.Select(c => c.Id).ToList();
+
+            foreach (var category in categories)
+            {
+                if (category.Subcategories != null)
+                {
+                    l.AddRange(CategoryIds(category.Subcategories));
+                }
+            }
+
+            return l;
+        }
+
+        protected virtual ValidationFailure TestCapabilities()
+        {
+            try
+            {
+                var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
+
+                var notSupported = Settings.Categories.Except(CategoryIds(capabilities.Categories));
+
+                if (notSupported.Any())
+                {
+                    _logger.Warn($"{Definition.Name} does not support the following categories: {string.Join(", ", notSupported)}. You should probably remove them.");
+                    if (notSupported.Count() == Settings.Categories.Count())
+                    {
+                        return new ValidationFailure(string.Empty, $"This indexer does not support any of the selected categories! (You may need to turn on advanced settings to see them)");
+                    }
+                }
+
+                if (capabilities.SupportedSearchParameters != null && capabilities.SupportedSearchParameters.Contains("q"))
+                {
+                    return null;
+                }
+
+                if (capabilities.SupportedMovieSearchParameters != null &&
+                    new[] { "q", "imdbid" }.Any(v => capabilities.SupportedMovieSearchParameters.Contains(v)) &&
+                    new[] { "imdbtitle", "imdbyear" }.All(v => capabilities.SupportedMovieSearchParameters.Contains(v)))
+                {
+                    return null;
+                }
+
+                return new ValidationFailure(string.Empty, "This indexer does not support searching for movies :(. Tell your indexer staff to enable this or force add the indexer by disabling search, adding the indexer and then enabling it again.");
+            }
+            catch (Exception ex)
+            {
+                _logger.Warn(ex, "Unable to connect to indexer: " + ex.Message);
+
+                return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log for more details");
+            }
+        }
+    }
+}
@@ -0,0 +1,17 @@
+using NzbDrone.Common.Exceptions;
+
+namespace NzbDrone.Core.Indexers.Torznab
+{
+    public class TorznabException : NzbDroneException
+    {
+        public TorznabException(string message, params object[] args)
+            : base(message, args)
+        {
+        }
+
+        public TorznabException(string message)
+            : base(message)
+        {
+        }
+    }
+}
@@ -0,0 +1,228 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Xml.Linq;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.Indexers.Torznab
+{
+    public class TorznabRssParser : TorrentRssParser
+    {
+        public const string ns = "{http://torznab.com/schemas/2015/feed}";
+
+        public TorznabRssParser()
+        {
+            UseEnclosureUrl = true;
+        }
+
+        protected override bool PreProcess(IndexerResponse indexerResponse)
+        {
+            var xdoc = LoadXmlDocument(indexerResponse);
+            var error = xdoc.Descendants("error").FirstOrDefault();
+
+            if (error == null)
+            {
+                return true;
+            }
+
+            var code = Convert.ToInt32(error.Attribute("code").Value);
+            var errorMessage = error.Attribute("description").Value;
+
+            if (code >= 100 && code <= 199)
+            {
+                throw new ApiKeyException("Invalid API key");
+            }
+
+            if (!indexerResponse.Request.Url.FullUri.Contains("apikey=") && errorMessage == "Missing parameter")
+            {
+                throw new ApiKeyException("Indexer requires an API key");
+            }
+
+            if (errorMessage == "Request limit reached")
+            {
+                throw new RequestLimitReachedException("API limit reached");
+            }
+
+            throw new TorznabException("Torznab error detected: {0}", errorMessage);
+        }
+
+        protected override ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo)
+        {
+            var torrentInfo = base.ProcessItem(item, releaseInfo) as TorrentInfo;
+
+            if (torrentInfo != null)
+            {
+                if (GetImdbId(item) != null)
+                {
+                    torrentInfo.ImdbId = int.Parse(GetImdbId(item).Substring(2));
+                }
+
+                torrentInfo.IndexerFlags = GetFlags(item);
+            }
+
+            return torrentInfo;
+        }
+
+        protected override bool PostProcess(IndexerResponse indexerResponse, List<XElement> items, List<ReleaseInfo> releases)
+        {
+            var enclosureTypes = items.SelectMany(GetEnclosures).Select(v => v.Type).Distinct().ToArray();
+            if (enclosureTypes.Any() && enclosureTypes.Intersect(PreferredEnclosureMimeTypes).Empty())
+            {
+                if (enclosureTypes.Intersect(UsenetEnclosureMimeTypes).Any())
+                {
+                    _logger.Warn("Feed does not contain {0}, found {1}, did you intend to add a Newznab indexer?", TorrentEnclosureMimeType, enclosureTypes[0]);
+                }
+                else
+                {
+                    _logger.Warn("Feed does not contain {0}, found {1}.", TorrentEnclosureMimeType, enclosureTypes[0]);
+                }
+            }
+
+            return true;
+        }
+
+        protected override string GetInfoUrl(XElement item)
+        {
+            return ParseUrl(item.TryGetValue("comments").TrimEnd("#comments"));
+        }
+
+        protected override string GetCommentUrl(XElement item)
+        {
+            return ParseUrl(item.TryGetValue("comments"));
+        }
+
+        protected override long GetSize(XElement item)
+        {
+            long size;
+
+            var sizeString = TryGetTorznabAttribute(item, "size");
+            if (!sizeString.IsNullOrWhiteSpace() && long.TryParse(sizeString, out size))
+            {
+                return size;
+            }
+
+            size = GetEnclosureLength(item);
+
+            return size;
+        }
+
+        protected override DateTime GetPublishDate(XElement item)
+        {
+            return base.GetPublishDate(item);
+        }
+
+        protected override string GetDownloadUrl(XElement item)
+        {
+            var url = base.GetDownloadUrl(item);
+
+            if (!Uri.IsWellFormedUriString(url, UriKind.Absolute))
+            {
+                url = ParseUrl((string)item.Element("enclosure").Attribute("url"));
+            }
+
+            return url;
+        }
+
+        protected virtual string GetImdbId(XElement item)
+        {
+            var imdbIdString = TryGetTorznabAttribute(item, "imdbid");
+            return !imdbIdString.IsNullOrWhiteSpace() ? imdbIdString.Substring(2) : null;
+        }
+
+        protected override string GetInfoHash(XElement item)
+        {
+            return TryGetTorznabAttribute(item, "infohash");
+        }
+
+        protected override string GetMagnetUrl(XElement item)
+        {
+            return TryGetTorznabAttribute(item, "magneturl");
+        }
+
+        protected override int? GetSeeders(XElement item)
+        {
+            var seeders = TryGetTorznabAttribute(item, "seeders");
+
+            if (seeders.IsNotNullOrWhiteSpace())
+            {
+                return int.Parse(seeders);
+            }
+
+            return base.GetSeeders(item);
+        }
+
+        protected override int? GetPeers(XElement item)
+        {
+            var peers = TryGetTorznabAttribute(item, "peers");
+
+            if (peers.IsNotNullOrWhiteSpace())
+            {
+                return int.Parse(peers);
+            }
+
+            var seeders = TryGetTorznabAttribute(item, "seeders");
+            var leechers = TryGetTorznabAttribute(item, "leechers");
+
+            if (seeders.IsNotNullOrWhiteSpace() && leechers.IsNotNullOrWhiteSpace())
+            {
+                return int.Parse(seeders) + int.Parse(leechers);
+            }
+
+            return base.GetPeers(item);
+        }
+
+        protected IndexerFlags GetFlags(XElement item)
+        {
+            IndexerFlags flags = 0;
+
+            var downloadFactor = TryGetFloatTorznabAttribute(item, "downloadvolumefactor", 1);
+
+            var uploadFactor = TryGetFloatTorznabAttribute(item, "uploadvolumefactor", 1);
+
+            if (uploadFactor == 2)
+            {
+                flags |= IndexerFlags.G_DoubleUpload;
+            }
+
+            if (downloadFactor == 0.5)
+            {
+                flags |= IndexerFlags.G_Halfleech;
+            }
+
+            if (downloadFactor == 0.0)
+            {
+                flags |= IndexerFlags.G_Freeleech;
+            }
+
+            return flags;
+        }
+
+        protected string TryGetTorznabAttribute(XElement item, string key, string defaultValue = "")
+        {
+            var attr = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.CurrentCultureIgnoreCase));
+
+            if (attr != null)
+            {
+                return attr.Attribute("value").Value;
+            }
+
+            return defaultValue;
+        }
+
+        protected float TryGetFloatTorznabAttribute(XElement item, string key, float defaultValue = 0)
+        {
+            var attr = TryGetTorznabAttribute(item, key, defaultValue.ToString());
+
+            float result = 0;
+
+            if (float.TryParse(attr, out result))
+            {
+                return result;
+            }
+
+            return defaultValue;
+        }
+    }
+}
@@ -0,0 +1,71 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using FluentValidation;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Indexers.Newznab;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Indexers.Torznab
+{
+    public class TorznabSettingsValidator : AbstractValidator<TorznabSettings>
+    {
+        private static readonly string[] ApiKeyWhiteList =
+        {
+            "hd4free.xyz",
+        };
+
+        private static bool ShouldHaveApiKey(TorznabSettings settings)
+        {
+            if (settings.BaseUrl == null)
+            {
+                return false;
+            }
+
+            return ApiKeyWhiteList.Any(c => settings.BaseUrl.ToLowerInvariant().Contains(c));
+        }
+
+        private static readonly Regex AdditionalParametersRegex = new Regex(@"(&.+?\=.+?)+", RegexOptions.Compiled);
+
+        public TorznabSettingsValidator()
+        {
+            RuleFor(c => c).Custom((c, context) =>
+            {
+                if (c.Categories.Empty())
+                {
+                    context.AddFailure("'Categories' must be provided");
+                }
+            });
+
+            RuleFor(c => c.BaseUrl).ValidRootUrl();
+            RuleFor(c => c.ApiPath).ValidUrlBase("/api");
+            RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey);
+            RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex)
+                                                .When(c => !c.AdditionalParameters.IsNullOrWhiteSpace());
+        }
+    }
+
+    public class TorznabSettings : NewznabSettings, ITorrentIndexerSettings
+    {
+        private static readonly TorznabSettingsValidator Validator = new TorznabSettingsValidator();
+
+        public TorznabSettings()
+        {
+            MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
+            RequiredFlags = new List<int>();
+        }
+
+        [FieldDefinition(8, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
+        public int MinimumSeeders { get; set; }
+
+        [FieldDefinition(9, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", HelpLink = "https://github.com/Prowlarr/Prowlarr/wiki/Indexer-Flags#1-required-flags", Advanced = true)]
+        public IEnumerable<int> RequiredFlags { get; set; }
+
+        public override NzbDroneValidationResult Validate()
+        {
+            return new NzbDroneValidationResult(Validator.Validate(this));
+        }
+    }
+}