New: Add TorrentRssIndexer

Co-authored-by: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com>
2026-04-25 22:59:10 -04:00 · 2023-05-20 01:36:25 +03:00
parent 596d3297da
commit 455b76c45c
9 changed files with 682 additions and 29 deletions
@@ -0,0 +1,92 @@
+using System.Collections.Generic;
+using NLog;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.ThingiProvider;
+
+namespace NzbDrone.Core.Indexers.Definitions.TorrentRss
+{
+    public class TorrentRssIndexer : TorrentIndexerBase<TorrentRssIndexerSettings>
+    {
+        private readonly ITorrentRssParserFactory _torrentRssParserFactory;
+
+        public override string Name => "Torrent RSS Feed";
+        public override string[] IndexerUrls => new[] { "" };
+        public override string Description => "Generic RSS Feed containing torrents";
+        public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
+        public override IndexerPrivacy Privacy => IndexerPrivacy.Public;
+        public override int PageSize => 0;
+        public override IndexerCapabilities Capabilities => SetCapabilities();
+
+        public TorrentRssIndexer(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger, ITorrentRssParserFactory torrentRssParserFactory)
+            : base(httpClient, eventAggregator, indexerStatusService, configService, logger)
+        {
+            _torrentRssParserFactory = torrentRssParserFactory;
+        }
+
+        public override IIndexerRequestGenerator GetRequestGenerator()
+        {
+            return new TorrentRssIndexerRequestGenerator { Settings = Settings };
+        }
+
+        public override IParseIndexerResponse GetParser()
+        {
+            return _torrentRssParserFactory.GetParser(Settings);
+        }
+
+        public override IEnumerable<ProviderDefinition> DefaultDefinitions
+        {
+            get
+            {
+                yield return GetDefinition("showRSS", "showRSS is a service that allows you to keep track of your favorite TV shows", GetSettings("https://showrss.info/other/all.rss", allowZeroSize: true, defaultReleaseSize: 512));
+                yield return GetDefinition("Torrent RSS Feed", "Generic RSS Feed containing torrents", GetSettings(""));
+            }
+        }
+
+        private IndexerDefinition GetDefinition(string name, string description, TorrentRssIndexerSettings settings)
+        {
+            return new IndexerDefinition
+            {
+                Enable = true,
+                Name = name,
+                Description = description,
+                Implementation = GetType().Name,
+                Settings = settings,
+                Protocol = DownloadProtocol.Torrent,
+                SupportsRss = SupportsRss,
+                SupportsSearch = SupportsSearch,
+                SupportsRedirect = SupportsRedirect,
+                SupportsPagination = SupportsPagination,
+                Capabilities = Capabilities
+            };
+        }
+
+        private TorrentRssIndexerSettings GetSettings(string url, bool? allowZeroSize = null, double? defaultReleaseSize = null)
+        {
+            var settings = new TorrentRssIndexerSettings
+            {
+                BaseUrl = url,
+                AllowZeroSize = allowZeroSize.GetValueOrDefault(false)
+            };
+
+            if (defaultReleaseSize.HasValue)
+            {
+                settings.DefaultReleaseSize = defaultReleaseSize;
+            }
+
+            return settings;
+        }
+
+        private IndexerCapabilities SetCapabilities()
+        {
+            var caps = new IndexerCapabilities
+            {
+                SearchParams = new List<SearchParam>(),
+            };
+
+            caps.Categories.AddCategoryMapping(1, NewznabStandardCategory.Other);
+
+            return caps;
+        }
+    }
+}
@@ -0,0 +1,12 @@
+namespace NzbDrone.Core.Indexers.Definitions.TorrentRss
+{
+    public class TorrentRssIndexerParserSettings
+    {
+        public bool UseEZTVFormat { get; set; }
+        public bool ParseSeedersInDescription { get; set; }
+        public bool UseEnclosureUrl { get; set; }
+        public bool UseEnclosureLength { get; set; }
+        public bool ParseSizeInDescription { get; set; }
+        public string SizeElementName { get; set; }
+    }
+}
@@ -0,0 +1,63 @@
+using System;
+using System.Collections.Generic;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.IndexerSearch.Definitions;
+
+namespace NzbDrone.Core.Indexers.Definitions.TorrentRss
+{
+    public class TorrentRssIndexerRequestGenerator : IIndexerRequestGenerator
+    {
+        public TorrentRssIndexerSettings Settings { get; set; }
+
+        public virtual IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
+        {
+            return new IndexerPageableRequestChain();
+        }
+
+        public virtual IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
+        {
+            return new IndexerPageableRequestChain();
+        }
+
+        public virtual IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
+        {
+            return new IndexerPageableRequestChain();
+        }
+
+        public virtual IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
+        {
+            return new IndexerPageableRequestChain();
+        }
+
+        public virtual IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            if (searchCriteria.IsRssSearch)
+            {
+                pageableRequests.Add(GetRssRequests());
+            }
+
+            return pageableRequests;
+        }
+
+        private IEnumerable<IndexerRequest> GetRssRequests()
+        {
+            var request = new IndexerRequest(Settings.BaseUrl.Trim().TrimEnd('/'), HttpAccept.Rss);
+
+            if (Settings.Cookie.IsNotNullOrWhiteSpace())
+            {
+                foreach (var cookie in HttpHeader.ParseCookies(Settings.Cookie))
+                {
+                    request.HttpRequest.Cookies[cookie.Key] = cookie.Value;
+                }
+            }
+
+            yield return request;
+        }
+
+        public Func<IDictionary<string, string>> GetCookies { get; set; }
+        public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
+    }
+}
@@ -0,0 +1,51 @@
+using FluentValidation;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Indexers.Definitions.TorrentRss
+{
+    public class TorrentRssIndexerSettingsValidator : AbstractValidator<TorrentRssIndexerSettings>
+    {
+        public TorrentRssIndexerSettingsValidator()
+        {
+            RuleFor(c => c.BaseUrl).ValidRootUrl();
+
+            RuleFor(c => c.BaseSettings).SetValidator(new IndexerCommonSettingsValidator());
+            RuleFor(c => c.TorrentBaseSettings).SetValidator(new IndexerTorrentSettingsValidator());
+        }
+    }
+
+    public class TorrentRssIndexerSettings : ITorrentIndexerSettings
+    {
+        private static readonly TorrentRssIndexerSettingsValidator Validator = new ();
+
+        public TorrentRssIndexerSettings()
+        {
+            BaseUrl = string.Empty;
+            AllowZeroSize = false;
+        }
+
+        [FieldDefinition(0, Label = "Full RSS Feed URL", HelpTextWarning = "To sync to your apps you will need to include the 8000(Other) in the Sync Categories", HelpLink = "https://wiki.servarr.com/en/prowlarr/faq#can-i-add-any-generic-torrent-rss-feed")]
+        public string BaseUrl { get; set; }
+
+        [FieldDefinition(1, Label = "Cookie", HelpText = "If your site requires a login cookie to access the RSS, you'll have to retrieve it via a browser.")]
+        public string Cookie { get; set; }
+
+        [FieldDefinition(2, Type = FieldType.Checkbox, Label = "Allow Zero Size", HelpText="Enabling this will allow you to use feeds that don't specify release size, but be careful, size related checks will not be performed.")]
+        public bool AllowZeroSize { get; set; }
+
+        [FieldDefinition(3, Type = FieldType.Number, Label = "Default Release Size", HelpText="Add a default size for feeds with missing sizes.", Unit = "MB", Advanced = true)]
+        public double? DefaultReleaseSize { get; set; }
+
+        [FieldDefinition(20)]
+        public IndexerBaseSettings BaseSettings { get; set; } = new ();
+
+        [FieldDefinition(21)]
+        public IndexerTorrentBaseSettings TorrentBaseSettings { get; set; } = new ();
+
+        public NzbDroneValidationResult Validate()
+        {
+            return new NzbDroneValidationResult(Validator.Validate(this));
+        }
+    }
+}
@@ -0,0 +1,65 @@
+using System;
+using NLog;
+using NzbDrone.Common.Cache;
+using NzbDrone.Common.Serializer;
+using NzbDrone.Core.Indexers.Exceptions;
+
+namespace NzbDrone.Core.Indexers.Definitions.TorrentRss
+{
+    public interface ITorrentRssParserFactory
+    {
+        TorrentRssParser GetParser(TorrentRssIndexerSettings settings);
+    }
+
+    public class TorrentRssParserFactory : ITorrentRssParserFactory
+    {
+        protected readonly Logger _logger;
+
+        private readonly ICached<TorrentRssIndexerParserSettings> _settingsCache;
+
+        private readonly ITorrentRssSettingsDetector _torrentRssSettingsDetector;
+
+        public TorrentRssParserFactory(ICacheManager cacheManager, ITorrentRssSettingsDetector torrentRssSettingsDetector, Logger logger)
+        {
+            _settingsCache = cacheManager.GetCache<TorrentRssIndexerParserSettings>(GetType());
+            _torrentRssSettingsDetector = torrentRssSettingsDetector;
+            _logger = logger;
+        }
+
+        public TorrentRssParser GetParser(TorrentRssIndexerSettings indexerSettings)
+        {
+            var key = indexerSettings.ToJson();
+            var parserSettings = _settingsCache.Get(key, () => DetectParserSettings(indexerSettings), TimeSpan.FromDays(7));
+
+            if (parserSettings.UseEZTVFormat)
+            {
+                return new EzrssTorrentRssParser();
+            }
+
+            return new TorrentRssParser
+            {
+                UseGuidInfoUrl = false,
+                ParseSeedersInDescription = parserSettings.ParseSeedersInDescription,
+
+                UseEnclosureUrl = parserSettings.UseEnclosureUrl,
+                UseEnclosureLength = parserSettings.UseEnclosureLength,
+                ParseSizeInDescription = parserSettings.ParseSizeInDescription,
+                SizeElementName = parserSettings.SizeElementName,
+
+                DefaultReleaseSize = indexerSettings.DefaultReleaseSize
+            };
+        }
+
+        private TorrentRssIndexerParserSettings DetectParserSettings(TorrentRssIndexerSettings indexerSettings)
+        {
+            var settings = _torrentRssSettingsDetector.Detect(indexerSettings);
+
+            if (settings == null)
+            {
+                throw new UnsupportedFeedException("Could not parse feed from {0}", indexerSettings.BaseUrl);
+            }
+
+            return settings;
+        }
+    }
+}
@@ -0,0 +1,305 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Xml;
+using System.Xml.Linq;
+using NLog;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.Indexers.Definitions.TorrentRss
+{
+    public interface ITorrentRssSettingsDetector
+    {
+        TorrentRssIndexerParserSettings Detect(TorrentRssIndexerSettings settings);
+    }
+
+    public class TorrentRssSettingsDetector : ITorrentRssSettingsDetector
+    {
+        private const long ValidSizeThreshold = 2 * 1024 * 1024;
+
+        private readonly IHttpClient _httpClient;
+        private readonly Logger _logger;
+
+        public TorrentRssSettingsDetector(IHttpClient httpClient, Logger logger)
+        {
+            _httpClient = httpClient;
+            _logger = logger;
+        }
+
+        public TorrentRssIndexerParserSettings Detect(TorrentRssIndexerSettings settings)
+        {
+            _logger.Debug("Evaluating TorrentRss feed '{0}'", settings.BaseUrl);
+
+            try
+            {
+                var requestGenerator = new TorrentRssIndexerRequestGenerator { Settings = settings };
+                var request = requestGenerator.GetSearchRequests(new BasicSearchCriteria()).GetAllTiers().First().First();
+
+                HttpResponse httpResponse;
+
+                try
+                {
+                    httpResponse = _httpClient.Execute(request.HttpRequest);
+                }
+                catch (Exception ex)
+                {
+                    _logger.Warn(ex, "Unable to connect to indexer {0}: {1}", request.Url, ex.Message);
+                    return null;
+                }
+
+                var indexerResponse = new IndexerResponse(request, httpResponse);
+
+                return GetParserSettings(indexerResponse, settings);
+            }
+            catch (Exception ex)
+            {
+                ex.WithData("FeedUrl", settings.BaseUrl);
+                throw;
+            }
+        }
+
+        private TorrentRssIndexerParserSettings GetParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)
+        {
+            var settings = GetEzrssParserSettings(response, indexerSettings);
+
+            if (settings != null)
+            {
+                return settings;
+            }
+
+            settings = GetGenericTorrentRssParserSettings(response, indexerSettings);
+
+            if (settings != null)
+            {
+                return settings;
+            }
+
+            return null;
+        }
+
+        private TorrentRssIndexerParserSettings GetEzrssParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)
+        {
+            if (!IsEZTVFeed(response))
+            {
+                return null;
+            }
+
+            _logger.Trace("Feed has Ezrss schema");
+
+            var parser = new EzrssTorrentRssParser();
+            var releases = ParseResponse(parser, response);
+
+            try
+            {
+                ValidateReleases(releases, indexerSettings);
+                ValidateReleaseSize(releases, indexerSettings);
+
+                _logger.Debug("Feed was parseable by Ezrss Parser");
+                return new TorrentRssIndexerParserSettings
+                {
+                    UseEZTVFormat = true
+                };
+            }
+            catch (Exception ex)
+            {
+                _logger.Trace(ex, "Feed wasn't parsable by Ezrss Parser");
+                return null;
+            }
+        }
+
+        private TorrentRssIndexerParserSettings GetGenericTorrentRssParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)
+        {
+            var parser = new TorrentRssParser
+            {
+                UseEnclosureUrl = true,
+                UseEnclosureLength = true,
+                ParseSeedersInDescription = true
+            };
+
+            var item = parser.GetItems(response).FirstOrDefault();
+            if (item == null)
+            {
+                throw new UnsupportedFeedException("Empty feed, cannot check if feed is parsable.");
+            }
+
+            var settings = new TorrentRssIndexerParserSettings()
+            {
+                UseEnclosureUrl = true,
+                UseEnclosureLength = true,
+                ParseSeedersInDescription = true
+            };
+
+            if (item.Element("enclosure") == null)
+            {
+                parser.UseEnclosureUrl = settings.UseEnclosureUrl = false;
+            }
+
+            var releases = ParseResponse(parser, response);
+            ValidateReleases(releases, indexerSettings);
+
+            if (!releases.Any(v => v.Seeders.HasValue))
+            {
+                _logger.Trace("Feed doesn't have Seeders in Description, disabling option.");
+                parser.ParseSeedersInDescription = settings.ParseSeedersInDescription = false;
+            }
+
+            if (!releases.Any(r => r.Size < ValidSizeThreshold))
+            {
+                _logger.Trace("Feed has valid size in enclosure.");
+                return settings;
+            }
+
+            parser.UseEnclosureLength = settings.UseEnclosureLength = false;
+
+            foreach (var sizeElementName in new[] { "size", "Size" })
+            {
+                parser.SizeElementName = settings.SizeElementName = sizeElementName;
+
+                releases = ParseResponse(parser, response);
+                ValidateReleases(releases, indexerSettings);
+
+                if (!releases.Any(r => r.Size < ValidSizeThreshold))
+                {
+                    _logger.Trace("Feed has valid size in Size element.");
+                    return settings;
+                }
+            }
+
+            parser.SizeElementName = settings.SizeElementName = null;
+            parser.ParseSizeInDescription = settings.ParseSizeInDescription = true;
+
+            releases = ParseResponse(parser, response);
+            ValidateReleases(releases, indexerSettings);
+
+            if (releases.Count(r => r.Size >= ValidSizeThreshold) > releases.Length / 2)
+            {
+                if (releases.Any(r => r.Size < ValidSizeThreshold))
+                {
+                    _logger.Debug("Feed {0} contains very small releases.", response.Request.Url);
+                }
+
+                _logger.Trace("Feed has valid size in description.");
+                return settings;
+            }
+
+            parser.ParseSizeInDescription = settings.ParseSizeInDescription = false;
+
+            _logger.Debug("Feed doesn't have release size.");
+
+            releases = ParseResponse(parser, response);
+            ValidateReleases(releases, indexerSettings);
+            ValidateReleaseSize(releases, indexerSettings);
+
+            return settings;
+        }
+
+        private bool IsEZTVFeed(IndexerResponse response)
+        {
+            var content = XmlCleaner.ReplaceEntities(response.Content);
+            content = XmlCleaner.ReplaceUnicode(content);
+
+            using var xmlTextReader = XmlReader.Create(new StringReader(content), new XmlReaderSettings { DtdProcessing = DtdProcessing.Parse, ValidationType = ValidationType.None, IgnoreComments = true, XmlResolver = null });
+            var document = XDocument.Load(xmlTextReader);
+
+            // Check Namespace
+            if (document.Root == null)
+            {
+                throw new InvalidDataException("Could not parse IndexerResponse into XML.");
+            }
+
+            var ns = document.Root.GetNamespaceOfPrefix("torrent");
+            if (ns == "http://xmlns.ezrss.it/0.1/")
+            {
+                _logger.Trace("Identified feed as EZTV compatible by EZTV Namespace");
+                return true;
+            }
+
+            // Check DTD in DocType
+            if (document.DocumentType != null && document.DocumentType.SystemId == "http://xmlns.ezrss.it/0.1/dtd/")
+            {
+                _logger.Trace("Identified feed as EZTV compatible by EZTV DTD");
+                return true;
+            }
+
+            // Check namespaces
+            if (document.Descendants().Any(v => v.GetDefaultNamespace().NamespaceName == "http://xmlns.ezrss.it/0.1/"))
+            {
+                _logger.Trace("Identified feed as EZTV compatible by EZTV Namespace");
+                return true;
+            }
+
+            return false;
+        }
+
+        private TorrentInfo[] ParseResponse(IParseIndexerResponse parser, IndexerResponse response)
+        {
+            try
+            {
+                var releases = parser.ParseResponse(response).Cast<TorrentInfo>().ToArray();
+                return releases;
+            }
+            catch (Exception ex)
+            {
+                _logger.Debug(ex, "Unable to parse indexer feed: " + ex.Message);
+                throw new UnsupportedFeedException("Unable to parse indexer: " + ex.Message);
+            }
+        }
+
+        private void ValidateReleases(TorrentInfo[] releases, TorrentRssIndexerSettings indexerSettings)
+        {
+            if (releases == null || releases.Empty())
+            {
+                throw new UnsupportedFeedException("Empty feed, cannot check if feed is parsable.");
+            }
+
+            var torrentInfo = releases.First();
+
+            _logger.Trace("TorrentInfo: \n{0}", torrentInfo.ToString("L"));
+
+            if (releases.Any(r => r.Title.IsNullOrWhiteSpace()))
+            {
+                throw new UnsupportedFeedException("Feed contains releases without title.");
+            }
+
+            if (releases.Any(r => !IsValidDownloadUrl(r.DownloadUrl)))
+            {
+                throw new UnsupportedFeedException("Failed to find a valid download url in the feed.");
+            }
+
+            var total = releases.Where(v => v.Guid != null).Select(v => v.Guid).ToArray();
+            var distinct = total.Distinct().ToArray();
+
+            if (distinct.Length != total.Length)
+            {
+                throw new UnsupportedFeedException("Feed contains releases with same guid, rejecting malformed rss feed.");
+            }
+        }
+
+        private void ValidateReleaseSize(TorrentInfo[] releases, TorrentRssIndexerSettings indexerSettings)
+        {
+            if (!indexerSettings.AllowZeroSize && releases.Any(r => r.Size == 0))
+            {
+                throw new UnsupportedFeedException("Feed doesn't contain the release content size.");
+            }
+
+            if (releases.Any(r => r.Size != 0 && r.Size < ValidSizeThreshold))
+            {
+                throw new UnsupportedFeedException("Size of one or more releases lower than {0}, feed must contain release content size.", ValidSizeThreshold.SizeSuffix());
+            }
+        }
+
+        private static bool IsValidDownloadUrl(string url)
+        {
+            if (url.IsNullOrWhiteSpace())
+            {
+                return false;
+            }
+
+            return url.StartsWith("magnet:") || url.StartsWith("http:") || url.StartsWith("https:");
+        }
+    }
+}