Cleanup Search UI, Newznab Caps API

2026-04-24 22:55:21 -04:00 · 2020-10-20 13:46:58 -04:00
parent f290afa68c
commit 84cbfe870f
105 changed files with 562 additions and 791 deletions
@@ -0,0 +1,113 @@
+using System;
+using System.Collections.Generic;
+using NLog;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Exceptions;
+using NzbDrone.Core.Http.CloudFlare;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Indexers.Rarbg
+{
+    public class Rarbg : HttpIndexerBase<RarbgSettings>
+    {
+        private readonly IRarbgTokenProvider _tokenProvider;
+
+        public override string Name => "Rarbg";
+
+        public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
+
+        public override IndexerPrivacy Privacy => IndexerPrivacy.Public;
+        public override TimeSpan RateLimit => TimeSpan.FromSeconds(2);
+
+        public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
+            : base(httpClient, indexerStatusService, configService, logger)
+        {
+            _tokenProvider = tokenProvider;
+        }
+
+        public override IIndexerRequestGenerator GetRequestGenerator()
+        {
+            return new RarbgRequestGenerator(_tokenProvider) { Settings = Settings };
+        }
+
+        public override IParseIndexerResponse GetParser()
+        {
+            return new RarbgParser();
+        }
+
+        public override object RequestAction(string action, IDictionary<string, string> query)
+        {
+            if (action == "checkCaptcha")
+            {
+                Settings.Validate().Filter("BaseUrl").ThrowOnError();
+
+                try
+                {
+                    var request = new HttpRequestBuilder(Settings.BaseUrl.Trim('/'))
+                           .Resource("/pubapi_v2.php?get_token=get_token")
+                           .Accept(HttpAccept.Json)
+                           .Build();
+
+                    _httpClient.Get(request);
+                }
+                catch (CloudFlareCaptchaException ex)
+                {
+                    return new
+                    {
+                        captchaRequest = new
+                        {
+                            host = ex.CaptchaRequest.Host,
+                            ray = ex.CaptchaRequest.Ray,
+                            siteKey = ex.CaptchaRequest.SiteKey,
+                            secretToken = ex.CaptchaRequest.SecretToken,
+                            responseUrl = ex.CaptchaRequest.ResponseUrl.FullUri,
+                        }
+                    };
+                }
+
+                return new
+                {
+                    captchaToken = ""
+                };
+            }
+            else if (action == "getCaptchaCookie")
+            {
+                if (query["responseUrl"].IsNullOrWhiteSpace())
+                {
+                    throw new BadRequestException("QueryParam responseUrl invalid.");
+                }
+
+                if (query["ray"].IsNullOrWhiteSpace())
+                {
+                    throw new BadRequestException("QueryParam ray invalid.");
+                }
+
+                if (query["captchaResponse"].IsNullOrWhiteSpace())
+                {
+                    throw new BadRequestException("QueryParam captchaResponse invalid.");
+                }
+
+                var request = new HttpRequestBuilder(query["responseUrl"])
+                    .AddQueryParam("id", query["ray"])
+                    .AddQueryParam("g-recaptcha-response", query["captchaResponse"])
+                    .Build();
+
+                request.UseSimplifiedUserAgent = true;
+                request.AllowAutoRedirect = false;
+
+                var response = _httpClient.Get(request);
+
+                var cfClearanceCookie = response.GetCookies()["cf_clearance"];
+
+                return new
+                {
+                    captchaToken = cfClearanceCookie
+                };
+            }
+
+            return new { };
+        }
+    }
+}
@@ -0,0 +1,97 @@
+using System;
+using System.Collections.Generic;
+using System.Net;
+using System.Text.RegularExpressions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.Indexers.Rarbg
+{
+    public class RarbgParser : IParseIndexerResponse
+    {
+        private static readonly Regex RegexGuid = new Regex(@"^magnet:\?xt=urn:btih:([a-f0-9]+)", RegexOptions.Compiled);
+
+        public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
+        {
+            var results = new List<ReleaseInfo>();
+
+            switch (indexerResponse.HttpResponse.StatusCode)
+            {
+                default:
+                    if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
+                    {
+                        throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
+                    }
+
+                    break;
+            }
+
+            var jsonResponse = new HttpResponse<RarbgResponse>(indexerResponse.HttpResponse);
+
+            if (jsonResponse.Resource.error_code.HasValue)
+            {
+                if (jsonResponse.Resource.error_code == 20 || jsonResponse.Resource.error_code == 8
+                    || jsonResponse.Resource.error_code == 9 || jsonResponse.Resource.error_code == 10)
+                {
+                    // No results or imdbid not found
+                    return results;
+                }
+
+                throw new IndexerException(indexerResponse, "Indexer API call returned error {0}: {1}", jsonResponse.Resource.error_code, jsonResponse.Resource.error);
+            }
+
+            if (jsonResponse.Resource.torrent_results == null)
+            {
+                return results;
+            }
+
+            foreach (var torrent in jsonResponse.Resource.torrent_results)
+            {
+                var torrentInfo = new TorrentInfo();
+
+                torrentInfo.Guid = GetGuid(torrent);
+                torrentInfo.Title = torrent.title;
+                torrentInfo.Size = torrent.size;
+                torrentInfo.DownloadUrl = torrent.download;
+                torrentInfo.InfoUrl = torrent.info_page + "&app_id=Prowlarr";
+                torrentInfo.PublishDate = torrent.pubdate.ToUniversalTime();
+                torrentInfo.Seeders = torrent.seeders;
+                torrentInfo.Peers = torrent.leechers + torrent.seeders;
+
+                if (torrent.movie_info != null)
+                {
+                    if (torrent.movie_info.tvdb != null)
+                    {
+                        torrentInfo.TvdbId = torrent.movie_info.tvdb.Value;
+                    }
+
+                    if (torrent.movie_info.tvrage != null)
+                    {
+                        torrentInfo.TvRageId = torrent.movie_info.tvrage.Value;
+                    }
+                }
+
+                results.Add(torrentInfo);
+            }
+
+            return results;
+        }
+
+        public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
+
+        private string GetGuid(RarbgTorrent torrent)
+        {
+            var match = RegexGuid.Match(torrent.download);
+
+            if (match.Success)
+            {
+                return string.Format("rarbg-{0}", match.Groups[1].Value);
+            }
+            else
+            {
+                return string.Format("rarbg-{0}", torrent.download);
+            }
+        }
+    }
+}
@@ -0,0 +1,122 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.IndexerSearch.Definitions;
+
+namespace NzbDrone.Core.Indexers.Rarbg
+{
+    public class RarbgRequestGenerator : IIndexerRequestGenerator
+    {
+        private readonly IRarbgTokenProvider _tokenProvider;
+
+        public RarbgSettings Settings { get; set; }
+
+        public RarbgRequestGenerator(IRarbgTokenProvider tokenProvider)
+        {
+            _tokenProvider = tokenProvider;
+        }
+
+        public virtual IndexerPageableRequestChain GetRecentRequests()
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetPagedRequests("list", null, null));
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetMovieRequest(searchCriteria));
+            return pageableRequests;
+        }
+
+        private IEnumerable<IndexerRequest> GetPagedRequests(string mode, int? imdbId, string query, params object[] args)
+        {
+            var requestBuilder = new HttpRequestBuilder(Settings.BaseUrl)
+                .Resource("/pubapi_v2.php")
+                .Accept(HttpAccept.Json);
+
+            if (Settings.CaptchaToken.IsNotNullOrWhiteSpace())
+            {
+                requestBuilder.UseSimplifiedUserAgent = true;
+                requestBuilder.SetCookie("cf_clearance", Settings.CaptchaToken);
+            }
+
+            requestBuilder.AddQueryParam("mode", mode);
+
+            if (imdbId.HasValue)
+            {
+                requestBuilder.AddQueryParam("search_imdb", imdbId.Value);
+            }
+
+            if (query.IsNotNullOrWhiteSpace())
+            {
+                requestBuilder.AddQueryParam("search_string", string.Format(query, args));
+            }
+
+            if (!Settings.RankedOnly)
+            {
+                requestBuilder.AddQueryParam("ranked", "0");
+            }
+
+            var categoryParam = string.Join(";", Settings.Categories.Distinct());
+
+            requestBuilder.AddQueryParam("category", categoryParam);
+            requestBuilder.AddQueryParam("limit", "100");
+            requestBuilder.AddQueryParam("token", _tokenProvider.GetToken(Settings));
+            requestBuilder.AddQueryParam("format", "json_extended");
+            requestBuilder.AddQueryParam("app_id", "Prowlarr");
+
+            yield return new IndexerRequest(requestBuilder.Build());
+        }
+
+        private IEnumerable<IndexerRequest> GetMovieRequest(MovieSearchCriteria searchCriteria)
+        {
+            var requestBuilder = new HttpRequestBuilder(Settings.BaseUrl)
+                .Resource("/pubapi_v2.php")
+                .Accept(HttpAccept.Json);
+
+            if (Settings.CaptchaToken.IsNotNullOrWhiteSpace())
+            {
+                requestBuilder.UseSimplifiedUserAgent = true;
+                requestBuilder.SetCookie("cf_clearance", Settings.CaptchaToken);
+            }
+
+            requestBuilder.AddQueryParam("mode", "search");
+
+            if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace())
+            {
+                requestBuilder.AddQueryParam("search_imdb", searchCriteria.ImdbId);
+            }
+            else if (searchCriteria.TmdbId > 0)
+            {
+                requestBuilder.AddQueryParam("search_themoviedb", searchCriteria.TmdbId);
+            }
+            else if (searchCriteria.QueryTitles.Count > 0)
+            {
+                requestBuilder.AddQueryParam("search_string", $"{searchCriteria.QueryTitles.First()}");
+            }
+
+            if (!Settings.RankedOnly)
+            {
+                requestBuilder.AddQueryParam("ranked", "0");
+            }
+
+            var categoryParam = string.Join(";", Settings.Categories.Distinct());
+
+            requestBuilder.AddQueryParam("category", categoryParam);
+            requestBuilder.AddQueryParam("limit", "100");
+            requestBuilder.AddQueryParam("token", _tokenProvider.GetToken(Settings));
+            requestBuilder.AddQueryParam("format", "json_extended");
+            requestBuilder.AddQueryParam("app_id", BuildInfo.AppName);
+
+            yield return new IndexerRequest(requestBuilder.Build());
+        }
+
+        public Func<IDictionary<string, string>> GetCookies { get; set; }
+        public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
+    }
+}
@@ -0,0 +1,33 @@
+using System;
+using System.Collections.Generic;
+
+namespace NzbDrone.Core.Indexers.Rarbg
+{
+    public class RarbgResponse
+    {
+        public string error { get; set; }
+        public int? error_code { get; set; }
+        public List<RarbgTorrent> torrent_results { get; set; }
+    }
+
+    public class RarbgTorrent
+    {
+        public string title { get; set; }
+        public string category { get; set; }
+        public string download { get; set; }
+        public int? seeders { get; set; }
+        public int? leechers { get; set; }
+        public long size { get; set; }
+        public DateTime pubdate { get; set; }
+        public RarbgTorrentInfo movie_info { get; set; }
+        public int? ranked { get; set; }
+        public string info_page { get; set; }
+    }
+
+    public class RarbgTorrentInfo
+    {
+        public string imdb { get; set; }
+        public int? tvrage { get; set; }
+        public int? tvdb { get; set; }
+    }
+}
@@ -0,0 +1,60 @@
+using System.Collections.Generic;
+using FluentValidation;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Languages;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Indexers.Rarbg
+{
+    public class RarbgSettingsValidator : AbstractValidator<RarbgSettings>
+    {
+        public RarbgSettingsValidator()
+        {
+            RuleFor(c => c.BaseUrl).ValidRootUrl();
+
+            RuleFor(c => c.Categories).NotEmpty();
+        }
+    }
+
+    public class RarbgSettings : ITorrentIndexerSettings
+    {
+        private static readonly RarbgSettingsValidator Validator = new RarbgSettingsValidator();
+
+        public RarbgSettings()
+        {
+            BaseUrl = "https://torrentapi.org";
+            RankedOnly = false;
+            MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
+            Categories = new[] { 14, 48, 17, 44, 45, 47, 50, 51, 52, 42, 46 };
+            MultiLanguages = new List<int>();
+            RequiredFlags = new List<int>();
+        }
+
+        [FieldDefinition(0, Label = "API URL", HelpText = "URL to Rarbg api, not the website.")]
+        public string BaseUrl { get; set; }
+
+        [FieldDefinition(1, Type = FieldType.Checkbox, Label = "Ranked Only", HelpText = "Only include ranked results.")]
+        public bool RankedOnly { get; set; }
+
+        [FieldDefinition(2, Type = FieldType.Captcha, Label = "CAPTCHA Token", HelpText = "CAPTCHA Clearance token used to handle CloudFlare Anti-DDOS measures on shared-ip VPNs.")]
+        public string CaptchaToken { get; set; }
+
+        [FieldDefinition(3, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
+        public IEnumerable<int> MultiLanguages { get; set; }
+
+        [FieldDefinition(4, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
+        public int MinimumSeeders { get; set; }
+
+        [FieldDefinition(5, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", HelpLink = "https://github.com/Prowlarr/Prowlarr/wiki/Indexer-Flags#1-required-flags", Advanced = true)]
+        public IEnumerable<int> RequiredFlags { get; set; }
+
+        [FieldDefinition(6, Type = FieldType.Textbox, Label = "Categories", HelpText = "Comma Separated list, you can retrieve the ID by checking the URL behind the category on the website (i.e. Movie/x264/1080 = 44)", HelpLink = "https://rarbgmirror.org/torrents.php?category=movies", Advanced = true)]
+        public IEnumerable<int> Categories { get; set; }
+
+        public NzbDroneValidationResult Validate()
+        {
+            return new NzbDroneValidationResult(Validator.Validate(this));
+        }
+    }
+}
@@ -0,0 +1,51 @@
+using System;
+using Newtonsoft.Json.Linq;
+using NLog;
+using NzbDrone.Common.Cache;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+
+namespace NzbDrone.Core.Indexers.Rarbg
+{
+    public interface IRarbgTokenProvider
+    {
+        string GetToken(RarbgSettings settings);
+    }
+
+    public class RarbgTokenProvider : IRarbgTokenProvider
+    {
+        private readonly IHttpClient _httpClient;
+        private readonly ICached<string> _tokenCache;
+        private readonly Logger _logger;
+
+        public RarbgTokenProvider(IHttpClient httpClient, ICacheManager cacheManager, Logger logger)
+        {
+            _httpClient = httpClient;
+            _tokenCache = cacheManager.GetCache<string>(GetType());
+            _logger = logger;
+        }
+
+        public string GetToken(RarbgSettings settings)
+        {
+            return _tokenCache.Get(settings.BaseUrl,
+                () =>
+                {
+                    var requestBuilder = new HttpRequestBuilder(settings.BaseUrl.Trim('/'))
+                        .WithRateLimit(3.0)
+                        .Resource("/pubapi_v2.php?get_token=get_token&app_id=Prowlarr")
+                        .Accept(HttpAccept.Json);
+
+                    if (settings.CaptchaToken.IsNotNullOrWhiteSpace())
+                    {
+                        requestBuilder.UseSimplifiedUserAgent = true;
+                        requestBuilder.SetCookie("cf_clearance", settings.CaptchaToken);
+                    }
+
+                    var response = _httpClient.Get<JObject>(requestBuilder.Build());
+
+                    return response.Resource["token"].ToString();
+                },
+                TimeSpan.FromMinutes(14.0));
+        }
+    }
+}