Fixed: (Indexers) Rate limit auth requests

Currently translated at 100.0% (471 of 471 strings)

Translated using Weblate (Portuguese (Brazil))

Currently translated at 100.0% (471 of 471 strings)

Translated using Weblate (Italian)

Currently translated at 100.0% (471 of 471 strings)

Co-authored-by: Csaba <csab0825@gmail.com>
Co-authored-by: Davide Palma <github@davidepalma.it>
Co-authored-by: Iagocds <cdsiago@gmail.com>
Co-authored-by: Weblate <noreply@weblate.org>
Translate-URL: https://translate.servarr.com/projects/servarr/prowlarr/hu/
Translate-URL: https://translate.servarr.com/projects/servarr/prowlarr/it/
Translate-URL: https://translate.servarr.com/projects/servarr/prowlarr/pt_BR/
Translation: Servarr/Prowlarr

azure-pipelines.yml

@@ -9,7 +9,7 @@ variables:
   testsFolder: './_tests'
   yarnCacheFolder: $(Pipeline.Workspace)/.yarn
   nugetCacheFolder: $(Pipeline.Workspace)/.nuget/packages
-  majorVersion: '1.0.1'
+  majorVersion: '1.1.0'
   minorVersion: $[counter('minorVersion', 1)]
   prowlarrVersion: '$(majorVersion).$(minorVersion)'
   buildName: '$(Build.SourceBranchName).$(prowlarrVersion)'

src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs

@@ -50,7 +50,7 @@ namespace NzbDrone.Core.Test.IndexerTests.FileListTests
             torrentInfo.InfoUrl.Should().Be("https://filelist.io/details.php?id=665873");
             torrentInfo.CommentUrl.Should().BeNullOrEmpty();
             torrentInfo.Indexer.Should().Be(Subject.Definition.Name);
-            torrentInfo.PublishDate.Should().Be(DateTime.Parse("2020-01-25 22:20:19"));
+            torrentInfo.PublishDate.Should().Be(DateTime.Parse("2020-01-25 20:20:19").ToUniversalTime());
             torrentInfo.Size.Should().Be(8300512414);
             torrentInfo.InfoHash.Should().Be(null);
             torrentInfo.MagnetUrl.Should().Be(null);

src/NzbDrone.Core/Download/DownloadService.cs

@@ -61,14 +61,6 @@ namespace NzbDrone.Core.Download
 
             // Get the seed configuration for this release.
             // remoteMovie.SeedConfiguration = _seedConfigProvider.GetSeedConfiguration(remoteMovie);
-
-            // Limit grabs to 2 per second.
-            if (release.DownloadUrl.IsNotNullOrWhiteSpace() && !release.DownloadUrl.StartsWith("magnet:"))
-            {
-                var url = new HttpUri(release.DownloadUrl);
-                _rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2));
-            }
-
             var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(release.IndexerId));
 
             string downloadClientId;

src/NzbDrone.Core/Http/CloudFlare/CloudFlareDetectionService.cs

@@ -28,7 +28,11 @@ namespace NzbDrone.Core.Http.CloudFlare
             if (response.StatusCode.Equals(HttpStatusCode.ServiceUnavailable) ||
                 response.StatusCode.Equals(HttpStatusCode.Forbidden))
             {
-                return true; // Defected CloudFlare and DDoS-GUARD
+                var responseHtml = response.Content;
+                if (responseHtml.Contains("<title>Just a moment...") || responseHtml.Contains("<title>DDOS-GUARD"))
+                {
+                    return true;
+                }
             }
 
             // detect Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands

src/NzbDrone.Core/IndexerProxies/FlareSolverr/FlareSolverr.cs

@@ -170,6 +170,7 @@ namespace NzbDrone.Core.IndexerProxies.FlareSolverr
             newRequest.Headers.ContentType = "application/json";
             newRequest.Method = HttpMethod.Post;
             newRequest.LogResponseContent = true;
+            newRequest.RequestTimeout = TimeSpan.FromSeconds(Settings.RequestTimeout + 5);
             newRequest.SetContent(req.ToJson());
 
             _logger.Debug("Cloudflare Detected, Applying FlareSolverr Proxy {0} to request {1}", Name, request.Url);

src/NzbDrone.Core/Indexers/Definitions/Avistaz/AvistazSettings.cs

@@ -33,7 +33,7 @@ namespace NzbDrone.Core.Indexers.Definitions.Avistaz
         [FieldDefinition(3, Label = "Password", HelpText = "Site Password", Privacy = PrivacyLevel.Password, Type = FieldType.Password)]
         public string Password { get; set; }
 
-        [FieldDefinition(4, Label = "PID", HelpText = "PID from My Account or My Profile page")]
+        [FieldDefinition(4, Label = "PID", HelpText = "PID from My Account or My Profile page", Privacy = PrivacyLevel.Password, Type = FieldType.Password)]
         public string Pid { get; set; }
 
         [FieldDefinition(5, Label = "Freeleech Only", Type = FieldType.Checkbox, HelpText = "Search freeleech only")]

src/NzbDrone.Core/Indexers/Definitions/BeyondHD.cs

@@ -223,7 +223,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                     InfoUrl = details,
                     Guid = details,
                     Categories = _categories.MapTrackerCatDescToNewznab(row.Category),
-                    PublishDate = DateTime.Parse(row.CreatedAt, CultureInfo.InvariantCulture),
+                    PublishDate = DateTime.Parse(row.CreatedAt, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal),
                     Size = row.Size,
                     Grabs = row.Grabs,
                     Seeders = row.Seeders,

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -30,9 +30,24 @@ namespace NzbDrone.Core.Indexers.Cardigann
         public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
 
         // Page size is different per indexer, setting to 1 ensures we don't break out of paging logic
-        // thinking its a partial page and insteaad all search_path requests are run for each indexer
+        // thinking its a partial page and instead all search_path requests are run for each indexer
         public override int PageSize => 1;
 
+        public override TimeSpan RateLimit
+        {
+            get
+            {
+                var definition = _definitionService.GetCachedDefinition(Settings.DefinitionFile);
+
+                if (definition.RequestDelay.HasValue && definition.RequestDelay.Value > base.RateLimit.TotalSeconds)
+                {
+                    return TimeSpan.FromSeconds(definition.RequestDelay.Value);
+                }
+
+                return base.RateLimit;
+            }
+        }
+
         public override IIndexerRequestGenerator GetRequestGenerator()
         {
             var generator = _generatorCache.Get(Settings.DefinitionFile, () =>
@@ -165,60 +180,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
             await generator.DoLogin();
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override async Task<HttpRequest> GetDownloadRequest(Uri link)
         {
             var generator = (CardigannRequestGenerator)GetRequestGenerator();
 
             var request = await generator.DownloadRequest(link);
 
-            if (request.Url.Scheme == "magnet")
-            {
-                ValidateMagnet(request.Url.FullUri);
-                return Encoding.UTF8.GetBytes(request.Url.FullUri);
-            }
-
             request.AllowAutoRedirect = true;
 
-            var downloadBytes = Array.Empty<byte>();
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                downloadBytes = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", request.Url.FullUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            return downloadBytes;
+            return request;
         }
 
         protected override async Task Test(List<ValidationFailure> failures)

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinition.cs

@@ -41,6 +41,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
         public string Type { get; set; }
         public string Language { get; set; }
         public string Encoding { get; set; }
+        public double? RequestDelay { get; set; }
         public List<string> Links { get; set; }
         public List<string> Legacylinks { get; set; }
         public bool Followredirect { get; set; } = false;

src/NzbDrone.Core/Indexers/Definitions/FileList/FileListApi.cs

@@ -22,7 +22,7 @@ namespace NzbDrone.Core.Indexers.FileList
         [JsonProperty(PropertyName = "doubleup")]
         public bool DoubleUp { get; set; }
         [JsonProperty(PropertyName = "upload_date")]
-        public DateTime UploadDate { get; set; }
+        public string UploadDate { get; set; }
         public string Category { get; set; }
     }
 }

src/NzbDrone.Core/Indexers/Definitions/FileList/FileListParser.cs

@@ -36,7 +36,7 @@ namespace NzbDrone.Core.Indexers.FileList
             {
                 var id = result.Id;
 
-                var flags = new List<IndexerFlag>();
+                var flags = new HashSet<IndexerFlag>();
 
                 if (result.Internal)
                 {
@@ -62,7 +62,7 @@ namespace NzbDrone.Core.Indexers.FileList
                     InfoUrl = GetInfoUrl(id),
                     Seeders = result.Seeders,
                     Peers = result.Leechers + result.Seeders,
-                    PublishDate = result.UploadDate,
+                    PublishDate = DateTime.Parse(result.UploadDate + " +0200"),
                     ImdbId = imdbId,
                     IndexerFlags = flags,
                     Files = (int)result.Files,

src/NzbDrone.Core/Indexers/Definitions/GreatPosterWall.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Net;
 using Newtonsoft.Json;
 using NLog;
@@ -80,6 +81,8 @@ public class GreatPosterWallRequestGenerator : GazelleRequestGenerator
 
 public class GreatPosterWallParser : GazelleParser
 {
+    private readonly HashSet<string> _hdResolutions = new HashSet<string> { "1080p", "1080i", "720p" };
+
     public GreatPosterWallParser(GazelleSettings settings, IndexerCapabilities capabilities)
         : base(settings, capabilities)
     {
@@ -130,8 +133,8 @@ public class GreatPosterWallParser : GazelleParser
                     Guid = infoUrl,
                     PosterUrl = GetPosterUrl(result.Cover),
                     DownloadUrl = GetDownloadUrl(torrent.TorrentId, torrent.CanUseToken),
-                    PublishDate = new DateTimeOffset(time, TimeSpan.FromHours(8)).LocalDateTime, // Time is Chinese Time, add 8 hours difference from UTC and then convert back to local time
-                    Categories = new List<IndexerCategory> { NewznabStandardCategory.Movies },
+                    PublishDate = new DateTimeOffset(time, TimeSpan.FromHours(8)).UtcDateTime, // Time is Chinese Time, add 8 hours difference from UTC
+                    Categories = ParseCategories(torrent),
                     Size = torrent.Size,
                     Seeders = torrent.Seeders,
                     Peers = torrent.Seeders + torrent.Leechers,
@@ -173,7 +176,9 @@ public class GreatPosterWallParser : GazelleParser
             }
         }
 
-        return torrentInfos;
+        return torrentInfos
+            .OrderByDescending(o => o.PublishDate)
+            .ToArray();
     }
 
     protected string GetDownloadUrl(int torrentId, bool canUseToken)
@@ -186,6 +191,20 @@ public class GreatPosterWallParser : GazelleParser
 
         return url.FullUri;
     }
+
+    protected virtual List<IndexerCategory> ParseCategories(GreatPosterWallTorrent torrent)
+    {
+        var cats = new List<IndexerCategory>();
+
+        cats.Add(torrent.Resolution switch
+        {
+            var res when _hdResolutions.Contains(res) => NewznabStandardCategory.MoviesHD,
+            "2160p" => NewznabStandardCategory.MoviesUHD,
+            _ => NewznabStandardCategory.MoviesSD
+        });
+
+        return cats;
+    }
 }
 
 public class GreatPosterWallResponse

src/NzbDrone.Core/Indexers/Definitions/HDBits/HDBitsParser.cs

@@ -55,7 +55,7 @@ namespace NzbDrone.Core.Indexers.HDBits
                 var id = result.Id;
                 var internalRelease = result.TypeOrigin == 1 ? true : false;
 
-                var flags = new List<IndexerFlag>();
+                var flags = new HashSet<IndexerFlag>();
 
                 if (internalRelease)
                 {

src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs

@@ -165,7 +165,7 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
         }
 
-        private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories, string imdbId = null)
+        private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories, int limit, int offset, string imdbId = null)
         {
             var searchUrl = Settings.BaseUrl + "t";
 
@@ -194,6 +194,8 @@ namespace NzbDrone.Core.Indexers.Definitions
                 qc.Add(cat, string.Empty);
             }
 
+            qc.Add("p", ((offset / limit) + 1).ToString());
+
             searchUrl = searchUrl + "?" + qc.GetQueryString();
 
             var request = new IndexerRequest(searchUrl, HttpAccept.Html);
@@ -210,7 +212,7 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SearchTerm), searchCriteria.Categories, searchCriteria.FullImdbId));
+            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SearchTerm), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0, searchCriteria.FullImdbId));
 
             return pageableRequests;
         }
@@ -219,7 +221,7 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
+            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
 
             return pageableRequests;
         }
@@ -228,7 +230,7 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedTvSearchString), searchCriteria.Categories, searchCriteria.FullImdbId));
+            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedTvSearchString), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0, searchCriteria.FullImdbId));
 
             return pageableRequests;
         }
@@ -237,7 +239,7 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
+            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
 
             return pageableRequests;
         }
@@ -246,7 +248,7 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
+            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
 
             return pageableRequests;
         }

src/NzbDrone.Core/Indexers/Definitions/Nebulance.cs

@@ -208,7 +208,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                     Categories = new List<IndexerCategory> { TvCategoryFromQualityParser.ParseTvShowQuality(row.ReleaseTitle) },
                     Size = ParseUtil.CoerceLong(row.Size),
                     Files = row.FileList.Length,
-                    PublishDate = DateTime.Parse(row.PublishDateUtc, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal),
+                    PublishDate = DateTime.Parse(row.PublishDateUtc, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal),
                     Grabs = ParseUtil.CoerceInt(row.Snatch),
                     Seeders = ParseUtil.CoerceInt(row.Seed),
                     Peers = ParseUtil.CoerceInt(row.Seed) + ParseUtil.CoerceInt(row.Leech),

src/NzbDrone.Core/Indexers/Definitions/PassThePopcorn/PassThePopcornParser.cs

@@ -72,7 +72,7 @@ namespace NzbDrone.Core.Indexers.PassThePopcorn
                     var id = torrent.Id;
                     var title = torrent.ReleaseName;
 
-                    var flags = new List<IndexerFlag>();
+                    var flags = new HashSet<IndexerFlag>();
 
                     if (torrent.GoldenPopcorn)
                     {

src/NzbDrone.Core/Indexers/Definitions/Redacted.cs

@@ -80,26 +80,18 @@ namespace NzbDrone.Core.Indexers.Definitions
             return caps;
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
         {
-            var request = new HttpRequestBuilder(link.AbsoluteUri)
-                .SetHeader("Authorization", Settings.Apikey)
-                .Build();
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
 
-            var downloadBytes = Array.Empty<byte>();
-
-            try
+            if (Cookies != null)
             {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                downloadBytes = response.ResponseData;
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Download failed");
+                requestBuilder.SetCookies(Cookies);
             }
 
-            return downloadBytes;
+            var request = requestBuilder.SetHeader("Authorization", Settings.Apikey).Build();
+
+            return Task.FromResult(request);
         }
     }
 

src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs

@@ -103,16 +103,8 @@ namespace NzbDrone.Core.Indexers.Definitions
             request.HttpRequest.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
         {
-            Cookies = GetCookies();
-
-            if (link.Scheme == "magnet")
-            {
-                ValidateMagnet(link.OriginalString);
-                return Encoding.UTF8.GetBytes(link.OriginalString);
-            }
-
             var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
 
             if (Cookies != null)
@@ -124,46 +116,7 @@ namespace NzbDrone.Core.Indexers.Definitions
             request.AllowAutoRedirect = FollowRedirect;
             request.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
 
-            byte[] torrentData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                torrentData = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            return torrentData;
+            return Task.FromResult(request);
         }
 
         protected virtual IndexerCapabilities SetCapabilities()

src/NzbDrone.Core/Indexers/Definitions/Torznab/TorznabRssParser.cs

@@ -229,15 +229,14 @@ namespace NzbDrone.Core.Indexers.Torznab
             return base.GetPeers(item);
         }
 
-        protected List<IndexerFlag> GetFlags(XElement item)
+        protected HashSet<IndexerFlag> GetFlags(XElement item)
         {
-            var flags = new List<IndexerFlag>();
+            var flags = new HashSet<IndexerFlag>();
 
             var downloadFactor = TryGetFloatTorznabAttribute(item, "downloadvolumefactor", 1);
-
             var uploadFactor = TryGetFloatTorznabAttribute(item, "uploadvolumefactor", 1);
 
-            if (uploadFactor == 2)
+            if (uploadFactor == 2.0)
             {
                 flags.Add(IndexerFlag.DoubleUpload);
             }

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -5,10 +5,12 @@ using System.Net;
 using System.Text;
 using System.Threading.Tasks;
 using FluentValidation.Results;
+using MonoTorrent;
 using NLog;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Http.CloudFlare;
 using NzbDrone.Core.Indexers.Events;
 using NzbDrone.Core.Indexers.Exceptions;
@@ -100,6 +102,93 @@ namespace NzbDrone.Core.Indexers
             return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
         }
 
+        public override async Task<byte[]> Download(Uri link)
+        {
+            Cookies = GetCookies();
+
+            var request = await GetDownloadRequest(link);
+
+            if (request.Url.Scheme == "magnet")
+            {
+                ValidateMagnet(request.Url.FullUri);
+                return Encoding.UTF8.GetBytes(request.Url.FullUri);
+            }
+
+            if (request.RateLimit < RateLimit)
+            {
+                request.RateLimit = RateLimit;
+            }
+
+            byte[] fileData;
+
+            try
+            {
+                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
+                fileData = response.ResponseData;
+
+                _logger.Debug("Downloaded for release finished ({0} bytes from {1})", fileData.Length, link.AbsoluteUri);
+            }
+            catch (HttpException ex)
+            {
+                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
+                {
+                    _logger.Error(ex, "Downloading file for release failed since it no longer exists ({0})", link.AbsoluteUri);
+                    throw new ReleaseUnavailableException("Downloading nzb failed", ex);
+                }
+
+                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
+                {
+                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
+                }
+                else
+                {
+                    _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
+                }
+
+                throw new ReleaseDownloadException("Downloading failed", ex);
+            }
+            catch (WebException ex)
+            {
+                _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
+
+                throw new ReleaseDownloadException("Downloading failed", ex);
+            }
+            catch (Exception)
+            {
+                _indexerStatusService.RecordFailure(Definition.Id);
+                _logger.Error("Downloading failed");
+                throw;
+            }
+
+            ValidateDownloadData(fileData);
+
+            return fileData;
+        }
+
+        protected virtual Task<HttpRequest> GetDownloadRequest(Uri link)
+        {
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
+
+            if (Cookies != null)
+            {
+                requestBuilder.SetCookies(Cookies);
+            }
+
+            var request = requestBuilder.Build();
+            request.AllowAutoRedirect = FollowRedirect;
+
+            return Task.FromResult(request);
+        }
+
+        protected virtual void ValidateDownloadData(byte[] fileData)
+        {
+        }
+
+        protected void ValidateMagnet(string link)
+        {
+            MagnetLink.Parse(link);
+        }
+
         protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator)
         {
             //A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page.
@@ -404,6 +493,11 @@ namespace NzbDrone.Core.Indexers
         {
             request.Encoding = Encoding;
 
+            if (request.RateLimit < RateLimit)
+            {
+                request.RateLimit = RateLimit;
+            }
+
             var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
 
             _eventAggregator.PublishEvent(new IndexerAuthEvent(Definition.Id, !response.HasHttpError, response.ElapsedTime));

src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs

@@ -1,12 +1,5 @@
-using System;
-using System.Net;
-using System.Text;
-using System.Threading.Tasks;
-using MonoTorrent;
 using NLog;
-using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Messaging.Events;
 
 namespace NzbDrone.Core.Indexers
@@ -18,72 +11,5 @@ namespace NzbDrone.Core.Indexers
             : base(httpClient, eventAggregator, indexerStatusService, configService, logger)
         {
         }
-
-        public override async Task<byte[]> Download(Uri link)
-        {
-            Cookies = GetCookies();
-
-            if (link.Scheme == "magnet")
-            {
-                ValidateMagnet(link.OriginalString);
-                return Encoding.UTF8.GetBytes(link.OriginalString);
-            }
-
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-
-            byte[] torrentData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                torrentData = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            return torrentData;
-        }
-
-        protected void ValidateMagnet(string link)
-        {
-            MagnetLink.Parse(link);
-        }
     }
 }

src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs

@@ -1,11 +1,6 @@
-using System;
-using System.Net;
-using System.Threading.Tasks;
 using NLog;
-using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Download;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Messaging.Events;
 
 namespace NzbDrone.Core.Indexers
@@ -21,64 +16,9 @@ namespace NzbDrone.Core.Indexers
             _nzbValidationService = nzbValidationService;
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override void ValidateDownloadData(byte[] fileData)
         {
-            Cookies = GetCookies();
-
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-
-            byte[] nzbData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                nzbData = response.ResponseData;
-
-                _logger.Debug("Downloaded nzb for release finished ({0} bytes from {1})", nzbData.Length, link.AbsoluteUri);
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading nzb file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading nzb failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading nzb failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading nzb failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading nzb failed");
-                throw;
-            }
-
-            _nzbValidationService.Validate(nzbData);
-
-            return nzbData;
+            _nzbValidationService.Validate(fileData);
         }
     }
 }

src/NzbDrone.Core/Localization/Core/en.json

@@ -47,7 +47,7 @@
   "AuthenticationMethodHelpText": "Require Username and Password to access Prowlarr",
   "AuthenticationRequired": "Authentication Required",
   "AuthenticationRequiredHelpText": "Change which requests authentication is required for. Do not change unless you understand the risks.",
-  "AuthenticationRequiredWarning": "To prevent remote access without authentication, Prowlarr now requires authentication to be enabled. You can optionally disable authentication from local addresses.",
+  "AuthenticationRequiredWarning": "To prevent remote access without authentication, Prowlarr now requires authentication to be enabled. Configure your authentication method and credentials. You can optionally disable authentication from local addresses. Refer to the FAQ for additional information.",
   "Automatic": "Automatic",
   "AutomaticSearch": "Automatic Search",
   "Backup": "Backup",

src/NzbDrone.Core/Localization/Core/hu.json

@@ -466,5 +466,8 @@
     "AreYouSureYouWantToDeleteCategory": "Biztosan törölni szeretnéd a leképezett kategóriát?",
     "DeleteClientCategory": "Letöltési kliens kategória törlése",
     "MappedCategories": "Térképezett kategóriák",
-    "DownloadClientCategory": "Letöltési kliens kategória"
+    "DownloadClientCategory": "Letöltési kliens kategória",
+    "AuthenticationRequired": "Azonosítás szükséges",
+    "AuthenticationRequiredHelpText": "Módosítsd, hogy mely kérésekhez van szükség hitelesítésre. Ne változtasson, hacsak nem érti a kockázatokat.",
+    "AuthenticationRequiredWarning": "A hitelesítés nélküli távoli hozzáférés megakadályozása érdekében a Prowlarrnak mostantól engedélyezni kell a hitelesítést. Opcionálisan letilthatja a helyi címekről történő hitelesítést."
 }

src/NzbDrone.Core/Localization/Core/it.json

@@ -20,7 +20,7 @@
     "AppDataLocationHealthCheckMessage": "L'aggiornamento non sarà possibile per evitare la cancellazione di AppData durante l'aggiornamento",
     "Analytics": "Analitica",
     "Added": "Aggiunto",
-    "About": "Informazioni",
+    "About": "Info",
     "Updates": "Aggiornamenti",
     "UpdateCheckUINotWritableMessage": "Impossibile installare l'aggiornamento perché l'utente '{1}' non ha i permessi di scrittura per la cartella dell'interfaccia '{0}'.",
     "UpdateCheckStartupNotWritableMessage": "Impossibile installare l'aggiornamento perché l'utente '{1}' non ha i permessi di scrittura per la cartella di avvio '{0}'.",
@@ -79,7 +79,7 @@
     "Connect": "Notifiche",
     "Connections": "Collegamenti",
     "ConnectSettingsSummary": "Notifiche e script personalizzati",
-    "BackupNow": "Effettua il Backup adesso",
+    "BackupNow": "Esegui backup ora",
     "Backup": "Backup",
     "All": "Tutti",
     "Actions": "Azioni",
@@ -112,7 +112,7 @@
     "Type": "Tipo",
     "Title": "Titolo",
     "Time": "Ora",
-    "TestAll": "Testa Tutti",
+    "TestAll": "Prova Tutti",
     "Test": "Test",
     "TableOptionsColumnsMessage": "Scegli quali colonne rendere visibili ed il loro ordine",
     "TableOptions": "Opzioni della tabella",
@@ -260,7 +260,7 @@
     "UILanguageHelpText": "Lingua che Prowlarr userà per l'interfaccia",
     "UILanguage": "Lingua dell'Interfaccia",
     "Torrents": "Torrents",
-    "TestAllClients": "Testa Tutti i Client",
+    "TestAllClients": "Testa tutti i client",
     "TagsHelpText": "Si applica agli Indicizzatori con almeno un etichetta corrispondente",
     "TagIsNotUsedAndCanBeDeleted": "L'etichetta non è in uso e può essere eliminata",
     "TagCannotBeDeletedWhileInUse": "Non può essere cancellato mentre è in uso",
@@ -294,7 +294,7 @@
     "MovieIndexScrollBottom": "Indice Film: scorri in basso",
     "FocusSearchBox": "Evidenzia casella di ricerca",
     "CloseCurrentModal": "Chiudi la Modale Attuale",
-    "AcceptConfirmationModal": "Acetta Conferma Modale",
+    "AcceptConfirmationModal": "Accetta Conferma Modale",
     "IndexerLongTermStatusCheckSingleClientMessage": "Alcuni Indicizzatori non sono disponibili da più di 6 ore a causa di errori: {0}",
     "IndexerLongTermStatusCheckAllClientMessage": "Nessun Indicizzatore è disponibile da più di 6 ore a causa di errori",
     "PrioritySettings": "Priorità",
@@ -305,7 +305,7 @@
     "EnableIndexer": "Abilita Indicizzatore",
     "AddNewIndexer": "Aggiungi nuovo Indicizzatore",
     "IndexerAuth": "Autenticazione dell'Indicizzatore",
-    "AddDownloadClient": "Aggiungi Client di Download",
+    "AddDownloadClient": "Aggiungi Downloader",
     "Category": "Categoria",
     "ClearHistory": "Cancella cronologia",
     "ClearHistoryMessageText": "Sei sicuro di voler cancellare tutta la cronologia di Prowlarr?",
@@ -348,7 +348,7 @@
     "OnApplicationUpdateHelpText": "All'aggiornamento dell'applicazione",
     "OnGrab": "Al Prelievo",
     "OnHealthIssue": "Quando c'è un problema",
-    "TestAllIndexers": "Testa tutti gli Indicizzatori",
+    "TestAllIndexers": "Prova tutti gli indicizzatori",
     "UserAgentProvidedByTheAppThatCalledTheAPI": "User-Agent fornito dalla app che ha chiamato la API",
     "GrabReleases": "Prendi Release(s)",
     "Link": "Collegamenti",
@@ -360,7 +360,7 @@
     "AudioSearch": "Ricerca Audio",
     "BookSearch": "Ricerca Libri",
     "AddRemoveOnly": "Solo Aggiunta e Rimozione",
-    "AppProfileSelectHelpText": "I profili delle App sono utilizzati per controllare la sincronizzazione delle impostazioni RSS, Ricerca Automatica e Ricerca Interattiva",
+    "AppProfileSelectHelpText": "I profili delle App sono usati per controllare l'RSS, la ricerca automatica e ricerca interattiva durante la sincronizzazione dell'app",
     "DeleteApplication": "Cancella Applicazione",
     "DeleteAppProfile": "Cancella Profilo App",
     "AddedToDownloadClient": "Release aggiunta al client",
@@ -383,7 +383,7 @@
     "IndexerVipCheckExpiringClientMessage": "I benefici VIP dell'Indicizzatore scadranno a breve: {0}",
     "IndexerProxies": "Proxy degli Indicizzatori",
     "Stats": "Statistiche",
-    "SyncAppIndexers": "Sincronizza gli Indicizzatori dell'App",
+    "SyncAppIndexers": "Sincronizza tutti gli indicizzatori",
     "SyncLevel": "Livello Sincronizzazione",
     "IndexerProxy": "Proxy dell'Indicizzatore",
     "Proxies": "Proxy",
@@ -392,7 +392,7 @@
     "IndexersSelectedInterp": "{0} Indicizzatore(i) Selezionato(i)",
     "Privacy": "Privacy",
     "SettingsIndexerLogging": "Logging Migliorato dell'Indicizzatore",
-    "TestAllApps": "Testa Tutte le App",
+    "TestAllApps": "Prova Tutte le App",
     "UnableToLoadAppProfiles": "Impossibile caricare i profili delle app",
     "UnableToLoadIndexerProxies": "Impossibile caricare i Proxy degli Indicizzatori",
     "FullSync": "Sincronizzazione completa",
@@ -405,7 +405,7 @@
     "IndexerRss": "RSS dell'Indicizzatore",
     "IndexerSite": "Sito dell'Indicizzatore",
     "MassEditor": "Editor di Massa",
-    "IndexerTagsHelpText": "Usa le etichette per specificare i Proxy degli Indicizzatori, con che app un indicizzatore si sincronizza o semplcemente per organizzarli.",
+    "IndexerTagsHelpText": "Usa le etichette per specificare i Proxy degli Indicizzatori, con che app un indicizzatore si sincronizza o semplicemente per organizzarli.",
     "MovieSearch": "Ricerca Film",
     "MovieSearchTypes": "Tipi di Ricerca Film",
     "MusicSearchTypes": "Tipi di Ricerca Musica",
@@ -440,21 +440,21 @@
     "SettingsIndexerLoggingHelpText": "Logga i dati aggiuntivi dell'indicizzatore includendo la risposta",
     "SettingsSqlLoggingHelpText": "Scrivi a log tutte le query SQL di Prowlarr",
     "SyncLevelAddRemove": "Solo aggiunte e rimozioni: Quando gli indicizzatori vengono aggiunti o rimossi da Prowlarr, verrà aggiornata questa applicazione remota.",
-    "SyncLevelFull": "Sincronizzazione completa: Manterrà gli indicizzatori di questa app completamente sincronizzati. Le modifiche apportate agli indicizzatori in Prowlarr sono poi sincronizzate con questa applicazione. Qualsiasi cambiamento fatto agli indicizzatori da remoto all'interno di questa applicazione sarà sovrascritto da Prowlarr alla prossima sincronizzazione.",
+    "SyncLevelFull": "Sincronizzazione completa: Mantiene gli indicizzatori dell'app completamente sincronizzati. Le modifiche apportate agli indicizzatori in Prowlarr sono sincronizzate in questa app. Qualsiasi cambiamento fatto agli indicizzatori da remoto all'interno di questa applicazione verrà sovrascritto da Prowlarr alla prossima sincronizzazione.",
     "MinimumSeedersHelpText": "Seeder minimi richiesti dall'Applicazione per far sì che l'indicizzatore li prenda",
     "SyncProfile": "Profilo Sincronizzazione",
-    "SyncProfiles": "Profili Sincronizzazione",
+    "SyncProfiles": "Profili di sincronizzazione",
     "AddSyncProfile": "Aggiungi Profilo di Sincronizzazione",
     "EditSyncProfile": "Modifica Profilo di Sincronizzazione",
     "MinimumSeeders": "Seeder Minimi",
     "InstanceName": "Nome Istanza",
-    "InstanceNameHelpText": "Nome dell'istanza nella scheda e per il nome dell'applicazione Syslog",
+    "InstanceNameHelpText": "Nome istanza nella scheda e per il nome dell'app nel Syslog",
     "ThemeHelpText": "Cambia il Tema dell'interfaccia dell’applicazione, il Tema 'Auto' userà il suo Tema di Sistema per impostare la modalità Chiara o Scura. Ispirato da {0}",
     "LastDuration": "Ultima Durata",
     "LastExecution": "Ultima esecuzione",
-    "Queued": "Messo in coda",
-    "ApplicationLongTermStatusCheckAllClientMessage": "Nessun Indicizzatore è disponibile da più di 6 ore a causa di errori",
-    "ApplicationLongTermStatusCheckSingleClientMessage": "Alcuni Indicizzatori non sono disponibili da più di 6 ore a causa di errori: {0}",
+    "Queued": "In coda",
+    "ApplicationLongTermStatusCheckAllClientMessage": "Tutte le app non disponibili da almeno 6 ore a causa di errori",
+    "ApplicationLongTermStatusCheckSingleClientMessage": "Alcune app non sono disponibili da almeno 6 ore a causa di errori: {0}",
     "Duration": "Durata",
     "Ended": "Finito",
     "NextExecution": "Prossima esecuzione",
@@ -462,5 +462,12 @@
     "Parameters": "Parametri",
     "ElapsedTime": "Tempo trascorso",
     "EnabledRedirected": "Abilitato, Reindirizzato",
-    "Started": "Iniziato"
+    "Started": "Iniziato",
+    "AreYouSureYouWantToDeleteCategory": "Vuoi davvero cancellare la categoria mappata?",
+    "AuthenticationRequired": "Autenticazione richiesta",
+    "AuthenticationRequiredHelpText": "Cambia a quali richieste l'autenticazione verrà richiesta. Non cambiare se non comprendi i rischi.",
+    "AuthenticationRequiredWarning": "Per impedire accessi remoti senza autenticazione, Prowlarr ora richiede che l'autenticazione venga attivata. Puoi disattivare l'autenticazione per indirizzi privati.",
+    "DeleteClientCategory": "Cancella categoria del client di download",
+    "DownloadClientCategory": "Categoria client di download",
+    "MappedCategories": "Categorie mappate"
 }

src/NzbDrone.Core/Localization/Core/pt_BR.json

@@ -229,7 +229,7 @@
     "Reset": "Redefinir",
     "Restart": "Reiniciar",
     "RestartNow": "Reiniciar agora",
-    "RestartRequiredHelpTextWarning": "Requer reinício para ter efeito",
+    "RestartRequiredHelpTextWarning": "Requer reinicialização para ter efeito",
     "Restore": "Restaurar",
     "RestoreBackup": "Restaurar backup",
     "Result": "Resultado",

src/NzbDrone.Core/Parser/DateTimeUtil.cs

@@ -109,7 +109,7 @@ namespace NzbDrone.Core.Parser
             if (DateTimeRoutines.TryParseDateOrTime(
                 str, dtFormat, out DateTimeRoutines.ParsedDateTime dt))
             {
-                return dt.DateTime.ToUniversalTime();
+                return dt.DateTime;
             }
 
             throw new InvalidDateException($"FromFuzzyTime parsing failed for string {str}");

src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs

@@ -10,7 +10,7 @@ namespace NzbDrone.Core.Parser.Model
     {
         public ReleaseInfo()
         {
-            IndexerFlags = new List<IndexerFlag>();
+            IndexerFlags = new HashSet<IndexerFlag>();
             Categories = new List<IndexerCategory>();
             Languages = new List<string>();
             Subs = new List<string>();
@@ -57,11 +57,11 @@ namespace NzbDrone.Core.Parser.Model
         public ICollection<string> Languages { get; set; }
         public ICollection<string> Subs { get; set; }
         public ICollection<IndexerCategory> Categories { get; set; }
-        public ICollection<IndexerFlag> IndexerFlags { get; set; }
+        public HashSet<IndexerFlag> IndexerFlags { get; set; }
 
         public int Age
         {
-            get { return DateTime.UtcNow.Subtract(PublishDate).Days; }
+            get { return DateTime.UtcNow.Subtract(PublishDate.ToUniversalTime()).Days; }
 
             //This prevents manually downloading a release from blowing up in mono
             //TODO: Is there a better way?
@@ -70,7 +70,7 @@ namespace NzbDrone.Core.Parser.Model
 
         public double AgeHours
         {
-            get { return DateTime.UtcNow.Subtract(PublishDate).TotalHours; }
+            get { return DateTime.UtcNow.Subtract(PublishDate.ToUniversalTime()).TotalHours; }
 
             //This prevents manually downloading a release from blowing up in mono
             //TODO: Is there a better way?
@@ -79,7 +79,7 @@ namespace NzbDrone.Core.Parser.Model
 
         public double AgeMinutes
         {
-            get { return DateTime.UtcNow.Subtract(PublishDate).TotalMinutes; }
+            get { return DateTime.UtcNow.Subtract(PublishDate.ToUniversalTime()).TotalMinutes; }
 
             //This prevents manually downloading a release from blowing up in mono
             //TODO: Is there a better way?