Fixed: (Indexers) Rate limit auth requests

src/NzbDrone.Core/Download/DownloadService.cs

@@ -61,14 +61,6 @@ namespace NzbDrone.Core.Download
 
             // Get the seed configuration for this release.
             // remoteMovie.SeedConfiguration = _seedConfigProvider.GetSeedConfiguration(remoteMovie);
-
-            // Limit grabs to 2 per second.
-            if (release.DownloadUrl.IsNotNullOrWhiteSpace() && !release.DownloadUrl.StartsWith("magnet:"))
-            {
-                var url = new HttpUri(release.DownloadUrl);
-                _rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2));
-            }
-
             var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(release.IndexerId));
 
             string downloadClientId;

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -180,60 +180,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
             await generator.DoLogin();
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override async Task<HttpRequest> GetDownloadRequest(Uri link)
         {
             var generator = (CardigannRequestGenerator)GetRequestGenerator();
 
             var request = await generator.DownloadRequest(link);
 
-            if (request.Url.Scheme == "magnet")
-            {
-                ValidateMagnet(request.Url.FullUri);
-                return Encoding.UTF8.GetBytes(request.Url.FullUri);
-            }
-
             request.AllowAutoRedirect = true;
 
-            var downloadBytes = Array.Empty<byte>();
+            return request;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                downloadBytes = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", request.Url.FullUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            return downloadBytes;
         }
 
         protected override async Task Test(List<ValidationFailure> failures)

src/NzbDrone.Core/Indexers/Definitions/Redacted.cs

@@ -80,26 +80,18 @@ namespace NzbDrone.Core.Indexers.Definitions
             return caps;
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
         {
-            var request = new HttpRequestBuilder(link.AbsoluteUri)
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-                .SetHeader("Authorization", Settings.Apikey)
-                .Build();
 
-            var downloadBytes = Array.Empty<byte>();
+            if (Cookies != null)
-
-            try
             {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
+                requestBuilder.SetCookies(Cookies);
-                downloadBytes = response.ResponseData;
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Download failed");
             }
 
-            return downloadBytes;
+            var request = requestBuilder.SetHeader("Authorization", Settings.Apikey).Build();
+
+            return Task.FromResult(request);
         }
     }
 

src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs

@@ -103,16 +103,8 @@ namespace NzbDrone.Core.Indexers.Definitions
             request.HttpRequest.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
         {
-            Cookies = GetCookies();
-
-            if (link.Scheme == "magnet")
-            {
-                ValidateMagnet(link.OriginalString);
-                return Encoding.UTF8.GetBytes(link.OriginalString);
-            }
-
             var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
 
             if (Cookies != null)
@@ -124,46 +116,7 @@ namespace NzbDrone.Core.Indexers.Definitions
             request.AllowAutoRedirect = FollowRedirect;
             request.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
 
-            byte[] torrentData;
+            return Task.FromResult(request);
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                torrentData = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            return torrentData;
         }
 
         protected virtual IndexerCapabilities SetCapabilities()

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -5,10 +5,12 @@ using System.Net;
 using System.Text;
 using System.Threading.Tasks;
 using FluentValidation.Results;
+using MonoTorrent;
 using NLog;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Http.CloudFlare;
 using NzbDrone.Core.Indexers.Events;
 using NzbDrone.Core.Indexers.Exceptions;
@@ -100,6 +102,93 @@ namespace NzbDrone.Core.Indexers
             return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
         }
 
+        public override async Task<byte[]> Download(Uri link)
+        {
+            Cookies = GetCookies();
+
+            var request = await GetDownloadRequest(link);
+
+            if (request.Url.Scheme == "magnet")
+            {
+                ValidateMagnet(request.Url.FullUri);
+                return Encoding.UTF8.GetBytes(request.Url.FullUri);
+            }
+
+            if (request.RateLimit < RateLimit)
+            {
+                request.RateLimit = RateLimit;
+            }
+
+            byte[] fileData;
+
+            try
+            {
+                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
+                fileData = response.ResponseData;
+
+                _logger.Debug("Downloaded for release finished ({0} bytes from {1})", fileData.Length, link.AbsoluteUri);
+            }
+            catch (HttpException ex)
+            {
+                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
+                {
+                    _logger.Error(ex, "Downloading file for release failed since it no longer exists ({0})", link.AbsoluteUri);
+                    throw new ReleaseUnavailableException("Downloading nzb failed", ex);
+                }
+
+                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
+                {
+                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
+                }
+                else
+                {
+                    _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
+                }
+
+                throw new ReleaseDownloadException("Downloading failed", ex);
+            }
+            catch (WebException ex)
+            {
+                _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
+
+                throw new ReleaseDownloadException("Downloading failed", ex);
+            }
+            catch (Exception)
+            {
+                _indexerStatusService.RecordFailure(Definition.Id);
+                _logger.Error("Downloading failed");
+                throw;
+            }
+
+            ValidateDownloadData(fileData);
+
+            return fileData;
+        }
+
+        protected virtual Task<HttpRequest> GetDownloadRequest(Uri link)
+        {
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
+
+            if (Cookies != null)
+            {
+                requestBuilder.SetCookies(Cookies);
+            }
+
+            var request = requestBuilder.Build();
+            request.AllowAutoRedirect = FollowRedirect;
+
+            return Task.FromResult(request);
+        }
+
+        protected virtual void ValidateDownloadData(byte[] fileData)
+        {
+        }
+
+        protected void ValidateMagnet(string link)
+        {
+            MagnetLink.Parse(link);
+        }
+
         protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator)
         {
             //A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page.
@@ -404,6 +493,11 @@ namespace NzbDrone.Core.Indexers
         {
             request.Encoding = Encoding;
 
+            if (request.RateLimit < RateLimit)
+            {
+                request.RateLimit = RateLimit;
+            }
+
             var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
 
             _eventAggregator.PublishEvent(new IndexerAuthEvent(Definition.Id, !response.HasHttpError, response.ElapsedTime));

src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs

@@ -1,12 +1,5 @@
-using System;
-using System.Net;
-using System.Text;
-using System.Threading.Tasks;
-using MonoTorrent;
 using NLog;
-using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Messaging.Events;
 
 namespace NzbDrone.Core.Indexers
@@ -18,72 +11,5 @@ namespace NzbDrone.Core.Indexers
             : base(httpClient, eventAggregator, indexerStatusService, configService, logger)
         {
         }
-
-        public override async Task<byte[]> Download(Uri link)
-        {
-            Cookies = GetCookies();
-
-            if (link.Scheme == "magnet")
-            {
-                ValidateMagnet(link.OriginalString);
-                return Encoding.UTF8.GetBytes(link.OriginalString);
-            }
-
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-
-            byte[] torrentData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                torrentData = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            return torrentData;
-        }
-
-        protected void ValidateMagnet(string link)
-        {
-            MagnetLink.Parse(link);
-        }
     }
 }

src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs

@@ -1,11 +1,6 @@
-using System;
-using System.Net;
-using System.Threading.Tasks;
 using NLog;
-using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Download;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Messaging.Events;
 
 namespace NzbDrone.Core.Indexers
@@ -21,64 +16,9 @@ namespace NzbDrone.Core.Indexers
             _nzbValidationService = nzbValidationService;
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override void ValidateDownloadData(byte[] fileData)
         {
-            Cookies = GetCookies();
+            _nzbValidationService.Validate(fileData);
-
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-
-            byte[] nzbData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                nzbData = response.ResponseData;
-
-                _logger.Debug("Downloaded nzb for release finished ({0} bytes from {1})", nzbData.Length, link.AbsoluteUri);
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading nzb file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading nzb failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading nzb failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading nzb failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading nzb failed");
-                throw;
-            }
-
-            _nzbValidationService.Validate(nzbData);
-
-            return nzbData;
         }
     }
 }