Fixed: (Indexers) Rate limit for download and auth

azure-pipelines.yml

@@ -9,7 +9,7 @@ variables:
   testsFolder: './_tests'
   yarnCacheFolder: $(Pipeline.Workspace)/.yarn
   nugetCacheFolder: $(Pipeline.Workspace)/.nuget/packages
-  majorVersion: '1.2.1'
+  majorVersion: '1.2.2'
   minorVersion: $[counter('minorVersion', 1)]
   prowlarrVersion: '$(majorVersion).$(minorVersion)'
   buildName: '$(Build.SourceBranchName).$(prowlarrVersion)'

src/NzbDrone.Common/Http/TooManyRequestsException.cs

@@ -11,15 +11,13 @@ namespace NzbDrone.Common.Http
         {
             if (response.Headers.ContainsKey("Retry-After"))
             {
-                var retryAfter = response.Headers["Retry-After"].ToString();
-                int seconds;
-                DateTime date;
+                var retryAfter = response.Headers["Retry-After"];
 
-                if (int.TryParse(retryAfter, out seconds))
+                if (int.TryParse(retryAfter, out var seconds))
                 {
                     RetryAfter = TimeSpan.FromSeconds(seconds);
                 }
-                else if (DateTime.TryParse(retryAfter, out date))
+                else if (DateTime.TryParse(retryAfter, out var date))
                 {
                     RetryAfter = date.ToUniversalTime() - DateTime.UtcNow;
                 }

src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs

@@ -53,7 +53,7 @@ namespace NzbDrone.Core.Test.IndexerTests.RarbgTests
             torrentInfo.Title.Should().Be("Sense8.S01E01.WEBRip.x264-FGT");
             torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent);
             torrentInfo.DownloadUrl.Should().Be("magnet:?xt=urn:btih:d8bde635f573acb390c7d7e7efc1556965fdc802&dn=Sense8.S01E01.WEBRip.x264-FGT&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce");
-            torrentInfo.InfoUrl.Should().Be($"https://torrentapi.org/redirect_to_info.php?token=i5cx7b9agd&p=8_6_4_4_5_6__d8bde635f5&app_id={BuildInfo.AppName}");
+            torrentInfo.InfoUrl.Should().Be($"https://torrentapi.org/redirect_to_info.php?token=i5cx7b9agd&p=8_6_4_4_5_6__d8bde635f5&app_id={BuildInfo.AppName}_{BuildInfo.Version}");
             torrentInfo.Indexer.Should().Be(Subject.Definition.Name);
             torrentInfo.PublishDate.Should().Be(DateTime.Parse("2015-06-05 16:58:11 +0000").ToUniversalTime());
             torrentInfo.Size.Should().Be(564198371);

src/NzbDrone.Core/Download/DownloadService.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Threading.Tasks;
 using NLog;
-using NzbDrone.Common.EnsureThat;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Common.Instrumentation.Extensions;
@@ -61,14 +60,6 @@ namespace NzbDrone.Core.Download
 
             // Get the seed configuration for this release.
             // remoteMovie.SeedConfiguration = _seedConfigProvider.GetSeedConfiguration(remoteMovie);
-
-            // Limit grabs to 2 per second.
-            if (release.DownloadUrl.IsNotNullOrWhiteSpace() && !release.DownloadUrl.StartsWith("magnet:"))
-            {
-                var url = new HttpUri(release.DownloadUrl);
-                _rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2));
-            }
-
             var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(release.IndexerId));
 
             string downloadClientId;
@@ -92,8 +83,7 @@ namespace NzbDrone.Core.Download
             }
             catch (ReleaseDownloadException ex)
             {
-                var http429 = ex.InnerException as TooManyRequestsException;
-                if (http429 != null)
+                if (ex.InnerException is TooManyRequestsException http429)
                 {
                     _indexerStatusService.RecordFailure(release.IndexerId, http429.RetryAfter);
                 }
@@ -141,8 +131,7 @@ namespace NzbDrone.Core.Download
             }
             catch (ReleaseDownloadException ex)
             {
-                var http429 = ex.InnerException as TooManyRequestsException;
-                if (http429 != null)
+                if (ex.InnerException is TooManyRequestsException http429)
                 {
                     _indexerStatusService.RecordFailure(indexerId, http429.RetryAfter);
                 }

src/NzbDrone.Core/History/HistoryRepository.cs

@@ -19,6 +19,7 @@ namespace NzbDrone.Core.History
         List<History> Since(DateTime date, HistoryEventType? eventType);
         void Cleanup(int days);
         int CountSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes);
+        History FindFirstForIndexerSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes, int limit);
     }
 
     public class HistoryRepository : BasicRepository<History>, IHistoryRepository
@@ -115,5 +116,24 @@ namespace NzbDrone.Core.History
                 return conn.ExecuteScalar<int>(sql.RawSql, sql.Parameters);
             }
         }
+
+        public History FindFirstForIndexerSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes, int limit)
+        {
+            var intEvents = eventTypes.Select(t => (int)t).ToList();
+
+            var builder = Builder()
+                .Where<History>(x => x.IndexerId == indexerId)
+                .Where<History>(x => x.Date >= date)
+                .Where<History>(x => intEvents.Contains((int)x.EventType));
+
+            var query = Query(builder);
+
+            if (limit > 0)
+            {
+                query = query.OrderByDescending(h => h.Date).Take(limit).ToList();
+            }
+
+            return query.MinBy(h => h.Date);
+        }
     }
 }

src/NzbDrone.Core/History/HistoryService.cs

@@ -27,6 +27,7 @@ namespace NzbDrone.Core.History
         List<History> Between(DateTime start, DateTime end);
         List<History> Since(DateTime date, HistoryEventType? eventType);
         int CountSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes);
+        History FindFirstForIndexerSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes, int limit);
     }
 
     public class HistoryService : IHistoryService,
@@ -232,5 +233,10 @@ namespace NzbDrone.Core.History
         {
             return _historyRepository.CountSince(indexerId, date, eventTypes);
         }
+
+        public History FindFirstForIndexerSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes, int limit)
+        {
+            return _historyRepository.FindFirstForIndexerSince(indexerId, date, eventTypes, limit);
+        }
     }
 }

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -1,15 +1,12 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Net;
-using System.Text;
 using System.Threading.Tasks;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.IndexerVersions;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.ThingiProvider;
@@ -53,7 +50,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
             var generator = _generatorCache.Get(Settings.DefinitionFile, () =>
                 new CardigannRequestGenerator(_configService,
                     _definitionService.GetCachedDefinition(Settings.DefinitionFile),
-                    _logger)
+                    _logger,
+                    RateLimit)
                 {
                     HttpClient = _httpClient,
                     Definition = Definition,
@@ -180,63 +178,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
             await generator.DoLogin();
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override async Task<HttpRequest> GetDownloadRequest(Uri link)
         {
             var generator = (CardigannRequestGenerator)GetRequestGenerator();
 
             var request = await generator.DownloadRequest(link);
 
-            if (request.Url.Scheme == "magnet")
-            {
-                ValidateMagnet(request.Url.FullUri);
-
-                return Encoding.UTF8.GetBytes(request.Url.FullUri);
-            }
-
             request.AllowAutoRedirect = true;
 
-            var downloadBytes = Array.Empty<byte>();
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                downloadBytes = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", request.Url.FullUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            ValidateTorrent(downloadBytes);
-
-            return downloadBytes;
+            return request;
         }
 
         protected override async Task Test(List<ValidationFailure> failures)

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs

@@ -300,7 +300,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 }
                 else if (setting.Type == "checkbox")
                 {
-                    variables[name] = ((bool)value) ? ".True" : null;
+                    if (value is string stringValue && bool.TryParse(stringValue, out var result))
+                    {
+                        value = result;
+                    }
+
+                    variables[name] = (bool)value ? ".True" : null;
                 }
                 else if (setting.Type == "select")
                 {
@@ -328,12 +333,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 }
                 else
                 {
-                    throw new NotSupportedException();
+                    throw new NotSupportedException($"Type {setting.Type} is not supported.");
                 }
 
-                if (setting.Type != "password" && setting.Name != "apikey" && setting.Name != "rsskey" && indexerLogging)
+                if (setting.Type != "password" && setting.Name != "apikey" && setting.Name != "rsskey" && indexerLogging && variables.ContainsKey(name))
                 {
-                    _logger.Debug($"Setting {setting.Name} to {variables[name]}");
+                    _logger.Debug($"Setting {setting.Name} to {variables[name].ToJson()}");
                 }
             }
 

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs

@@ -4,7 +4,6 @@ using System.Collections.Specialized;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
-using System.Text;
 using System.Threading.Tasks;
 using AngleSharp.Html.Dom;
 using AngleSharp.Html.Parser;
@@ -29,11 +28,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
         protected IHtmlDocument landingResultDocument;
         protected override string SiteLink => ResolveSiteLink();
 
+        private readonly TimeSpan _rateLimit;
+
         public CardigannRequestGenerator(IConfigService configService,
                                          CardigannDefinition definition,
-                                         Logger logger)
+                                         Logger logger,
+                                         TimeSpan rateLimit)
         : base(configService, definition, logger)
         {
+            _rateLimit = rateLimit;
         }
 
         public Func<IDictionary<string, string>> GetCookies { get; set; }
@@ -218,9 +221,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     requestBuilder.AddFormParameter(pair.Key, pair.Value);
                 }
 
-                requestBuilder.Headers.Add("Referer", SiteLink);
+                var request = requestBuilder
+                    .SetHeader("Referer", SiteLink)
+                    .WithRateLimit(_rateLimit.TotalSeconds)
+                    .Build();
 
-                var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
                 Cookies = response.GetCookies();
 
@@ -356,11 +362,13 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         Encoding = _encoding
                     };
 
-                    requestBuilder.SetCookies(Cookies);
+                    var request = requestBuilder
+                        .SetCookies(Cookies)
+                        .SetHeader("Referer", loginUrl)
+                        .WithRateLimit(_rateLimit.TotalSeconds)
+                        .Build();
 
-                    requestBuilder.Headers.Add("Referer", loginUrl);
-
-                    var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                    var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
                     var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.Content);
                     var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString();
@@ -424,10 +432,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         Encoding = _encoding
                     };
 
-                    requestBuilder.Headers.Add("Referer", SiteLink);
-
-                    requestBuilder.SetCookies(Cookies);
-
                     foreach (var pair in pairs)
                     {
                         requestBuilder.AddFormParameter(pair.Key, pair.Value);
@@ -438,7 +442,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         requestBuilder.SetHeader(header.Key, header.Value);
                     }
 
-                    var request = requestBuilder.Build();
+                    var request = requestBuilder
+                        .SetCookies(Cookies)
+                        .SetHeader("Referer", SiteLink)
+                        .WithRateLimit(_rateLimit.TotalSeconds)
+                        .Build();
+
                     request.SetContent(body);
 
                     loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
@@ -454,15 +463,18 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         Encoding = _encoding
                     };
 
-                    requestBuilder.SetCookies(Cookies);
-                    requestBuilder.Headers.Add("Referer", loginUrl);
-
                     foreach (var pair in pairs)
                     {
                         requestBuilder.AddFormParameter(pair.Key, pair.Value);
                     }
 
-                    loginResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                    var request = requestBuilder
+                        .SetCookies(Cookies)
+                        .SetHeader("Referer", loginUrl)
+                        .WithRateLimit(_rateLimit.TotalSeconds)
+                        .Build();
+
+                    loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
                 }
 
                 Cookies = loginResult.GetCookies();
@@ -496,9 +508,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     Encoding = _encoding
                 };
 
-                requestBuilder.Headers.Add("Referer", SiteLink);
+                var request = requestBuilder
+                    .SetHeader("Referer", SiteLink)
+                    .WithRateLimit(_rateLimit.TotalSeconds)
+                    .Build();
 
-                var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
                 Cookies = response.GetCookies();
 
@@ -521,9 +536,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     Encoding = _encoding
                 };
 
-                requestBuilder.Headers.Add("Referer", SiteLink);
+                var request = requestBuilder
+                    .SetHeader("Referer", SiteLink)
+                    .WithRateLimit(_rateLimit.TotalSeconds)
+                    .Build();
 
-                var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
                 Cookies = response.GetCookies();
 
@@ -594,14 +612,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 Encoding = _encoding
             };
 
-            requestBuilder.Headers.Add("Referer", SiteLink);
-
             if (Cookies != null)
             {
                 requestBuilder.SetCookies(Cookies);
             }
 
-            var request = requestBuilder.Build();
+            var request = requestBuilder
+                .SetHeader("Referer", SiteLink)
+                .WithRateLimit(_rateLimit.TotalSeconds)
+                .Build();
 
             landingResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
@@ -646,6 +665,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         .SetCookies(landingResult.GetCookies())
                         .SetHeader("Referer", loginUrl.AbsoluteUri)
                         .SetEncoding(_encoding)
+                        .WithRateLimit(_rateLimit.TotalSeconds)
                         .Build();
 
                     var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
@@ -656,10 +676,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         ImageData = response.ResponseData
                     };
                 }
-                else
-                {
-                    _logger.Debug("CardigannIndexer ({0}): No captcha image found", _definition.Id);
-                }
+
+                _logger.Debug("CardigannIndexer ({0}): No captcha image found", _definition.Id);
             }
             else
             {
@@ -724,23 +742,28 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 requestLinkStr += queryCollection.GetQueryString(_encoding, separator: request.Queryseparator);
             }
 
-            var httpRequest = new HttpRequestBuilder(requestLinkStr)
-                .SetCookies(Cookies ?? new Dictionary<string, string>())
-                .SetEncoding(_encoding)
-                .SetHeader("Referer", referer);
-
-            httpRequest.Method = method;
+            var httpRequestBuilder = new HttpRequestBuilder(requestLinkStr)
+            {
+                Method = method,
+                Encoding = _encoding
+            };
 
             // Add form data for POST requests
             if (method == HttpMethod.Post)
             {
                 foreach (var param in pairs)
                 {
-                    httpRequest.AddFormParameter(param.Key, param.Value);
+                    httpRequestBuilder.AddFormParameter(param.Key, param.Value);
                 }
             }
 
-            var response = await HttpClient.ExecuteProxiedAsync(httpRequest.Build(), Definition);
+            var httpRequest = httpRequestBuilder
+                .SetCookies(Cookies ?? new Dictionary<string, string>())
+                .SetHeader("Referer", referer)
+                .WithRateLimit(_rateLimit.TotalSeconds)
+                .Build();
+
+            var response = await HttpClient.ExecuteProxiedAsync(httpRequest, Definition);
 
             _logger.Debug("CardigannIndexer ({0}): handleRequest() remote server returned {1}", _definition.Id, response.StatusCode);
             return response;
@@ -766,6 +789,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     .SetCookies(Cookies ?? new Dictionary<string, string>())
                     .SetHeaders(headers ?? new Dictionary<string, string>())
                     .SetEncoding(_encoding)
+                    .WithRateLimit(_rateLimit.TotalSeconds)
                     .Build();
 
                 request.AllowAutoRedirect = true;
@@ -856,6 +880,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                                     .SetCookies(Cookies ?? new Dictionary<string, string>())
                                     .SetHeaders(headers ?? new Dictionary<string, string>())
                                     .SetEncoding(_encoding)
+                                    .WithRateLimit(_rateLimit.TotalSeconds)
                                     .Build();
 
                                 response = await HttpClient.ExecuteProxiedAsync(testLinkRequest, Definition);
@@ -875,6 +900,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                                 .SetCookies(Cookies ?? new Dictionary<string, string>())
                                 .SetHeaders(headers ?? new Dictionary<string, string>())
                                 .SetEncoding(_encoding)
+                                .WithRateLimit(_rateLimit.TotalSeconds)
                                 .Build();
 
                             selectorDownloadRequest.Method = method;
@@ -895,6 +921,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 .SetCookies(Cookies ?? new Dictionary<string, string>())
                 .SetHeaders(headers ?? new Dictionary<string, string>())
                 .SetEncoding(_encoding)
+                .WithRateLimit(_rateLimit.TotalSeconds)
                 .Build();
 
             downloadRequest.Method = method;
@@ -1096,16 +1123,18 @@ namespace NzbDrone.Core.Indexers.Cardigann
 
                 _logger.Info($"Adding request: {searchUrl}");
 
-                var requestbuilder = new HttpRequestBuilder(searchUrl);
-
-                requestbuilder.Method = method;
+                var requestBuilder = new HttpRequestBuilder(searchUrl)
+                {
+                    Method = method,
+                    Encoding = _encoding
+                };
 
                 // Add FormData for searchs that POST
                 if (method == HttpMethod.Post)
                 {
                     foreach (var param in queryCollection)
                     {
-                        requestbuilder.AddFormParameter(param.Key, param.Value);
+                        requestBuilder.AddFormParameter(param.Key, param.Value);
                     }
                 }
 
@@ -1113,14 +1142,22 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 if (search.Headers != null)
                 {
                     var headers = ParseCustomHeaders(search.Headers, variables);
-                    requestbuilder.SetHeaders(headers ?? new Dictionary<string, string>());
+                    requestBuilder.SetHeaders(headers ?? new Dictionary<string, string>());
                 }
 
-                var request = new CardigannRequest(requestbuilder.SetEncoding(_encoding).Build(), variables, searchPath);
+                var request = requestBuilder
+                    .WithRateLimit(_rateLimit.TotalSeconds)
+                    .Build();
 
-                request.HttpRequest.AllowAutoRedirect = searchPath.Followredirect;
+                var cardigannRequest = new CardigannRequest(request, variables, searchPath)
+                    {
+                        HttpRequest =
+                        {
+                            AllowAutoRedirect = searchPath.Followredirect
+                        }
+                    };
 
-                yield return request;
+                yield return cardigannRequest;
             }
         }
     }

src/NzbDrone.Core/Indexers/Definitions/Nebulance.cs

@@ -46,12 +46,17 @@ namespace NzbDrone.Core.Indexers.Definitions
             return new NebulanceParser(Settings);
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
         {
-            // Invalidate cookies before downloading to prevent redirect to login page.
-            UpdateCookies(null, null);
+            // Avoid using cookies to prevent redirects to login page
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
+            {
+                AllowAutoRedirect = FollowRedirect
+            };
 
-            return await base.Download(link);
+            var request = requestBuilder.Build();
+
+            return Task.FromResult(request);
         }
 
         private IndexerCapabilities SetCapabilities()

src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs

@@ -112,7 +112,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                 _logger.Error("Download failed");
             }
 
-            ValidateTorrent(downloadBytes);
+            ValidateDownloadData(downloadBytes);
 
             return downloadBytes;
         }

src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs

@@ -151,7 +151,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
                 Settings.Validate().Filter("BaseUrl").ThrowOnError();
 
                 var request = new HttpRequestBuilder(Settings.BaseUrl.Trim('/'))
-                    .Resource($"/pubapi_v2.php?get_token=get_token&app_id={BuildInfo.AppName}")
+                    .Resource($"/pubapi_v2.php?get_token=get_token&app_id={BuildInfo.AppName}_{BuildInfo.Version}")
                     .Accept(HttpAccept.Json)
                     .Build();
 

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgParser.cs

@@ -84,7 +84,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
                     Title = torrent.title,
                     Size = torrent.size,
                     DownloadUrl = torrent.download,
-                    InfoUrl = $"{torrent.info_page}&app_id={BuildInfo.AppName}",
+                    InfoUrl = $"{torrent.info_page}&app_id={BuildInfo.AppName}_{BuildInfo.Version}",
                     PublishDate = torrent.pubdate.ToUniversalTime(),
                     Seeders = torrent.seeders,
                     Peers = torrent.leechers + torrent.seeders,

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs

@@ -61,7 +61,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
             requestBuilder.AddQueryParam("limit", "100");
             requestBuilder.AddQueryParam("token", _tokenProvider.GetToken(Settings));
             requestBuilder.AddQueryParam("format", "json_extended");
-            requestBuilder.AddQueryParam("app_id", BuildInfo.AppName);
+            requestBuilder.AddQueryParam("app_id", $"{BuildInfo.AppName}_{BuildInfo.Version}");
 
             yield return new IndexerRequest(requestBuilder.Build());
         }

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgTokenProvider.cs

@@ -38,7 +38,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
                 {
                     var requestBuilder = new HttpRequestBuilder(settings.BaseUrl.Trim('/'))
                         .WithRateLimit(5.0)
-                        .Resource($"/pubapi_v2.php?get_token=get_token&app_id={BuildInfo.AppName}")
+                        .Resource($"/pubapi_v2.php?get_token=get_token&app_id={BuildInfo.AppName}_{BuildInfo.Version}")
                         .Accept(HttpAccept.Json);
 
                     var response = _httpClient.Get<JObject>(requestBuilder.Build());

src/NzbDrone.Core/Indexers/Definitions/Redacted.cs

@@ -50,6 +50,20 @@ namespace NzbDrone.Core.Indexers.Definitions
             return new RedactedParser(Settings, Capabilities.Categories);
         }
 
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
+        {
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
+            {
+                AllowAutoRedirect = FollowRedirect
+            };
+
+            var request = requestBuilder
+                .SetHeader("Authorization", Settings.Apikey)
+                .Build();
+
+            return Task.FromResult(request);
+        }
+
         private IndexerCapabilities SetCapabilities()
         {
             var caps = new IndexerCapabilities
@@ -74,30 +88,6 @@ namespace NzbDrone.Core.Indexers.Definitions
 
             return caps;
         }
-
-        public override async Task<byte[]> Download(Uri link)
-        {
-            var request = new HttpRequestBuilder(link.AbsoluteUri)
-                .SetHeader("Authorization", Settings.Apikey)
-                .Build();
-
-            var downloadBytes = Array.Empty<byte>();
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                downloadBytes = response.ResponseData;
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Download failed");
-            }
-
-            ValidateTorrent(downloadBytes);
-
-            return downloadBytes;
-        }
     }
 
     public class RedactedRequestGenerator : IIndexerRequestGenerator
@@ -188,18 +178,12 @@ namespace NzbDrone.Core.Indexers.Definitions
                 queryCats.ForEach(cat => parameters.Set($"filter_cat[{cat}]", "1"));
             }
 
-            var request = RequestBuilder()
-                .Resource($"/ajax.php?{parameters.GetQueryString()}")
-                .Build();
+            var searchUrl = _settings.BaseUrl.TrimEnd('/') + $"/ajax.php?{parameters.GetQueryString()}";
 
-            yield return new IndexerRequest(request);
-        }
+            var request = new IndexerRequest(searchUrl, HttpAccept.Json);
+            request.HttpRequest.Headers.Set("Authorization", _settings.Apikey);
 
-        private HttpRequestBuilder RequestBuilder()
-        {
-            return new HttpRequestBuilder($"{_settings.BaseUrl.TrimEnd('/')}")
-                .Accept(HttpAccept.Json)
-                .SetHeader("Authorization", _settings.Apikey);
+            yield return request;
         }
     }
 

src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs

@@ -15,7 +15,6 @@ using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Annotations;
 using NzbDrone.Core.Configuration;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.Indexers.Settings;
 using NzbDrone.Core.IndexerSearch.Definitions;
@@ -104,70 +103,18 @@ namespace NzbDrone.Core.Indexers.Definitions
             request.HttpRequest.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
         {
-            Cookies = GetCookies();
-
-            if (link.Scheme == "magnet")
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
             {
-                ValidateMagnet(link.OriginalString);
+                AllowAutoRedirect = FollowRedirect
+            };
 
-                return Encoding.UTF8.GetBytes(link.OriginalString);
-            }
+            var request = requestBuilder
+                .SetHeader("Authorization", $"Bearer {Settings.ApiKey}")
+                .Build();
 
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-            request.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
-
-            byte[] torrentData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                torrentData = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            ValidateTorrent(torrentData);
-
-            return torrentData;
+            return Task.FromResult(request);
         }
 
         protected virtual IndexerCapabilities SetCapabilities()
@@ -276,7 +223,6 @@ namespace NzbDrone.Core.Indexers.Definitions
             var searchUrl = _settings.BaseUrl + "api/torrent?" + parameters.GetQueryString(duplicateKeysIfMulti: true);
 
             var request = new IndexerRequest(searchUrl, HttpAccept.Json);
-
             request.HttpRequest.Headers.Set("Authorization", $"Bearer {_settings.ApiKey}");
 
             yield return request;

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -6,10 +6,12 @@ using System.Net.Http;
 using System.Text;
 using System.Threading.Tasks;
 using FluentValidation.Results;
+using MonoTorrent;
 using NLog;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Http.CloudFlare;
 using NzbDrone.Core.Indexers.Events;
 using NzbDrone.Core.Indexers.Exceptions;
@@ -101,6 +103,95 @@ namespace NzbDrone.Core.Indexers
             return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
         }
 
+        public override async Task<byte[]> Download(Uri link)
+        {
+            Cookies = GetCookies();
+
+            var request = await GetDownloadRequest(link);
+
+            if (request.Url.Scheme == "magnet")
+            {
+                ValidateMagnet(request.Url.FullUri);
+                return Encoding.UTF8.GetBytes(request.Url.FullUri);
+            }
+
+            if (request.RateLimit < RateLimit)
+            {
+                request.RateLimit = RateLimit;
+            }
+
+            byte[] fileData;
+
+            try
+            {
+                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
+                fileData = response.ResponseData;
+
+                _logger.Debug("Downloaded for release finished ({0} bytes from {1})", fileData.Length, link.AbsoluteUri);
+            }
+            catch (HttpException ex)
+            {
+                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
+                {
+                    _logger.Error(ex, "Downloading file for release failed since it no longer exists ({0})", link.AbsoluteUri);
+                    throw new ReleaseUnavailableException("Download failed", ex);
+                }
+
+                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
+                {
+                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
+                }
+                else
+                {
+                    _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
+                }
+
+                throw new ReleaseDownloadException("Download failed", ex);
+            }
+            catch (WebException ex)
+            {
+                _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
+
+                throw new ReleaseDownloadException("Download failed", ex);
+            }
+            catch (Exception)
+            {
+                _indexerStatusService.RecordFailure(Definition.Id);
+                _logger.Error("Download failed");
+                throw;
+            }
+
+            ValidateDownloadData(fileData);
+
+            return fileData;
+        }
+
+        protected virtual Task<HttpRequest> GetDownloadRequest(Uri link)
+        {
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
+            {
+                AllowAutoRedirect = FollowRedirect
+            };
+
+            if (Cookies != null)
+            {
+                requestBuilder.SetCookies(Cookies);
+            }
+
+            var request = requestBuilder.Build();
+
+            return Task.FromResult(request);
+        }
+
+        protected virtual void ValidateDownloadData(byte[] fileData)
+        {
+        }
+
+        protected void ValidateMagnet(string link)
+        {
+            MagnetLink.Parse(link);
+        }
+
         protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator)
         {
             //A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page.
@@ -420,6 +511,11 @@ namespace NzbDrone.Core.Indexers
                 request.RequestTimeout = TimeSpan.FromSeconds(15);
             }
 
+            if (request.RateLimit < RateLimit)
+            {
+                request.RateLimit = RateLimit;
+            }
+
             var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
 
             _eventAggregator.PublishEvent(new IndexerAuthEvent(Definition.Id, !response.HasHttpError, response.ElapsedTime));

src/NzbDrone.Core/Indexers/IndexerLimitService.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using NLog;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Core.History;
 
 namespace NzbDrone.Core.Indexers
@@ -10,6 +9,8 @@ namespace NzbDrone.Core.Indexers
     {
         bool AtDownloadLimit(IndexerDefinition indexer);
         bool AtQueryLimit(IndexerDefinition indexer);
+        int CalculateRetryAfterDownloadLimit(IndexerDefinition indexer);
+        int CalculateRetryAfterQueryLimit(IndexerDefinition indexer);
     }
 
     public class IndexerLimitService : IIndexerLimitService
@@ -31,9 +32,9 @@ namespace NzbDrone.Core.Indexers
                 var grabCount = _historyService.CountSince(indexer.Id, DateTime.Now.AddHours(-24), new List<HistoryEventType> { HistoryEventType.ReleaseGrabbed });
                 var grabLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit;
 
-                if (grabCount > grabLimit)
+                if (grabCount >= grabLimit)
                 {
-                    _logger.Info("Indexer {0} has exceeded maximum grab limit for last 24 hours", indexer.Name);
+                    _logger.Info("Indexer {0} has performed {1} of possible {2} grabs in last 24 hours, exceeding the maximum grab limit", indexer.Name, grabCount, grabLimit);
 
                     return true;
                 }
@@ -51,9 +52,9 @@ namespace NzbDrone.Core.Indexers
                 var queryCount = _historyService.CountSince(indexer.Id, DateTime.Now.AddHours(-24), new List<HistoryEventType> { HistoryEventType.IndexerQuery, HistoryEventType.IndexerRss });
                 var queryLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit;
 
-                if (queryCount > queryLimit)
+                if (queryCount >= queryLimit)
                 {
-                    _logger.Info("Indexer {0} has exceeded maximum query limit for last 24 hours", indexer.Name);
+                    _logger.Info("Indexer {0} has performed {1} of possible {2} queries in last 24 hours, exceeding the maximum query limit", indexer.Name, queryCount, queryLimit);
 
                     return true;
                 }
@@ -63,5 +64,39 @@ namespace NzbDrone.Core.Indexers
 
             return false;
         }
+
+        public int CalculateRetryAfterDownloadLimit(IndexerDefinition indexer)
+        {
+            if (indexer.Id > 0 && ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit.HasValue)
+            {
+                var grabLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit.GetValueOrDefault();
+
+                var firstHistorySince = _historyService.FindFirstForIndexerSince(indexer.Id, DateTime.Now.AddHours(-24), new List<HistoryEventType> { HistoryEventType.ReleaseGrabbed }, grabLimit);
+
+                if (firstHistorySince != null)
+                {
+                    return Convert.ToInt32(firstHistorySince.Date.ToLocalTime().AddHours(24).Subtract(DateTime.Now).TotalSeconds);
+                }
+            }
+
+            return 0;
+        }
+
+        public int CalculateRetryAfterQueryLimit(IndexerDefinition indexer)
+        {
+            if (indexer.Id > 0 && ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit.HasValue)
+            {
+                var queryLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit.GetValueOrDefault();
+
+                var firstHistorySince = _historyService.FindFirstForIndexerSince(indexer.Id, DateTime.Now.AddHours(-24), new List<HistoryEventType> { HistoryEventType.IndexerQuery, HistoryEventType.IndexerRss }, queryLimit);
+
+                if (firstHistorySince != null)
+                {
+                    return Convert.ToInt32(firstHistorySince.Date.ToLocalTime().AddHours(24).Subtract(DateTime.Now).TotalSeconds);
+                }
+            }
+
+            return 0;
+        }
     }
 }

src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs

@@ -1,12 +1,7 @@
-using System;
-using System.Net;
 using System.Text;
-using System.Threading.Tasks;
 using MonoTorrent;
 using NLog;
-using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Messaging.Events;
 
 namespace NzbDrone.Core.Indexers
@@ -19,85 +14,15 @@ namespace NzbDrone.Core.Indexers
         {
         }
 
-        public override async Task<byte[]> Download(Uri link)
-        {
-            Cookies = GetCookies();
-
-            if (link.Scheme == "magnet")
-            {
-                ValidateMagnet(link.OriginalString);
-
-                return Encoding.UTF8.GetBytes(link.OriginalString);
-            }
-
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-
-            byte[] torrentData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                torrentData = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            ValidateTorrent(torrentData);
-
-            return torrentData;
-        }
-
-        protected void ValidateMagnet(string link)
-        {
-            MagnetLink.Parse(link);
-        }
-
-        protected void ValidateTorrent(byte[] torrentData)
+        protected override void ValidateDownloadData(byte[] fileData)
         {
             try
             {
-                Torrent.Load(torrentData);
+                Torrent.Load(fileData);
             }
             catch
             {
-                _logger.Trace("Invalid torrent file contents: {0}", Encoding.ASCII.GetString(torrentData));
+                _logger.Trace("Invalid torrent file contents: {0}", Encoding.ASCII.GetString(fileData));
                 throw;
             }
         }

src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs

@@ -1,11 +1,6 @@
-using System;
-using System.Net;
-using System.Threading.Tasks;
 using NLog;
-using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Download;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Messaging.Events;
 
 namespace NzbDrone.Core.Indexers
@@ -21,64 +16,9 @@ namespace NzbDrone.Core.Indexers
             _nzbValidationService = nzbValidationService;
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override void ValidateDownloadData(byte[] fileData)
         {
-            Cookies = GetCookies();
-
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-
-            byte[] nzbData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                nzbData = response.ResponseData;
-
-                _logger.Debug("Downloaded nzb for release finished ({0} bytes from {1})", nzbData.Length, link.AbsoluteUri);
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading nzb file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading nzb failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading nzb failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading nzb failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading nzb failed");
-                throw;
-            }
-
-            _nzbValidationService.Validate(nzbData);
-
-            return nzbData;
+            _nzbValidationService.Validate(fileData);
         }
     }
 }

src/Prowlarr.Api.V1/Indexers/NewznabController.cs

@@ -1,21 +1,25 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Net;
+using System.Net.Http.Headers;
 using System.Text;
 using System.Threading.Tasks;
 using System.Xml.Linq;
 using Microsoft.AspNetCore.Cors;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Download;
-using NzbDrone.Core.History;
+using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Indexers;
 using NzbDrone.Core.IndexerSearch;
-using NzbDrone.Core.Parser;
 using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.ThingiProvider.Status;
 using Prowlarr.Http.Extensions;
 using Prowlarr.Http.REST;
+using BadRequestException = NzbDrone.Core.Exceptions.BadRequestException;
 
 namespace NzbDrone.Api.V1.Indexers
 {
@@ -27,18 +31,21 @@ namespace NzbDrone.Api.V1.Indexers
         private IIndexerFactory _indexerFactory { get; set; }
         private ISearchForNzb _nzbSearchService { get; set; }
         private IIndexerLimitService _indexerLimitService { get; set; }
+        private IIndexerStatusService _indexerStatusService;
         private IDownloadMappingService _downloadMappingService { get; set; }
         private IDownloadService _downloadService { get; set; }
 
         public NewznabController(IndexerFactory indexerFactory,
             ISearchForNzb nzbSearchService,
             IIndexerLimitService indexerLimitService,
+            IIndexerStatusService indexerStatusService,
             IDownloadMappingService downloadMappingService,
             IDownloadService downloadService)
         {
             _indexerFactory = indexerFactory;
             _nzbSearchService = nzbSearchService;
             _indexerLimitService = indexerLimitService;
+            _indexerStatusService = indexerStatusService;
             _downloadMappingService = downloadMappingService;
             _downloadService = downloadService;
         }
@@ -54,7 +61,7 @@ namespace NzbDrone.Api.V1.Indexers
 
             if (requestType.IsNullOrWhiteSpace())
             {
-                return Content(CreateErrorXML(200, "Missing parameter (t)"), "application/rss+xml");
+                return CreateResponse(CreateErrorXML(200, "Missing parameter (t)"), statusCode: StatusCodes.Status400BadRequest);
             }
 
             request.imdbid = request.imdbid?.TrimStart('t') ?? null;
@@ -63,7 +70,7 @@ namespace NzbDrone.Api.V1.Indexers
             {
                 if (!int.TryParse(request.imdbid, out var imdb) || imdb == 0)
                 {
-                    return Content(CreateErrorXML(201, "Incorrect parameter (imdbid)"), "application/rss+xml");
+                    return CreateResponse(CreateErrorXML(201, "Incorrect parameter (imdbid)"), statusCode: StatusCodes.Status400BadRequest);
                 }
             }
 
@@ -97,25 +104,27 @@ namespace NzbDrone.Api.V1.Indexers
                             caps.Categories.AddCategoryMapping(1, cat);
                         }
 
-                        return Content(caps.ToXml(), "application/rss+xml");
+                        return CreateResponse(caps.ToXml());
                     case "search":
                     case "tvsearch":
                     case "music":
                     case "book":
                     case "movie":
-                        var results = new NewznabResults();
-                        results.Releases = new List<ReleaseInfo>
+                        var results = new NewznabResults
                         {
-                            new ReleaseInfo
+                            Releases = new List<ReleaseInfo>
                             {
-                                Title = "Test Release",
-                                Guid = "https://prowlarr.com",
-                                DownloadUrl = "https://prowlarr.com",
-                                PublishDate = DateTime.Now
+                                new ()
+                                {
+                                    Title = "Test Release",
+                                    Guid = "https://prowlarr.com",
+                                    DownloadUrl = "https://prowlarr.com",
+                                    PublishDate = DateTime.Now
+                                }
                             }
                         };
 
-                        return Content(results.ToXml(DownloadProtocol.Usenet), "application/rss+xml");
+                        return CreateResponse(results.ToXml(DownloadProtocol.Usenet));
                 }
             }
 
@@ -126,19 +135,37 @@ namespace NzbDrone.Api.V1.Indexers
                 throw new NotFoundException("Indexer Not Found");
             }
 
+            if (!indexerDef.Enable)
+            {
+                return CreateResponse(CreateErrorXML(410, "Indexer is disabled"), statusCode: StatusCodes.Status410Gone);
+            }
+
             var indexer = _indexerFactory.GetInstance(indexerDef);
 
+            var blockedIndexerStatus = GetBlockedIndexerStatus(indexer);
+
+            if (blockedIndexerStatus?.DisabledTill != null)
+            {
+                var retryAfterDisabledTill = Convert.ToInt32(blockedIndexerStatus.DisabledTill.Value.ToLocalTime().Subtract(DateTime.Now).TotalSeconds);
+                AddRetryAfterHeader(retryAfterDisabledTill);
+
+                return CreateResponse(CreateErrorXML(429, $"Indexer is disabled till {blockedIndexerStatus.DisabledTill.Value.ToLocalTime()} due to recent failures."), statusCode: StatusCodes.Status429TooManyRequests);
+            }
+
             //TODO Optimize this so it's not called here and in NzbSearchService (for manual search)
             if (_indexerLimitService.AtQueryLimit(indexerDef))
             {
-                return Content(CreateErrorXML(429, $"Request limit reached ({((IIndexerSettings)indexer.Definition.Settings).BaseSettings.QueryLimit})"), "application/rss+xml");
+                var retryAfterQueryLimit = _indexerLimitService.CalculateRetryAfterQueryLimit(indexerDef);
+                AddRetryAfterHeader(retryAfterQueryLimit);
+
+                return CreateResponse(CreateErrorXML(429, $"User configurable Indexer Query Limit of {((IIndexerSettings)indexer.Definition.Settings).BaseSettings.QueryLimit} reached."), statusCode: StatusCodes.Status429TooManyRequests);
             }
 
             switch (requestType)
             {
                 case "caps":
                     var caps = indexer.GetCapabilities();
-                    return Content(caps.ToXml(), "application/rss+xml");
+                    return CreateResponse(caps.ToXml());
                 case "search":
                 case "tvsearch":
                 case "music":
@@ -156,9 +183,9 @@ namespace NzbDrone.Api.V1.Indexers
                         }
                     }
 
-                    return Content(results.ToXml(indexer.Protocol), "application/rss+xml");
+                    return CreateResponse(results.ToXml(indexer.Protocol));
                 default:
-                    return Content(CreateErrorXML(202, $"No such function ({requestType})"), "application/rss+xml");
+                    return CreateResponse(CreateErrorXML(202, $"No such function ({requestType})"), statusCode: StatusCodes.Status400BadRequest);
             }
         }
 
@@ -167,11 +194,35 @@ namespace NzbDrone.Api.V1.Indexers
         public async Task<object> GetDownload(int id, string link, string file)
         {
             var indexerDef = _indexerFactory.Get(id);
+
+            if (indexerDef == null)
+            {
+                throw new NotFoundException("Indexer Not Found");
+            }
+
+            if (!indexerDef.Enable)
+            {
+                return CreateResponse(CreateErrorXML(410, "Indexer is disabled"), statusCode: StatusCodes.Status410Gone);
+            }
+
             var indexer = _indexerFactory.GetInstance(indexerDef);
 
+            var blockedIndexerStatus = GetBlockedIndexerStatus(indexer);
+
+            if (blockedIndexerStatus?.DisabledTill != null)
+            {
+                var retryAfterDisabledTill = Convert.ToInt32(blockedIndexerStatus.DisabledTill.Value.ToLocalTime().Subtract(DateTime.Now).TotalSeconds);
+                AddRetryAfterHeader(retryAfterDisabledTill);
+
+                return CreateResponse(CreateErrorXML(429, $"Indexer is disabled till {blockedIndexerStatus.DisabledTill.Value.ToLocalTime()} due to recent failures."), statusCode: StatusCodes.Status429TooManyRequests);
+            }
+
             if (_indexerLimitService.AtDownloadLimit(indexerDef))
             {
-                return Content(CreateErrorXML(429, $"Grab limit reached ({((IIndexerSettings)indexer.Definition.Settings).BaseSettings.GrabLimit})"), "application/rss+xml");
+                var retryAfterDownloadLimit = _indexerLimitService.CalculateRetryAfterDownloadLimit(indexerDef);
+                AddRetryAfterHeader(retryAfterDownloadLimit);
+
+                return CreateResponse(CreateErrorXML(429, $"User configurable Indexer Grab Limit of {((IIndexerSettings)indexer.Definition.Settings).BaseSettings.GrabLimit} reached."), statusCode: StatusCodes.Status429TooManyRequests);
             }
 
             if (link.IsNullOrWhiteSpace() || file.IsNullOrWhiteSpace())
@@ -181,11 +232,6 @@ namespace NzbDrone.Api.V1.Indexers
 
             file = WebUtility.UrlDecode(file);
 
-            if (indexerDef == null)
-            {
-                throw new NotFoundException("Indexer Not Found");
-            }
-
             var source = UserAgentParser.ParseSource(Request.Headers["User-Agent"]);
             var host = Request.GetHostName();
 
@@ -198,8 +244,27 @@ namespace NzbDrone.Api.V1.Indexers
                 return RedirectPermanent(unprotectedlLink);
             }
 
-            var downloadBytes = Array.Empty<byte>();
-            downloadBytes = await _downloadService.DownloadReport(unprotectedlLink, id, source, host, file);
+            byte[] downloadBytes;
+
+            try
+            {
+                downloadBytes = await _downloadService.DownloadReport(unprotectedlLink, id, source, host, file);
+            }
+            catch (ReleaseUnavailableException ex)
+            {
+                return CreateResponse(CreateErrorXML(410, ex.Message), statusCode: StatusCodes.Status410Gone);
+            }
+            catch (ReleaseDownloadException ex) when (ex.InnerException is TooManyRequestsException http429)
+            {
+                var http429RetryAfter = Convert.ToInt32(http429.RetryAfter.TotalSeconds);
+                AddRetryAfterHeader(http429RetryAfter);
+
+                return CreateResponse(CreateErrorXML(429, ex.Message), statusCode: StatusCodes.Status429TooManyRequests);
+            }
+            catch (Exception ex)
+            {
+                return CreateResponse(CreateErrorXML(500, ex.Message), statusCode: StatusCodes.Status500InternalServerError);
+            }
 
             // handle magnet URLs
             if (downloadBytes.Length >= 7
@@ -232,5 +297,32 @@ namespace NzbDrone.Api.V1.Indexers
 
             return xdoc.Declaration + Environment.NewLine + xdoc;
         }
+
+        private ContentResult CreateResponse(string content, string contentType = "application/rss+xml", int statusCode = StatusCodes.Status200OK)
+        {
+            var mediaTypeHeaderValue = MediaTypeHeaderValue.Parse(contentType);
+
+            return new ContentResult
+            {
+                StatusCode = statusCode,
+                Content = content,
+                ContentType = mediaTypeHeaderValue.ToString()
+            };
+        }
+
+        private ProviderStatusBase GetBlockedIndexerStatus(IIndexer indexer)
+        {
+            var blockedIndexers = _indexerStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId, v => v);
+
+            return blockedIndexers.TryGetValue(indexer.Definition.Id, out var blockedIndexerStatus) ? blockedIndexerStatus : null;
+        }
+
+        private void AddRetryAfterHeader(int retryAfterSeconds)
+        {
+            if (!HttpContext.Response.Headers.ContainsKey("Retry-After") && retryAfterSeconds > 0)
+            {
+                HttpContext.Response.Headers.Add("Retry-After", $"{retryAfterSeconds}");
+            }
+        }
     }
 }