New: Indexer Messaging and Error Improvements

(cherry picked from commit 3b505d8734dcbe3fa53acba7f94f1361151e6a44)

azure-pipelines.yml

@@ -9,7 +9,7 @@ variables:
   testsFolder: './_tests'
   yarnCacheFolder: $(Pipeline.Workspace)/.yarn
   nugetCacheFolder: $(Pipeline.Workspace)/.nuget/packages
-  majorVersion: '0.1.9'
+  majorVersion: '0.2.0'
   minorVersion: $[counter('minorVersion', 1)]
   readarrVersion: '$(majorVersion).$(minorVersion)'
   buildName: '$(Build.SourceBranchName).$(readarrVersion)'

src/NzbDrone.Common/Http/HttpResponse.cs

@@ -50,6 +50,8 @@ namespace NzbDrone.Common.Http
 
         public bool HasHttpError => (int)StatusCode >= 400;
 
+        public bool HasHttpServerError => (int)StatusCode >= 500;
+
         public bool HasHttpRedirect => StatusCode == HttpStatusCode.Moved ||
                                        StatusCode == HttpStatusCode.MovedPermanently ||
                                        StatusCode == HttpStatusCode.Found ||

src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleases.cs

@@ -16,7 +16,7 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers
 
         public void Clean()
         {
-            var mapper = _database.OpenConnection();
+            using var mapper = _database.OpenConnection();
 
             if (_database.DatabaseType == DatabaseType.PostgreSQL)
             {
@@ -31,7 +31,7 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers
             }
             else
             {
-            mapper.Execute(@"DELETE FROM ""PendingReleases""
+                mapper.Execute(@"DELETE FROM ""PendingReleases""
                             WHERE ""Added"" < @TwoWeeksAgo
                             AND ""REASON"" IN @Reasons",
                           new

src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs

@@ -14,7 +14,7 @@ namespace NzbDrone.Core.Housekeeping.Housekeepers
 
         public void Clean()
         {
-            var mapper = _database.OpenConnection();
+            using var mapper = _database.OpenConnection();
 
             mapper.Execute(@"DELETE FROM ""DownloadClientStatus""
                                      WHERE ""Id"" IN (

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -2,6 +2,8 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Extensions;
@@ -77,6 +79,7 @@ namespace NzbDrone.Core.Indexers
         {
             var releases = new List<ReleaseInfo>();
             var url = string.Empty;
+            var minimumBackoff = TimeSpan.FromHours(1);
 
             try
             {
@@ -169,8 +172,7 @@ namespace NzbDrone.Core.Indexers
             }
             catch (WebException webException)
             {
-                if (webException.Status == WebExceptionStatus.NameResolutionFailure ||
+                if (webException.Status is WebExceptionStatus.NameResolutionFailure or WebExceptionStatus.ConnectFailure)
-                    webException.Status == WebExceptionStatus.ConnectFailure)
                 {
                     _indexerStatusService.RecordConnectionFailure(Definition.Id);
                 }
@@ -180,7 +182,7 @@ namespace NzbDrone.Core.Indexers
                 }
 
                 if (webException.Message.Contains("502") || webException.Message.Contains("503") ||
-                    webException.Message.Contains("timed out"))
+                    webException.Message.Contains("504") || webException.Message.Contains("timed out"))
                 {
                     _logger.Warn("{0} server is currently unavailable. {1} {2}", this, url, webException.Message);
                 }
@@ -191,34 +193,29 @@ namespace NzbDrone.Core.Indexers
             }
             catch (TooManyRequestsException ex)
             {
-                if (ex.RetryAfter != TimeSpan.Zero)
+                var retryTime = ex.RetryAfter != TimeSpan.Zero ? ex.RetryAfter : minimumBackoff;
-                {
+                _indexerStatusService.RecordFailure(Definition.Id, retryTime);
-                    _indexerStatusService.RecordFailure(Definition.Id, ex.RetryAfter);
-                }
-                else
-                {
-                    _indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
-                }
 
-                _logger.Warn("API Request Limit reached for {0}", this);
+                _logger.Warn("API Request Limit reached for {0}. Disabled for {1}", this, retryTime);
             }
             catch (HttpException ex)
             {
                 _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Warn("{0} {1}", this, ex.Message);
+                if (ex.Response.HasHttpServerError)
-            }
-            catch (RequestLimitReachedException ex)
-            {
-                if (ex.RetryAfter != TimeSpan.Zero)
                 {
-                    _indexerStatusService.RecordFailure(Definition.Id, ex.RetryAfter);
+                    _logger.Warn("Unable to connect to {0} at [{1}]. Indexer's server is unavailable. Try again later. {2}", this, url, ex.Message);
                 }
                 else
                 {
-                    _indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
+                    _logger.Warn("{0} {1}", this, ex.Message);
                 }
+            }
+            catch (RequestLimitReachedException ex)
+            {
+                var retryTime = ex.RetryAfter != TimeSpan.Zero ? ex.RetryAfter : minimumBackoff;
+                _indexerStatusService.RecordFailure(Definition.Id, retryTime);
 
-                _logger.Warn("API Request Limit reached for {0}", this);
+                _logger.Warn("API Request Limit reached for {0}. Disabled for {1}", this, retryTime);
             }
             catch (ApiKeyException)
             {
@@ -238,6 +235,11 @@ namespace NzbDrone.Core.Indexers
                     _logger.Error(ex, "CAPTCHA token required for {0}, check indexer settings.", this);
                 }
             }
+            catch (TaskCanceledException ex)
+            {
+                _indexerStatusService.RecordFailure(Definition.Id);
+                _logger.Warn(ex, "Unable to connect to indexer, possibly due to a timeout. {0}", url);
+            }
             catch (IndexerException ex)
             {
                 _indexerStatusService.RecordFailure(Definition.Id);
@@ -333,6 +335,8 @@ namespace NzbDrone.Core.Indexers
             catch (RequestLimitReachedException ex)
             {
                 _logger.Warn("Request limit reached: " + ex.Message);
+
+                return new ValidationFailure(string.Empty, "Request limit reached: " + ex.Message);
             }
             catch (CloudFlareCaptchaException ex)
             {
@@ -357,6 +361,55 @@ namespace NzbDrone.Core.Indexers
 
                 return new ValidationFailure(string.Empty, "Unable to connect to indexer. " + ex.Message);
             }
+            catch (HttpException ex)
+            {
+                if (ex.Response.StatusCode == HttpStatusCode.BadRequest &&
+                    ex.Response.Content.Contains("not support the requested query"))
+                {
+                    _logger.Warn(ex, "Indexer does not support the query");
+                    return new ValidationFailure(string.Empty, "Indexer does not support the current query. Check if the categories and or searching for seasons/episodes are supported. Check the log for more details.");
+                }
+
+                _logger.Warn(ex, "Unable to connect to indexer");
+                if (ex.Response.HasHttpServerError)
+                {
+                    return new ValidationFailure(string.Empty, "Unable to connect to indexer, indexer's server is unavailable. Try again later. " + ex.Message);
+                }
+
+                if (ex.Response.StatusCode is HttpStatusCode.Forbidden or HttpStatusCode.Unauthorized)
+                {
+                    return new ValidationFailure(string.Empty, "Unable to connect to indexer, invalid credentials. " + ex.Message);
+                }
+
+                return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log above the ValidationFailure for more details. " + ex.Message);
+            }
+            catch (HttpRequestException ex)
+            {
+                _logger.Warn(ex, "Unable to connect to indexer");
+
+                return new ValidationFailure(string.Empty, "Unable to connect to indexer, please check your DNS settings and ensure IPv6 is working or disabled. " + ex.Message);
+            }
+            catch (TaskCanceledException ex)
+            {
+                _logger.Warn(ex, "Unable to connect to indexer");
+
+                return new ValidationFailure(string.Empty, "Unable to connect to indexer, possibly due to a timeout. Try again or check your network settings. " + ex.Message);
+            }
+            catch (WebException webException)
+            {
+                _logger.Warn("Unable to connect to indexer.");
+
+                if (webException.Status is WebExceptionStatus.NameResolutionFailure or WebExceptionStatus.ConnectFailure)
+                {
+                    return new ValidationFailure(string.Empty, "Unable to connect to indexer connection failure. Check your connection to the indexer's server and DNS." + webException.Message);
+                }
+
+                if (webException.Message.Contains("502") || webException.Message.Contains("503") ||
+                    webException.Message.Contains("504") || webException.Message.Contains("timed out"))
+                {
+                    return new ValidationFailure(string.Empty, "Unable to connect to indexer, indexer's server is unavailable. Try again later. " + webException.Message);
+                }
+            }
             catch (Exception ex)
             {
                 _logger.Warn(ex, "Unable to connect to indexer");

src/NzbDrone.Core/Indexers/RssParser.cs

@@ -121,7 +121,8 @@ namespace NzbDrone.Core.Indexers
 
         protected virtual bool PreProcess(IndexerResponse indexerResponse)
         {
-            if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
+            // Server Down HTTP Errors are handled in HTTPIndexerBase so ignore them here
+            if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK && !indexerResponse.HttpResponse.HasHttpServerError)
             {
                 throw new IndexerException(indexerResponse, "Indexer API call resulted in an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
             }

src/NzbDrone.Core/Localization/Core/pt_BR.json

@@ -939,5 +939,6 @@
     "RemoveSelectedItemsQueueMessageText": "Tem certeza de que deseja remover {0} itens da fila?",
     "Required": "Requerido",
     "ResetQualityDefinitions": "Redefinir Configurações de Qualidade",
-    "ResetQualityDefinitionsMessageText": "Tem certeza de que deseja redefinir as configurações de qualidade?"
+    "ResetQualityDefinitionsMessageText": "Tem certeza de que deseja redefinir as configurações de qualidade?",
+    "BlocklistReleaseHelpText": "Evita que o Readarr pegue automaticamente esses arquivos novamente"
 }

src/Readarr.Api.V1/Queue/QueueController.cs

@@ -189,6 +189,8 @@ namespace Readarr.Api.V1.Queue
                     return q => q.Book?.ReleaseDate ?? DateTime.MinValue;
                 case "quality":
                     return q => q.Quality;
+                case "size":
+                    return q => q.Size;
                 case "progress":
                     // Avoid exploding if a download's size is 0
                     return q => 100 - (q.Sizeleft / Math.Max(q.Size * 100, 1));