New: (UI) Detailed error message for inner exception in indexers validation

Co-authored-by: Michael Goodnow <mmgoodnow@gmail.com>

Closes #2140

azure-pipelines.yml

@@ -9,7 +9,7 @@ variables:
   testsFolder: './_tests'
   yarnCacheFolder: $(Pipeline.Workspace)/.yarn
   nugetCacheFolder: $(Pipeline.Workspace)/.nuget/packages
-  majorVersion: '1.17.2'
+  majorVersion: '1.18.0'
   minorVersion: $[counter('minorVersion', 1)]
   prowlarrVersion: '$(majorVersion).$(minorVersion)'
   buildName: '$(Build.SourceBranchName).$(prowlarrVersion)'

frontend/src/Components/Form/Form.css

@@ -1,3 +1,7 @@
 .validationFailures {
   margin-bottom: 20px;
 }
+
+.details {
+  margin-left: 5px;
+}

frontend/src/Components/Form/Form.css.d.ts

@@ -1,6 +1,7 @@
 // This file is automatically generated.
 // Please do not change this file!
 interface CssExports {
+  'details': string;
   'validationFailures': string;
 }
 export const cssExports: CssExports;

frontend/src/Components/Form/Form.js

@@ -1,7 +1,8 @@
 import PropTypes from 'prop-types';
 import React from 'react';
 import Alert from 'Components/Alert';
-import { kinds } from 'Helpers/Props';
+import Icon from 'Components/Icon';
+import { icons, kinds } from 'Helpers/Props';
 import styles from './Form.css';
 
 function Form(props) {
@@ -26,6 +27,16 @@ function Form(props) {
                     kind={kinds.DANGER}
                   >
                     {error.errorMessage}
+
+                    {
+                      error.detailedDescription ?
+                        <Icon
+                          containerClassName={styles.details}
+                          name={icons.INFO}
+                          title={error.detailedDescription}
+                        /> :
+                        null
+                    }
                   </Alert>
                 );
               })
@@ -39,6 +50,16 @@ function Form(props) {
                     kind={kinds.WARNING}
                   >
                     {warning.errorMessage}
+
+                    {
+                      warning.detailedDescription ?
+                        <Icon
+                          containerClassName={styles.details}
+                          name={icons.INFO}
+                          title={warning.detailedDescription}
+                        /> :
+                        null
+                    }
                   </Alert>
                 );
               })

src/Directory.Build.props

@@ -130,7 +130,7 @@
 
   <!-- Standard testing packages -->
   <ItemGroup Condition="'$(TestProject)'=='true'">
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
     <PackageReference Include="NUnit" Version="3.14.0" />
     <PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
     <PackageReference Include="NunitXml.TestLogger" Version="3.0.131" />

src/NzbDrone.Common/Prowlarr.Common.csproj

@@ -10,7 +10,7 @@
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="NLog" Version="5.2.0" />
     <PackageReference Include="NLog.Extensions.Logging" Version="5.3.0" />
-    <PackageReference Include="Npgsql" Version="7.0.6" />
+    <PackageReference Include="Npgsql" Version="7.0.7" />
     <PackageReference Include="Sentry" Version="4.0.2" />
     <PackageReference Include="NLog.Targets.Syslog" Version="7.0.0" />
     <PackageReference Include="SharpZipLib" Version="1.4.2" />

src/NzbDrone.Core/Configuration/ConfigFileProvider.cs

@@ -245,7 +245,7 @@ namespace NzbDrone.Core.Configuration
         {
             get
             {
-                var urlBase = _serverOptions.UrlBase ?? GetValue("UrlBase", "").Trim('/');
+                var urlBase = (_serverOptions.UrlBase ?? GetValue("UrlBase", "")).Trim('/');
 
                 if (urlBase.IsNullOrWhiteSpace())
                 {

src/NzbDrone.Core/Download/DownloadService.cs

@@ -1,10 +1,8 @@
 using System;
 using System.Threading.Tasks;
 using NLog;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Common.Instrumentation.Extensions;
-using NzbDrone.Common.TPL;
 using NzbDrone.Core.Download.Clients;
 using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Indexers;
@@ -27,7 +25,6 @@ namespace NzbDrone.Core.Download
         private readonly IDownloadClientStatusService _downloadClientStatusService;
         private readonly IIndexerFactory _indexerFactory;
         private readonly IIndexerStatusService _indexerStatusService;
-        private readonly IRateLimitService _rateLimitService;
         private readonly IEventAggregator _eventAggregator;
         private readonly Logger _logger;
 
@@ -35,7 +32,6 @@ namespace NzbDrone.Core.Download
                                IDownloadClientStatusService downloadClientStatusService,
                                IIndexerFactory indexerFactory,
                                IIndexerStatusService indexerStatusService,
-                               IRateLimitService rateLimitService,
                                IEventAggregator eventAggregator,
                                Logger logger)
         {
@@ -43,7 +39,6 @@ namespace NzbDrone.Core.Download
             _downloadClientStatusService = downloadClientStatusService;
             _indexerFactory = indexerFactory;
             _indexerStatusService = indexerStatusService;
-            _rateLimitService = rateLimitService;
             _eventAggregator = eventAggregator;
             _logger = logger;
         }
@@ -132,15 +127,7 @@ namespace NzbDrone.Core.Download
             _logger.Trace("Attempting download of {0}", link);
             var url = new Uri(link);
 
-            // Limit grabs to 2 per second.
-            if (link.IsNotNullOrWhiteSpace() && !link.StartsWith("magnet:"))
-            {
-                await _rateLimitService.WaitAndPulseAsync(url.Host, TimeSpan.FromSeconds(2));
-            }
-
             var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(indexerId));
-            var success = false;
-            var downloadedBytes = Array.Empty<byte>();
 
             var release = new ReleaseInfo
             {
@@ -151,12 +138,14 @@ namespace NzbDrone.Core.Download
                 DownloadProtocol = indexer.Protocol
             };
 
-            var grabEvent = new IndexerDownloadEvent(release, success, source, host, release.Title, release.DownloadUrl)
+            var grabEvent = new IndexerDownloadEvent(release, false, source, host, release.Title, release.DownloadUrl)
             {
                 Indexer = indexer,
                 GrabTrigger = source == "Prowlarr" ? GrabTrigger.Manual : GrabTrigger.Api
             };
 
+            byte[] downloadedBytes;
+
             try
             {
                 downloadedBytes = await indexer.Download(url);

src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs

@@ -29,7 +29,7 @@ namespace NzbDrone.Core.IndexerVersions
         /* Update Service will fall back if version # does not exist for an indexer  per Ta */
 
         private const string DEFINITION_BRANCH = "master";
-        private const int DEFINITION_VERSION = 9;
+        private const int DEFINITION_VERSION = 10;
 
         // Used when moving yml to C#
         private readonly List<string> _definitionBlocklist = new ()

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -4,10 +4,8 @@ using System.Linq;
 using System.Threading.Tasks;
 using FluentValidation.Results;
 using NLog;
-using NzbDrone.Common;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.Http;
-using NzbDrone.Common.Serializer;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.IndexerSearch.Definitions;
 using NzbDrone.Core.IndexerVersions;
@@ -24,7 +22,7 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
         private readonly ICached<CardigannRequestGenerator> _generatorCache;
 
         public override string Name => "Cardigann";
-        public override string[] IndexerUrls => new string[] { "" };
+        public override string[] IndexerUrls => new[] { "" };
         public override string Description => "";
         public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
 
@@ -49,8 +47,7 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
 
         public override IIndexerRequestGenerator GetRequestGenerator()
         {
-            var cacheKey = $"{Settings.DefinitionFile}.{HashUtil.ComputeSha256Hash(Settings.ToJson())}";
-            var generator = _generatorCache.Get(cacheKey, () =>
+            var generator = _generatorCache.Get(Settings.DefinitionFile, () =>
                 new CardigannRequestGenerator(_configService,
                     _definitionService.GetCachedDefinition(Settings.DefinitionFile),
                     _logger,
@@ -61,10 +58,11 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
                     Settings = Settings
                 });
 
-            generator = (CardigannRequestGenerator)SetCookieFunctions(generator);
-
+            generator.Definition = Definition;
             generator.Settings = Settings;
 
+            generator = (CardigannRequestGenerator)SetCookieFunctions(generator);
+
             _generatorCache.ClearExpired();
 
             return generator;

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs

@@ -337,9 +337,11 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
                         variables[name] = selected.Key;
                         break;
                     case "info":
-                        variables[name] = value;
-                        break;
+                    case "info_cookie":
+                    case "info_flaresolverr":
+                    case "info_useragent":
                     case "cardigannCaptcha":
+                        // no-op
                         break;
                     default:
                         throw new NotSupportedException($"Type {setting.Type} is not supported.");

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs

@@ -1178,14 +1178,14 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
 
                 if (method == HttpMethod.Get && searchUrls.Contains(searchUrl))
                 {
-                    _logger.Trace("Skip duplicated request {0}", searchUrl);
+                    _logger.Trace("Skip duplicated request for {0}: {1}", Definition.Name, searchUrl);
 
                     continue;
                 }
 
                 searchUrls.Add(searchUrl);
 
-                _logger.Debug($"Adding request: {searchUrl}");
+                _logger.Debug("Adding request for {0}: {1}", Definition.Name, searchUrl);
 
                 var requestBuilder = new HttpRequestBuilder(searchUrl)
                 {

src/NzbDrone.Core/Indexers/Definitions/MyAnonamouse.cs

@@ -387,14 +387,14 @@ namespace NzbDrone.Core.Indexers.Definitions
                 throw new IndexerException(indexerResponse, $"Unexpected response header {indexerResponse.HttpResponse.Headers.ContentType} from indexer request, expected {HttpAccept.Json.Value}");
             }
 
-            var torrentInfos = new List<TorrentInfo>();
+            var releaseInfos = new List<ReleaseInfo>();
 
             var jsonResponse = JsonConvert.DeserializeObject<MyAnonamouseResponse>(indexerResponse.Content);
 
             var error = jsonResponse.Error;
-            if (error is "Nothing returned, out of 0" or "Nothing returned, out of 1")
+            if (error.IsNotNullOrWhiteSpace() && error.StartsWithIgnoreCase("Nothing returned, out of"))
             {
-                return torrentInfos.ToArray();
+                return releaseInfos.ToArray();
             }
 
             var hasUserVip = HasUserVip();
@@ -462,10 +462,10 @@ namespace NzbDrone.Core.Indexers.Definitions
                 release.MinimumRatio = 1;
                 release.MinimumSeedTime = 259200; // 72 hours
 
-                torrentInfos.Add(release);
+                releaseInfos.Add(release);
             }
 
-            return torrentInfos.ToArray();
+            return releaseInfos.ToArray();
         }
 
         private bool HasUserVip()

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -18,6 +18,7 @@ using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.IndexerSearch.Definitions;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Validation;
 using Polly;
 using Polly.Retry;
 
@@ -814,7 +815,10 @@ namespace NzbDrone.Core.Indexers
             {
                 _logger.Warn(ex, "Unable to connect to indexer");
 
-                return new ValidationFailure(string.Empty, "Unable to connect to indexer, please check your DNS settings and ensure IPv6 is working or disabled. " + ex.Message);
+                return new NzbDroneValidationFailure(string.Empty, "Unable to connect to indexer, please check your DNS settings and ensure IPv6 is working or disabled. " + ex.Message)
+                {
+                    DetailedDescription = ex.InnerException?.Message
+                };
             }
             catch (TaskCanceledException ex)
             {
@@ -841,7 +845,10 @@ namespace NzbDrone.Core.Indexers
             {
                 _logger.Warn(ex, "Unable to connect to indexer");
 
-                return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log above the ValidationFailure for more details. " + ex.Message);
+                return new NzbDroneValidationFailure(string.Empty, "Unable to connect to indexer, check the log above the ValidationFailure for more details. " + ex.Message)
+                {
+                    DetailedDescription = ex.InnerException?.Message
+                };
             }
 
             return null;

src/NzbDrone.Core/Prowlarr.Core.csproj

@@ -9,8 +9,8 @@
     <PackageReference Include="Microsoft.AspNetCore.Cryptography.KeyDerivation" Version="6.0.29" />
     <PackageReference Include="Microsoft.AspNetCore.WebUtilities" Version="2.2.0" />
     <PackageReference Include="NLog.Targets.Syslog" Version="7.0.0" />
-    <PackageReference Include="Npgsql" Version="7.0.6" />
-    <PackageReference Include="Polly" Version="8.3.1" />
+    <PackageReference Include="Npgsql" Version="7.0.7" />
+    <PackageReference Include="Polly" Version="8.4.0" />
     <PackageReference Include="Servarr.FluentMigrator.Runner" Version="3.3.2.9" />
     <PackageReference Include="Servarr.FluentMigrator.Runner.Postgres" Version="3.3.2.9" />
     <PackageReference Include="Servarr.FluentMigrator.Runner.SQLite" Version="3.3.2.9" />

src/Prowlarr.Api.V1/Indexers/IndexerResource.cs

@@ -60,7 +60,7 @@ namespace Prowlarr.Api.V1.Indexers
 
             if (definition.Implementation == nameof(Cardigann))
             {
-                var extraFields = definition.ExtraFields?.Select(MapField).ToList() ?? new List<Field>();
+                var extraFields = definition.ExtraFields?.Select(MapCardigannField).ToList() ?? new List<Field>();
 
                 resource.Fields.AddRange(extraFields);
 
@@ -160,7 +160,7 @@ namespace Prowlarr.Api.V1.Indexers
             };
         }
 
-        private Field MapField(SettingsField setting, int order)
+        private Field MapCardigannField(SettingsField setting, int order)
         {
             var field = new Field
             {
@@ -185,6 +185,26 @@ namespace Prowlarr.Api.V1.Indexers
             {
                 field.Value = bool.TryParse(setting.Default, out var value) && value;
             }
+            else if (setting.Type is "info_cookie" or "info_flaresolverr" or "info_useragent")
+            {
+                field.Type = "info";
+
+                switch (setting.Type)
+                {
+                    case "info_cookie":
+                        field.Label = "How to get the Cookie";
+                        field.Value = "<ol><li>Login to this tracker with your browser</li><li>Open the <b>DevTools</b> panel by pressing <b>F12</b></li><li>Select the <b>Network</b> tab</li><li>Click on the <b>Doc</b> button (Chrome Browser) or <b>HTML</b> button (FireFox)</li><li>Refresh the page by pressing <b>F5</b></li><li>Click on the first row entry</li><li>Select the <b>Headers</b> tab on the Right panel</li><li>Find <b>'cookie:'</b> in the <b>Request Headers</b> section</li><li><b>Select</b> and <b>Copy</b> the whole cookie string <i>(everything after 'cookie: ')</i> and <b>Paste</b> here.</li></ol>";
+                        break;
+                    case "info_flaresolverr":
+                        field.Label = "FlareSolverr Info";
+                        field.Value = "This site may use Cloudflare DDoS Protection, therefore Prowlarr requires <a href=\"https://wiki.servarr.com/prowlarr/faq#can-i-use-flaresolverr-indexers\" target=\"_blank\" rel=\"noreferrer\">FlareSolverr</a> to access it.";
+                        break;
+                    case "info_useragent":
+                        field.Label = "How to get the User-Agent";
+                        field.Value = "<ol><li>From the same place you fetched the cookie,</li><li>Find <b>'user-agent:'</b> in the <b>Request Headers</b> section</li><li><b>Select</b> and <b>Copy</b> the whole user-agent string <i>(everything after 'user-agent: ')</i> and <b>Paste</b> here.</li></ol>";
+                        break;
+                }
+            }
             else
             {
                 field.Value = setting.Default;