Fixed: (Cardigann) Prevent fetching the first page multiple times

Co-authored-by: Sergey M <msergein@users.noreply.github.com>

azure-pipelines.yml

@@ -9,7 +9,7 @@ variables:
   testsFolder: './_tests'
   yarnCacheFolder: $(Pipeline.Workspace)/.yarn
   nugetCacheFolder: $(Pipeline.Workspace)/.nuget/packages
-  majorVersion: '1.2.1'
+  majorVersion: '1.3.1'
   minorVersion: $[counter('minorVersion', 1)]
   prowlarrVersion: '$(majorVersion).$(minorVersion)'
   buildName: '$(Build.SourceBranchName).$(prowlarrVersion)'

frontend/.eslintrc.js

@@ -39,6 +39,7 @@ module.exports = {
   plugins: [
     'filenames',
     'react',
+    'react-hooks',
     'simple-import-sort',
     'import'
   ],
@@ -308,7 +309,9 @@ module.exports = {
     'react/react-in-jsx-scope': 2,
     'react/self-closing-comp': 2,
     'react/sort-comp': 2,
-    'react/jsx-wrap-multilines': 2
+    'react/jsx-wrap-multilines': 2,
+    'react-hooks/rules-of-hooks': 'error',
+    'react-hooks/exhaustive-deps': 'error'
   },
   overrides: [
     {

frontend/src/App/ApplyTheme.js

@@ -1,5 +1,5 @@
 import PropTypes from 'prop-types';
-import React, { Fragment, useEffect } from 'react';
+import React, { Fragment, useCallback, useEffect } from 'react';
 import { connect } from 'react-redux';
 import { createSelector } from 'reselect';
 import themes from 'Styles/Themes';
@@ -19,7 +19,8 @@ function createMapStateToProps() {
 
 function ApplyTheme({ theme, children }) {
   // Update the CSS Variables
-  function updateCSSVariables() {
+
+  const updateCSSVariables = useCallback(() => {
     const arrayOfVariableKeys = Object.keys(themes[theme]);
     const arrayOfVariableValues = Object.values(themes[theme]);
 
@@ -31,12 +32,12 @@ function ApplyTheme({ theme, children }) {
         arrayOfVariableValues[index]
       );
     });
-  }
+  }, [theme]);
 
   // On Component Mount and Component Update
   useEffect(() => {
     updateCSSVariables(theme);
-  }, [theme]);
+  }, [updateCSSVariables, theme]);
 
   return <Fragment>{children}</Fragment>;
 }

frontend/src/Components/Modal/ConfirmModal.js

@@ -30,10 +30,10 @@ function ConfirmModal(props) {
   useEffect(() => {
     if (isOpen) {
       bindShortcut('enter', onConfirm);
-    } else {
-      unbindShortcut('enter', onConfirm);
+
+      return () => unbindShortcut('enter', onConfirm);
     }
-  }, [onConfirm]);
+  }, [bindShortcut, unbindShortcut, isOpen, onConfirm]);
 
   return (
     <Modal

frontend/src/Indexer/Index/Table/IndexerIndexRow.js

@@ -253,7 +253,7 @@ class IndexerIndexRow extends Component {
                         className={styles.externalLink}
                         name={icons.EXTERNAL_LINK}
                         title={translate('Website')}
-                        to={baseUrl.replace('api.', '')}
+                        to={baseUrl.replace(/(:\/\/)api\./, '$1')}
                       /> : null
                   }
 

frontend/src/Search/Table/SearchIndexRow.js

@@ -71,6 +71,19 @@ class SearchIndexRow extends Component {
     });
   };
 
+  onSavePress = () => {
+    const {
+      downloadUrl,
+      fileName,
+      onSavePress
+    } = this.props;
+
+    onSavePress({
+      downloadUrl,
+      fileName
+    });
+  };
+
   //
   // Render
 
@@ -85,7 +98,6 @@ class SearchIndexRow extends Component {
       publishDate,
       title,
       infoUrl,
-      downloadUrl,
       indexer,
       size,
       files,
@@ -300,7 +312,7 @@ class SearchIndexRow extends Component {
                     className={styles.downloadLink}
                     name={icons.SAVE}
                     title={translate('Save')}
-                    to={downloadUrl}
+                    onPress={this.onSavePress}
                   />
                 </VirtualTableRowCell>
               );
@@ -323,6 +335,7 @@ SearchIndexRow.propTypes = {
   ageMinutes: PropTypes.number.isRequired,
   publishDate: PropTypes.string.isRequired,
   title: PropTypes.string.isRequired,
+  fileName: PropTypes.string.isRequired,
   infoUrl: PropTypes.string.isRequired,
   downloadUrl: PropTypes.string.isRequired,
   indexerId: PropTypes.number.isRequired,
@@ -335,6 +348,7 @@ SearchIndexRow.propTypes = {
   indexerFlags: PropTypes.arrayOf(PropTypes.string).isRequired,
   columns: PropTypes.arrayOf(PropTypes.object).isRequired,
   onGrabPress: PropTypes.func.isRequired,
+  onSavePress: PropTypes.func.isRequired,
   isGrabbing: PropTypes.bool.isRequired,
   isGrabbed: PropTypes.bool.isRequired,
   grabError: PropTypes.string,

frontend/src/Search/Table/SearchIndexTable.js

@@ -51,7 +51,8 @@ class SearchIndexTable extends Component {
       timeFormat,
       selectedState,
       onSelectedChange,
-      onGrabPress
+      onGrabPress,
+      onSavePress
     } = this.props;
 
     const release = items[rowIndex];
@@ -71,6 +72,7 @@ class SearchIndexTable extends Component {
           longDateFormat={longDateFormat}
           timeFormat={timeFormat}
           onGrabPress={onGrabPress}
+          onSavePress={onSavePress}
         />
       </VirtualTableRow>
     );
@@ -134,6 +136,7 @@ SearchIndexTable.propTypes = {
   timeFormat: PropTypes.string.isRequired,
   onSortPress: PropTypes.func.isRequired,
   onGrabPress: PropTypes.func.isRequired,
+  onSavePress: PropTypes.func.isRequired,
   allSelected: PropTypes.bool.isRequired,
   allUnselected: PropTypes.bool.isRequired,
   selectedState: PropTypes.object.isRequired,

frontend/src/Search/Table/SearchIndexTableConnector.js

@@ -1,6 +1,6 @@
 import { connect } from 'react-redux';
 import { createSelector } from 'reselect';
-import { grabRelease, setReleasesSort } from 'Store/Actions/releaseActions';
+import { grabRelease, saveRelease, setReleasesSort } from 'Store/Actions/releaseActions';
 import createUISettingsSelector from 'Store/Selectors/createUISettingsSelector';
 import SearchIndexTable from './SearchIndexTable';
 
@@ -25,6 +25,9 @@ function createMapDispatchToProps(dispatch, props) {
     },
     onGrabPress(payload) {
       dispatch(grabRelease(payload));
+    },
+    onSavePress(payload) {
+      dispatch(saveRelease(payload));
     }
   };
 }

frontend/src/Settings/Notifications/Notifications/Notification.js

@@ -56,17 +56,9 @@ class Notification extends Component {
       id,
       name,
       onGrab,
-      onDownload,
-      onUpgrade,
-      onRename,
-      onDelete,
       onHealthIssue,
       onApplicationUpdate,
       supportsOnGrab,
-      supportsOnDownload,
-      supportsOnUpgrade,
-      supportsOnRename,
-      supportsOnDelete,
       supportsOnHealthIssue,
       supportsOnApplicationUpdate
     } = this.props;
@@ -88,34 +80,6 @@ class Notification extends Component {
             </Label>
         }
 
-        {
-          supportsOnDelete && onDelete &&
-            <Label kind={kinds.SUCCESS}>
-              {translate('OnDelete')}
-            </Label>
-        }
-
-        {
-          supportsOnDownload && onDownload &&
-            <Label kind={kinds.SUCCESS}>
-              {translate('OnImport')}
-            </Label>
-        }
-
-        {
-          supportsOnUpgrade && onDownload && onUpgrade &&
-            <Label kind={kinds.SUCCESS}>
-              {translate('OnUpgrade')}
-            </Label>
-        }
-
-        {
-          supportsOnRename && onRename &&
-            <Label kind={kinds.SUCCESS}>
-              {translate('OnRename')}
-            </Label>
-        }
-
         {
           supportsOnHealthIssue && onHealthIssue &&
             <Label kind={kinds.SUCCESS}>
@@ -132,7 +96,7 @@ class Notification extends Component {
         }
 
         {
-          !onGrab && !onDownload && !onRename && !onHealthIssue && !onDelete && !onApplicationUpdate ?
+          !onGrab && !onHealthIssue && !onApplicationUpdate ?
             <Label
               kind={kinds.DISABLED}
               outline={true}
@@ -167,17 +131,9 @@ Notification.propTypes = {
   id: PropTypes.number.isRequired,
   name: PropTypes.string.isRequired,
   onGrab: PropTypes.bool.isRequired,
-  onDownload: PropTypes.bool.isRequired,
-  onUpgrade: PropTypes.bool.isRequired,
-  onRename: PropTypes.bool.isRequired,
-  onDelete: PropTypes.bool.isRequired,
   onHealthIssue: PropTypes.bool.isRequired,
   onApplicationUpdate: PropTypes.bool.isRequired,
   supportsOnGrab: PropTypes.bool.isRequired,
-  supportsOnDownload: PropTypes.bool.isRequired,
-  supportsOnDelete: PropTypes.bool.isRequired,
-  supportsOnUpgrade: PropTypes.bool.isRequired,
-  supportsOnRename: PropTypes.bool.isRequired,
   supportsOnHealthIssue: PropTypes.bool.isRequired,
   supportsOnApplicationUpdate: PropTypes.bool.isRequired,
   onConfirmDeleteNotification: PropTypes.func.isRequired

frontend/src/Settings/Notifications/Notifications/NotificationEventItems.js

@@ -15,8 +15,11 @@ function NotificationEventItems(props) {
   } = props;
 
   const {
+    onGrab,
     onHealthIssue,
     onApplicationUpdate,
+    supportsOnGrab,
+    includeManualGrabs,
     supportsOnHealthIssue,
     includeHealthWarnings,
     supportsOnApplicationUpdate
@@ -31,6 +34,31 @@ function NotificationEventItems(props) {
           link="https://wiki.servarr.com/prowlarr/settings#connections"
         />
         <div className={styles.events}>
+          <div>
+            <FormInputGroup
+              type={inputTypes.CHECK}
+              name="onGrab"
+              helpText={translate('OnGrabHelpText')}
+              isDisabled={!supportsOnGrab.value}
+              {...onGrab}
+              onChange={onInputChange}
+            />
+          </div>
+
+          {
+            onGrab.value &&
+              <div>
+                <FormInputGroup
+                  type={inputTypes.CHECK}
+                  name="includeManualGrabs"
+                  helpText={translate('IncludeManualGrabsHelpText')}
+                  isDisabled={!supportsOnGrab.value}
+                  {...includeManualGrabs}
+                  onChange={onInputChange}
+                />
+              </div>
+          }
+
           <div>
             <FormInputGroup
               type={inputTypes.CHECK}

frontend/src/Settings/PendingChangesModal.js

@@ -20,7 +20,7 @@ function PendingChangesModal(props) {
 
   useEffect(() => {
     bindShortcut('enter', onConfirm);
-  }, [onConfirm]);
+  }, [bindShortcut, onConfirm]);
 
   return (
     <Modal

frontend/src/Store/Actions/Settings/notifications.js

@@ -103,9 +103,6 @@ export default {
     [SELECT_NOTIFICATION_SCHEMA]: (state, { payload }) => {
       return selectProviderSchema(state, section, payload, (selectedSchema) => {
         selectedSchema.onGrab = selectedSchema.supportsOnGrab;
-        selectedSchema.onDownload = selectedSchema.supportsOnDownload;
-        selectedSchema.onUpgrade = selectedSchema.supportsOnUpgrade;
-        selectedSchema.onRename = selectedSchema.supportsOnRename;
         selectedSchema.onApplicationUpdate = selectedSchema.supportsOnApplicationUpdate;
 
         return selectedSchema;

frontend/src/Store/Actions/releaseActions.js

@@ -1,3 +1,4 @@
+import $ from 'jquery';
 import { createAction } from 'redux-actions';
 import { batchActions } from 'redux-batched-actions';
 import { filterBuilderTypes, filterBuilderValueTypes, sortDirections } from 'Helpers/Props';
@@ -229,6 +230,7 @@ export const CANCEL_FETCH_RELEASES = 'releases/cancelFetchReleases';
 export const SET_RELEASES_SORT = 'releases/setReleasesSort';
 export const CLEAR_RELEASES = 'releases/clearReleases';
 export const GRAB_RELEASE = 'releases/grabRelease';
+export const SAVE_RELEASE = 'releases/saveRelease';
 export const BULK_GRAB_RELEASES = 'release/bulkGrabReleases';
 export const UPDATE_RELEASE = 'releases/updateRelease';
 export const SET_RELEASES_FILTER = 'releases/setReleasesFilter';
@@ -243,6 +245,7 @@ export const cancelFetchReleases = createThunk(CANCEL_FETCH_RELEASES);
 export const setReleasesSort = createAction(SET_RELEASES_SORT);
 export const clearReleases = createAction(CLEAR_RELEASES);
 export const grabRelease = createThunk(GRAB_RELEASE);
+export const saveRelease = createThunk(SAVE_RELEASE);
 export const bulkGrabReleases = createThunk(BULK_GRAB_RELEASES);
 export const updateRelease = createAction(UPDATE_RELEASE);
 export const setReleasesFilter = createAction(SET_RELEASES_FILTER);
@@ -304,6 +307,32 @@ export const actionHandlers = handleThunks({
     });
   },
 
+  [SAVE_RELEASE]: function(getState, payload, dispatch) {
+    const link = payload.downloadUrl;
+    const file = payload.fileName;
+
+    $.ajax({
+      url: link,
+      method: 'GET',
+      headers: {
+        'X-Prowlarr-Client': true
+      },
+      xhrFields: {
+        responseType: 'blob'
+      },
+      success: function(data) {
+        const a = document.createElement('a');
+        const url = window.URL.createObjectURL(data);
+        a.href = url;
+        a.download = file;
+        document.body.append(a);
+        a.click();
+        a.remove();
+        window.URL.revokeObjectURL(url);
+      }
+    });
+  },
+
   [BULK_GRAB_RELEASES]: function(getState, payload, dispatch) {
     dispatch(set({
       section,

frontend/src/Utilities/createAjaxRequest.js

@@ -16,6 +16,11 @@ function addApiKey(ajaxOptions) {
   ajaxOptions.headers['X-Api-Key'] = window.Prowlarr.apiKey;
 }
 
+function addUIHeader(ajaxOptions) {
+  ajaxOptions.headers = ajaxOptions.headers || {};
+  ajaxOptions.headers['X-Prowlarr-Client'] = true;
+}
+
 function addContentType(ajaxOptions) {
   if (
     ajaxOptions.contentType == null &&
@@ -42,6 +47,7 @@ export default function createAjaxRequest(originalAjaxOptions) {
   if (isRelative(ajaxOptions)) {
     addRootUrl(ajaxOptions);
     addApiKey(ajaxOptions);
+    addUIHeader(ajaxOptions);
     addContentType(ajaxOptions);
   }
 

package.json

@@ -105,6 +105,7 @@
     "eslint-plugin-filenames": "1.3.2",
     "eslint-plugin-import": "2.26.0",
     "eslint-plugin-react": "7.31.11",
+    "eslint-plugin-react-hooks": "4.6.0",
     "eslint-plugin-simple-import-sort": "8.0.0",
     "esprint": "3.6.0",
     "file-loader": "6.2.0",

src/NzbDrone.Common.Test/PathExtensionFixture.cs

@@ -278,7 +278,7 @@ namespace NzbDrone.Common.Test
         [Test]
         public void GetUpdateClientExePath()
         {
-            GetIAppDirectoryInfo().GetUpdateClientExePath(PlatformType.DotNet).Should().BeEquivalentTo(@"C:\Temp\prowlarr_update\Prowlarr.Update.exe".AsOsAgnostic());
+            GetIAppDirectoryInfo().GetUpdateClientExePath().Should().BeEquivalentTo(@"C:\Temp\prowlarr_update\Prowlarr.Update".AsOsAgnostic().ProcessNameToExe());
         }
 
         [Test]

src/NzbDrone.Common/EnvironmentInfo/IRuntimeInfo.cs

@@ -9,6 +9,7 @@ namespace NzbDrone.Common.EnvironmentInfo
         bool IsAdmin { get; }
         bool IsWindowsService { get; }
         bool IsWindowsTray { get; }
+        bool IsStarting { get; set; }
         bool IsExiting { get; set; }
         bool IsTray { get; }
         RuntimeMode Mode { get; }

src/NzbDrone.Common/EnvironmentInfo/PlatformInfo.cs

@@ -2,13 +2,6 @@ using System;
 
 namespace NzbDrone.Common.EnvironmentInfo
 {
-    public enum PlatformType
-    {
-        DotNet = 0,
-        Mono = 1,
-        NetCore = 2
-    }
-
     public interface IPlatformInfo
     {
         Version Version { get; }
@@ -16,36 +9,18 @@ namespace NzbDrone.Common.EnvironmentInfo
 
     public class PlatformInfo : IPlatformInfo
     {
-        private static PlatformType _platform;
         private static Version _version;
 
         static PlatformInfo()
         {
-            _platform = PlatformType.NetCore;
             _version = Environment.Version;
         }
 
-        public static PlatformType Platform => _platform;
-        public static bool IsMono => Platform == PlatformType.Mono;
-        public static bool IsDotNet => Platform == PlatformType.DotNet;
-        public static bool IsNetCore => Platform == PlatformType.NetCore;
-
         public static string PlatformName
         {
             get
             {
-                if (IsDotNet)
-                {
-                    return ".NET";
-                }
-                else if (IsMono)
-                {
-                    return "Mono";
-                }
-                else
-                {
-                    return ".NET Core";
-                }
+                return ".NET";
             }
         }
 

src/NzbDrone.Common/EnvironmentInfo/RuntimeInfo.cs

@@ -19,6 +19,7 @@ namespace NzbDrone.Common.EnvironmentInfo
             _logger = logger;
 
             IsWindowsService = hostLifetime is WindowsServiceLifetime;
+            IsStarting = true;
 
             // net6.0 will return Radarr.dll for entry assembly, we need the actual
             // executable name (Radarr on linux).  On mono this will return the location of
@@ -82,6 +83,7 @@ namespace NzbDrone.Common.EnvironmentInfo
 
         public bool IsWindowsService { get; private set; }
 
+        public bool IsStarting { get; set; }
         public bool IsExiting { get; set; }
         public bool IsTray
         {

src/NzbDrone.Common/Extensions/PathExtensions.cs

@@ -33,7 +33,7 @@ namespace NzbDrone.Common.Extensions
 
             var info = new FileInfo(path.Trim());
 
-            //UNC
+            // UNC
             if (OsInfo.IsWindows && info.FullName.StartsWith(@"\\"))
             {
                 return info.FullName.TrimEnd('/', '\\', ' ');
@@ -166,7 +166,7 @@ namespace NzbDrone.Common.Extensions
             var parentDirInfo = dirInfo.Parent;
             if (parentDirInfo == null)
             {
-                //Drive letter
+                // Drive letter
                 return dirInfo.Name.ToUpper();
             }
 
@@ -238,9 +238,9 @@ namespace NzbDrone.Common.Extensions
             return null;
         }
 
-        public static string ProcessNameToExe(this string processName, PlatformType runtime)
+        public static string ProcessNameToExe(this string processName)
         {
-            if (OsInfo.IsWindows || runtime != PlatformType.NetCore)
+            if (OsInfo.IsWindows)
             {
                 processName += ".exe";
             }
@@ -248,11 +248,6 @@ namespace NzbDrone.Common.Extensions
             return processName;
         }
 
-        public static string ProcessNameToExe(this string processName)
-        {
-            return processName.ProcessNameToExe(PlatformInfo.Platform);
-        }
-
         public static string GetAppDataPath(this IAppFolderInfo appFolderInfo)
         {
             return appFolderInfo.AppDataFolder;
@@ -318,9 +313,9 @@ namespace NzbDrone.Common.Extensions
             return Path.Combine(GetUpdatePackageFolder(appFolderInfo), UPDATE_CLIENT_FOLDER_NAME);
         }
 
-        public static string GetUpdateClientExePath(this IAppFolderInfo appFolderInfo, PlatformType runtime)
+        public static string GetUpdateClientExePath(this IAppFolderInfo appFolderInfo)
         {
-            return Path.Combine(GetUpdateSandboxFolder(appFolderInfo), UPDATE_CLIENT_EXE_NAME).ProcessNameToExe(runtime);
+            return Path.Combine(GetUpdateSandboxFolder(appFolderInfo), UPDATE_CLIENT_EXE_NAME).ProcessNameToExe();
         }
 
         public static string GetDatabase(this IAppFolderInfo appFolderInfo)

src/NzbDrone.Common/Extensions/StringExtensions.cs

@@ -257,5 +257,18 @@ namespace NzbDrone.Common.Extensions
         {
             return input.Contains(':') ? $"[{input}]" : input;
         }
+
+        public static bool IsAllDigits(this string input)
+        {
+            foreach (var c in input)
+            {
+                if (c < '0' || c > '9')
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
     }
 }

src/NzbDrone.Common/Http/TooManyRequestsException.cs

@@ -11,15 +11,13 @@ namespace NzbDrone.Common.Http
         {
             if (response.Headers.ContainsKey("Retry-After"))
             {
-                var retryAfter = response.Headers["Retry-After"].ToString();
-                int seconds;
-                DateTime date;
+                var retryAfter = response.Headers["Retry-After"];
 
-                if (int.TryParse(retryAfter, out seconds))
+                if (int.TryParse(retryAfter, out var seconds))
                 {
                     RetryAfter = TimeSpan.FromSeconds(seconds);
                 }
-                else if (DateTime.TryParse(retryAfter, out date))
+                else if (DateTime.TryParse(retryAfter, out var date))
                 {
                     RetryAfter = date.ToUniversalTime() - DateTime.UtcNow;
                 }

src/NzbDrone.Common/Http/UserAgentParser.cs

@@ -1,9 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
 using System.Text.RegularExpressions;
-using System.Threading.Tasks;
 
 namespace NzbDrone.Common.Http
 {

src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs

@@ -9,7 +9,7 @@ using NUnit.Framework;
 using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Indexers;
-using NzbDrone.Core.Indexers.Rarbg;
+using NzbDrone.Core.Indexers.Definitions.Rarbg;
 using NzbDrone.Core.IndexerSearch.Definitions;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.Test.Framework;
@@ -23,14 +23,14 @@ namespace NzbDrone.Core.Test.IndexerTests.RarbgTests
         [SetUp]
         public void Setup()
         {
-            Subject.Definition = new IndexerDefinition()
+            Subject.Definition = new IndexerDefinition
             {
                 Name = "Rarbg",
                 Settings = new RarbgSettings()
             };
 
             Mocker.GetMock<IRarbgTokenProvider>()
-                .Setup(v => v.GetToken(It.IsAny<RarbgSettings>()))
+                .Setup(v => v.GetToken(It.IsAny<RarbgSettings>(), Subject.RateLimit))
                 .Returns("validtoken");
         }
 
@@ -53,7 +53,7 @@ namespace NzbDrone.Core.Test.IndexerTests.RarbgTests
             torrentInfo.Title.Should().Be("Sense8.S01E01.WEBRip.x264-FGT");
             torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent);
             torrentInfo.DownloadUrl.Should().Be("magnet:?xt=urn:btih:d8bde635f573acb390c7d7e7efc1556965fdc802&dn=Sense8.S01E01.WEBRip.x264-FGT&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce");
-            torrentInfo.InfoUrl.Should().Be($"https://torrentapi.org/redirect_to_info.php?token=i5cx7b9agd&p=8_6_4_4_5_6__d8bde635f5&app_id={BuildInfo.AppName}");
+            torrentInfo.InfoUrl.Should().Be($"https://torrentapi.org/redirect_to_info.php?token=i5cx7b9agd&p=8_6_4_4_5_6__d8bde635f5&app_id=rralworP_{BuildInfo.Version}");
             torrentInfo.Indexer.Should().Be(Subject.Definition.Name);
             torrentInfo.PublishDate.Should().Be(DateTime.Parse("2015-06-05 16:58:11 +0000").ToUniversalTime());
             torrentInfo.Size.Should().Be(564198371);

src/NzbDrone.Core.Test/NotificationTests/NotificationBaseFixture.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using FluentAssertions;
 using FluentValidation.Results;
 using NUnit.Framework;
@@ -56,6 +55,11 @@ namespace NzbDrone.Core.Test.NotificationTests
             {
                 TestLogger.Info("OnApplicationUpdate was called");
             }
+
+            public override void OnGrab(GrabMessage message)
+            {
+                TestLogger.Info("OnGrab was called");
+            }
         }
 
         private class TestNotificationWithNoEvents : NotificationBase<TestSetting>
@@ -76,6 +80,7 @@ namespace NzbDrone.Core.Test.NotificationTests
 
             notification.SupportsOnHealthIssue.Should().BeTrue();
             notification.SupportsOnApplicationUpdate.Should().BeTrue();
+            notification.SupportsOnGrab.Should().BeTrue();
         }
 
         [Test]
@@ -85,6 +90,7 @@ namespace NzbDrone.Core.Test.NotificationTests
 
             notification.SupportsOnHealthIssue.Should().BeFalse();
             notification.SupportsOnApplicationUpdate.Should().BeFalse();
+            notification.SupportsOnGrab.Should().BeFalse();
         }
     }
 }

src/NzbDrone.Core.Test/ParserTests/DateTimeUtilFixture.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Globalization;
+using System.Threading;
+using FluentAssertions;
+using NUnit.Framework;
+using NzbDrone.Core.Parser;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.ParserTests
+{
+    [TestFixture]
+    public class DateTimeUtilFixture : CoreTest
+    {
+        [TestCase("pt-BR")]
+        [TestCase("en-US")]
+        public void should_format_date_invariant(string culture)
+        {
+            Thread.CurrentThread.CurrentCulture = new CultureInfo(culture);
+
+            var dateNow = DateTime.Now;
+
+            DateTimeUtil.FromUnknown(dateNow.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture))
+                .ToString(DateTimeUtil.Rfc1123ZPattern, CultureInfo.InvariantCulture)
+                .Should().Be(dateNow.ToString("ddd, dd MMM yyyy HH':'mm':'ss z", CultureInfo.InvariantCulture));
+        }
+
+        [TestCase("2022-08-08 02:07:39 -02:00", "2006-01-02 15:04:05 -07:00", "yyyy-MM-dd HH:mm:ss zzz", "2022-08-08 04:07:39 +00:00")]
+        [TestCase("2022-08-08 02:07:39 -02:00", "yyyy-MM-dd HH:mm:ss zzz", "yyyy-MM-dd HH:mm:ss zzz", "2022-08-08 04:07:39 +00:00")]
+        [TestCase("2022-08-08 -02:00", "2006-01-02 -07:00", "yyyy-MM-dd zzz", "2022-08-08 +00:00")]
+        [TestCase("2022-08-08 -02:00", "yyyy-MM-dd zzz", "yyyy-MM-dd zzz", "2022-08-08 +00:00")]
+        [TestCase("02:07:39 -02:00", "15:04:05 -07:00", "HH:mm:ss zzz", "04:07:39 +00:00")]
+        [TestCase("02:07:39 -02:00", "HH:mm:ss zzz", "HH:mm:ss zzz", "04:07:39 +00:00")]
+        [TestCase("-02:00", "zzz", "zzz", "+00:00")]
+        [TestCase("-02:00", "-07:00", "zzz", "+00:00")]
+        public void parse_datetime_golang(string dateInput, string format, string standardFormat, string expectedDate)
+        {
+            DateTimeUtil.ParseDateTimeGoLang(dateInput, format)
+                .ToUniversalTime()
+                .ToString(standardFormat, CultureInfo.InvariantCulture)
+                .Should().Be(expectedDate);
+        }
+    }
+}

src/NzbDrone.Core.Test/ParserTests/ParseUtilFixture.cs

@@ -52,5 +52,16 @@ namespace NzbDrone.Core.Test.ParserTests
         {
             ParseUtil.CoerceDouble(original).Should().Be(parsedInt);
         }
+
+        [TestCase(null, null)]
+        [TestCase("", null)]
+        [TestCase("1", 1)]
+        [TestCase("1000 grabs", 1000)]
+        [TestCase("asdf123asdf", 123)]
+        [TestCase("asdf123asdf456asdf", 123)]
+        public void should_parse_long_from_string(string original, long? parsedInt)
+        {
+            ParseUtil.GetLongFromString(original).Should().Be(parsedInt);
+        }
     }
 }

src/NzbDrone.Core.Test/UpdateTests/UpdatePackageProviderFixture.cs

@@ -40,12 +40,11 @@ namespace NzbDrone.Core.Test.UpdateTests
         }
 
         [Test]
-        [Ignore("TODO No Updates On Server")]
         public void should_get_recent_updates()
         {
             const string branch = "develop";
             UseRealHttp();
-            var recent = Subject.GetRecentUpdates(branch, new Version(2, 0), null);
+            var recent = Subject.GetRecentUpdates(branch, new Version(1, 0), null);
             var recentWithChanges = recent.Where(c => c.Changes != null);
 
             recent.Should().NotBeEmpty();

src/NzbDrone.Core.Test/UpdateTests/UpdateServiceFixture.cs

@@ -69,7 +69,7 @@ namespace NzbDrone.Core.Test.UpdateTests
                   .Returns(true);
 
             Mocker.GetMock<IDiskProvider>()
-                  .Setup(v => v.FileExists(It.Is<string>(s => s.EndsWith("Prowlarr.Update.exe"))))
+                  .Setup(v => v.FileExists(It.Is<string>(s => s.EndsWith("Prowlarr.Update".ProcessNameToExe()))))
                   .Returns(true);
 
             _sandboxFolder = Mocker.GetMock<IAppFolderInfo>().Object.GetUpdateSandboxFolder();
@@ -165,7 +165,7 @@ namespace NzbDrone.Core.Test.UpdateTests
         public void should_return_with_warning_if_updater_doesnt_exists()
         {
             Mocker.GetMock<IDiskProvider>()
-                  .Setup(v => v.FileExists(It.Is<string>(s => s.EndsWith("Prowlarr.Update.exe"))))
+                  .Setup(v => v.FileExists(It.Is<string>(s => s.EndsWith("Prowlarr.Update".ProcessNameToExe()))))
                   .Returns(false);
 
             Subject.Execute(new ApplicationUpdateCommand());

src/NzbDrone.Core/Datastore/Migration/029_add_on_grab_to_notifications.cs

@@ -0,0 +1,15 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(029)]
+    public class add_on_grab_to_notifications : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Alter.Table("Notifications").AddColumn("OnGrab").AsBoolean().WithDefaultValue(false);
+            Alter.Table("Notifications").AddColumn("IncludeManualGrabs").AsBoolean().WithDefaultValue(false).NotNullable();
+        }
+    }
+}

src/NzbDrone.Core/Datastore/TableMapping.cs

@@ -66,6 +66,7 @@ namespace NzbDrone.Core.Datastore
 
             Mapper.Entity<NotificationDefinition>("Notifications").RegisterModel()
                   .Ignore(x => x.ImplementationName)
+                  .Ignore(i => i.SupportsOnGrab)
                   .Ignore(i => i.SupportsOnHealthIssue)
                   .Ignore(i => i.SupportsOnApplicationUpdate);
 

src/NzbDrone.Core/Download/DownloadService.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Threading.Tasks;
 using NLog;
-using NzbDrone.Common.EnsureThat;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Common.Instrumentation.Extensions;
@@ -61,16 +60,17 @@ namespace NzbDrone.Core.Download
 
             // Get the seed configuration for this release.
             // remoteMovie.SeedConfiguration = _seedConfigProvider.GetSeedConfiguration(remoteMovie);
-
-            // Limit grabs to 2 per second.
-            if (release.DownloadUrl.IsNotNullOrWhiteSpace() && !release.DownloadUrl.StartsWith("magnet:"))
-            {
-                var url = new HttpUri(release.DownloadUrl);
-                _rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2));
-            }
-
             var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(release.IndexerId));
 
+            var grabEvent = new IndexerDownloadEvent(release, true, source, host, release.Title, release.DownloadUrl)
+            {
+                DownloadClient = downloadClient.Name,
+                DownloadClientId = downloadClient.Definition.Id,
+                DownloadClientName = downloadClient.Definition.Name,
+                Redirect = redirect,
+                GrabTrigger = source == "Prowlarr" ? GrabTrigger.Manual : GrabTrigger.Api
+            };
+
             string downloadClientId;
             try
             {
@@ -81,19 +81,20 @@ namespace NzbDrone.Core.Download
             catch (ReleaseUnavailableException)
             {
                 _logger.Trace("Release {0} no longer available on indexer.", release);
-                _eventAggregator.PublishEvent(new IndexerDownloadEvent(release.IndexerId, false, source, host, release.Title, release.DownloadUrl, redirect));
+                grabEvent.Successful = false;
+                _eventAggregator.PublishEvent(grabEvent);
                 throw;
             }
             catch (DownloadClientRejectedReleaseException)
             {
                 _logger.Trace("Release {0} rejected by download client, possible duplicate.", release);
-                _eventAggregator.PublishEvent(new IndexerDownloadEvent(release.IndexerId, false, source, host, release.Title, release.DownloadUrl, redirect));
+                grabEvent.Successful = false;
+                _eventAggregator.PublishEvent(grabEvent);
                 throw;
             }
             catch (ReleaseDownloadException ex)
             {
-                var http429 = ex.InnerException as TooManyRequestsException;
-                if (http429 != null)
+                if (ex.InnerException is TooManyRequestsException http429)
                 {
                     _indexerStatusService.RecordFailure(release.IndexerId, http429.RetryAfter);
                 }
@@ -102,14 +103,21 @@ namespace NzbDrone.Core.Download
                     _indexerStatusService.RecordFailure(release.IndexerId);
                 }
 
-                _eventAggregator.PublishEvent(new IndexerDownloadEvent(release.IndexerId, false, source, host, release.Title, release.DownloadUrl, redirect));
+                grabEvent.Successful = false;
+
+                _eventAggregator.PublishEvent(grabEvent);
 
                 throw;
             }
 
             _logger.ProgressInfo("Report sent to {0}. {1}", downloadClient.Definition.Name, downloadTitle);
 
-            _eventAggregator.PublishEvent(new IndexerDownloadEvent(release.IndexerId, true, source, host, release.Title, release.DownloadUrl, redirect));
+            if (!string.IsNullOrWhiteSpace(downloadClientId))
+            {
+                grabEvent.DownloadId = downloadClientId;
+            }
+
+            _eventAggregator.PublishEvent(grabEvent);
         }
 
         public async Task<byte[]> DownloadReport(string link, int indexerId, string source, string host, string title)
@@ -127,22 +135,35 @@ namespace NzbDrone.Core.Download
             var success = false;
             var downloadedBytes = Array.Empty<byte>();
 
+            var release = new ReleaseInfo
+            {
+                Title = title,
+                DownloadUrl = link,
+                IndexerId = indexerId,
+                Indexer = indexer.Definition.Name,
+                DownloadProtocol = indexer.Protocol
+            };
+
+            var grabEvent = new IndexerDownloadEvent(release, success, source, host, release.Title, release.DownloadUrl)
+            {
+                GrabTrigger = source == "Prowlarr" ? GrabTrigger.Manual : GrabTrigger.Api
+            };
+
             try
             {
                 downloadedBytes = await indexer.Download(url);
                 _indexerStatusService.RecordSuccess(indexerId);
-                success = true;
+                grabEvent.Successful = true;
             }
             catch (ReleaseUnavailableException)
             {
                 _logger.Trace("Release {0} no longer available on indexer.", link);
-                _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source, host, title, url.AbsoluteUri));
+                _eventAggregator.PublishEvent(grabEvent);
                 throw;
             }
             catch (ReleaseDownloadException ex)
             {
-                var http429 = ex.InnerException as TooManyRequestsException;
-                if (http429 != null)
+                if (ex.InnerException is TooManyRequestsException http429)
                 {
                     _indexerStatusService.RecordFailure(indexerId, http429.RetryAfter);
                 }
@@ -151,19 +172,36 @@ namespace NzbDrone.Core.Download
                     _indexerStatusService.RecordFailure(indexerId);
                 }
 
-                _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source, host, title, url.AbsoluteUri));
+                _eventAggregator.PublishEvent(grabEvent);
                 throw;
             }
 
             _logger.Trace("Downloaded {0} bytes from {1}", downloadedBytes.Length, link);
-            _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source, host, title, url.AbsoluteUri));
+            _eventAggregator.PublishEvent(grabEvent);
 
             return downloadedBytes;
         }
 
         public void RecordRedirect(string link, int indexerId, string source, string host, string title)
         {
-            _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, true, source, host, title, link, true));
+            var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(indexerId));
+
+            var release = new ReleaseInfo
+            {
+                Title = title,
+                DownloadUrl = link,
+                IndexerId = indexerId,
+                Indexer = indexer.Definition.Name,
+                DownloadProtocol = indexer.Protocol
+            };
+
+            var grabEvent = new IndexerDownloadEvent(release, true, source, host, release.Title, release.DownloadUrl)
+            {
+                Redirect = true,
+                GrabTrigger = source == "Prowlarr" ? GrabTrigger.Manual : GrabTrigger.Api
+            };
+
+            _eventAggregator.PublishEvent(grabEvent);
         }
     }
 }

src/NzbDrone.Core/HealthCheck/ServerSideNotificationService.cs

@@ -52,7 +52,7 @@ namespace NzbDrone.Core.HealthCheck
                                       .AddQueryParam("version", BuildInfo.Version)
                                       .AddQueryParam("os", OsInfo.Os.ToString().ToLowerInvariant())
                                       .AddQueryParam("arch", RuntimeInformation.OSArchitecture)
-                                      .AddQueryParam("runtime", PlatformInfo.Platform.ToString().ToLowerInvariant())
+                                      .AddQueryParam("runtime", "netcore")
                                       .AddQueryParam("branch", _configFileProvider.Branch)
                                       .Build();
             try

src/NzbDrone.Core/History/HistoryRepository.cs

@@ -19,6 +19,7 @@ namespace NzbDrone.Core.History
         List<History> Since(DateTime date, HistoryEventType? eventType);
         void Cleanup(int days);
         int CountSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes);
+        History FindFirstForIndexerSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes, int limit);
     }
 
     public class HistoryRepository : BasicRepository<History>, IHistoryRepository
@@ -115,5 +116,24 @@ namespace NzbDrone.Core.History
                 return conn.ExecuteScalar<int>(sql.RawSql, sql.Parameters);
             }
         }
+
+        public History FindFirstForIndexerSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes, int limit)
+        {
+            var intEvents = eventTypes.Select(t => (int)t).ToList();
+
+            var builder = Builder()
+                .Where<History>(x => x.IndexerId == indexerId)
+                .Where<History>(x => x.Date >= date)
+                .Where<History>(x => intEvents.Contains((int)x.EventType));
+
+            var query = Query(builder);
+
+            if (limit > 0)
+            {
+                query = query.OrderByDescending(h => h.Date).Take(limit).ToList();
+            }
+
+            return query.MinBy(h => h.Date);
+        }
     }
 }

src/NzbDrone.Core/History/HistoryService.cs

@@ -27,6 +27,7 @@ namespace NzbDrone.Core.History
         List<History> Between(DateTime start, DateTime end);
         List<History> Since(DateTime date, HistoryEventType? eventType);
         int CountSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes);
+        History FindFirstForIndexerSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes, int limit);
     }
 
     public class HistoryService : IHistoryService,
@@ -121,7 +122,7 @@ namespace NzbDrone.Core.History
             {
                 Date = DateTime.UtcNow,
                 IndexerId = message.IndexerId,
-                EventType = message.Query.RssSearch ? HistoryEventType.IndexerRss : HistoryEventType.IndexerQuery,
+                EventType = message.Query.IsRssSearch ? HistoryEventType.IndexerRss : HistoryEventType.IndexerQuery,
                 Successful = message.QueryResult.Response?.StatusCode == HttpStatusCode.OK
             };
 
@@ -184,7 +185,7 @@ namespace NzbDrone.Core.History
             var history = new History
             {
                 Date = DateTime.UtcNow,
-                IndexerId = message.IndexerId,
+                IndexerId = message.Release.IndexerId,
                 EventType = HistoryEventType.ReleaseGrabbed,
                 Successful = message.Successful
             };
@@ -232,5 +233,10 @@ namespace NzbDrone.Core.History
         {
             return _historyRepository.CountSince(indexerId, date, eventTypes);
         }
+
+        public History FindFirstForIndexerSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes, int limit)
+        {
+            return _historyRepository.FindFirstForIndexerSince(indexerId, date, eventTypes, limit);
+        }
     }
 }

src/NzbDrone.Core/IndexerSearch/Definitions/BookSearchCriteria.cs

@@ -10,6 +10,15 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
         public int? Year { get; set; }
         public string Genre { get; set; }
 
-        public override bool RssSearch => SearchTerm.IsNullOrWhiteSpace() && Author.IsNullOrWhiteSpace() && Title.IsNullOrWhiteSpace();
+        public override bool IsRssSearch =>
+            SearchTerm.IsNullOrWhiteSpace() &&
+            !IsIdSearch;
+
+        public override bool IsIdSearch =>
+            Author.IsNotNullOrWhiteSpace() ||
+            Title.IsNotNullOrWhiteSpace() ||
+            Publisher.IsNotNullOrWhiteSpace() ||
+            Genre.IsNotNullOrWhiteSpace() ||
+            Year.HasValue;
     }
 }

src/NzbDrone.Core/IndexerSearch/Definitions/MovieSearchCriteria.cs

@@ -13,7 +13,17 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
         public int? Year { get; set; }
         public string Genre { get; set; }
 
-        public override bool RssSearch => SearchTerm.IsNullOrWhiteSpace() && ImdbId.IsNullOrWhiteSpace() && !TmdbId.HasValue && !TraktId.HasValue;
+        public override bool IsRssSearch =>
+            SearchTerm.IsNullOrWhiteSpace() &&
+            !IsIdSearch;
+
+        public override bool IsIdSearch =>
+            ImdbId.IsNotNullOrWhiteSpace() ||
+            Genre.IsNotNullOrWhiteSpace() ||
+            TmdbId.HasValue ||
+            TraktId.HasValue ||
+            DoubanId.HasValue ||
+            Year.HasValue;
 
         public string FullImdbId => ParseUtil.GetFullImdbId(ImdbId);
 

src/NzbDrone.Core/IndexerSearch/Definitions/MusicSearchCriteria.cs

@@ -11,6 +11,16 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
         public string Track { get; set; }
         public int? Year { get; set; }
 
-        public override bool RssSearch => SearchTerm.IsNullOrWhiteSpace() && Album.IsNullOrWhiteSpace() && Artist.IsNullOrWhiteSpace() && Label.IsNullOrWhiteSpace();
+        public override bool IsRssSearch =>
+            SearchTerm.IsNullOrWhiteSpace() &&
+            !IsIdSearch;
+
+        public override bool IsIdSearch =>
+            Album.IsNotNullOrWhiteSpace() ||
+            Artist.IsNotNullOrWhiteSpace() ||
+            Label.IsNotNullOrWhiteSpace() ||
+            Genre.IsNotNullOrWhiteSpace() ||
+            Track.IsNotNullOrWhiteSpace() ||
+            Year.HasValue;
     }
 }

src/NzbDrone.Core/IndexerSearch/Definitions/SearchCriteriaBase.cs

@@ -24,7 +24,9 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
 
         public virtual string SearchQuery => $"Term: [{SearchTerm}]";
 
-        public virtual bool RssSearch => SearchTerm.IsNullOrWhiteSpace();
+        public virtual bool IsRssSearch => SearchTerm.IsNullOrWhiteSpace();
+
+        public virtual bool IsIdSearch => false;
 
         public string SanitizedSearchTerm => GetSanitizedTerm(SearchTerm);
 

src/NzbDrone.Core/IndexerSearch/Definitions/TvSearchCriteria.cs

@@ -26,7 +26,20 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
 
         public string FullImdbId => ParseUtil.GetFullImdbId(ImdbId);
 
-        public override bool RssSearch => SearchTerm.IsNullOrWhiteSpace() && ImdbId.IsNullOrWhiteSpace() && !TvdbId.HasValue && !RId.HasValue && !TraktId.HasValue && !TvMazeId.HasValue;
+        public override bool IsRssSearch =>
+            SearchTerm.IsNullOrWhiteSpace() &&
+            !IsIdSearch;
+
+        public override bool IsIdSearch =>
+            Episode.IsNotNullOrWhiteSpace() ||
+            ImdbId.IsNotNullOrWhiteSpace() ||
+            Season.HasValue ||
+            TvdbId.HasValue ||
+            RId.HasValue ||
+            TraktId.HasValue ||
+            TvMazeId.HasValue ||
+            TmdbId.HasValue ||
+            DoubanId.HasValue;
 
         public override string SearchQuery
         {

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -1,17 +1,16 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Net;
-using System.Text;
 using System.Threading.Tasks;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
-using NzbDrone.Core.Exceptions;
+using NzbDrone.Core.IndexerSearch.Definitions;
 using NzbDrone.Core.IndexerVersions;
 using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
 
@@ -53,7 +52,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
             var generator = _generatorCache.Get(Settings.DefinitionFile, () =>
                 new CardigannRequestGenerator(_configService,
                     _definitionService.GetCachedDefinition(Settings.DefinitionFile),
-                    _logger)
+                    _logger,
+                    RateLimit)
                 {
                     HttpClient = _httpClient,
                     Definition = Definition,
@@ -79,6 +79,18 @@ namespace NzbDrone.Core.Indexers.Cardigann
             };
         }
 
+        protected override IList<ReleaseInfo> CleanupReleases(IEnumerable<ReleaseInfo> releases, SearchCriteriaBase searchCriteria)
+        {
+            var cleanReleases = base.CleanupReleases(releases, searchCriteria);
+
+            if (_definitionService.GetCachedDefinition(Settings.DefinitionFile).Search?.Rows?.Filters?.Any(x => x.Name == "andmatch") ?? false)
+            {
+                cleanReleases = FilterReleasesByQuery(releases, searchCriteria).ToList();
+            }
+
+            return cleanReleases;
+        }
+
         protected override IDictionary<string, string> GetCookies()
         {
             if (Settings.ExtraFieldData.TryGetValue("cookie", out var cookies))
@@ -117,8 +129,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
         {
             var defaultSettings = new List<SettingsField>
             {
-                new SettingsField { Name = "username", Label = "Username", Type = "text" },
-                new SettingsField { Name = "password", Label = "Password", Type = "password" }
+                new () { Name = "username", Label = "Username", Type = "text" },
+                new () { Name = "password", Label = "Password", Type = "password" }
             };
 
             var settings = definition.Settings ?? defaultSettings;
@@ -180,63 +192,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
             await generator.DoLogin();
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override async Task<HttpRequest> GetDownloadRequest(Uri link)
         {
             var generator = (CardigannRequestGenerator)GetRequestGenerator();
 
             var request = await generator.DownloadRequest(link);
 
-            if (request.Url.Scheme == "magnet")
-            {
-                ValidateMagnet(request.Url.FullUri);
-
-                return Encoding.UTF8.GetBytes(request.Url.FullUri);
-            }
-
             request.AllowAutoRedirect = true;
 
-            var downloadBytes = Array.Empty<byte>();
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                downloadBytes = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", request.Url.FullUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            ValidateTorrent(downloadBytes);
-
-            return downloadBytes;
+            return request;
         }
 
         protected override async Task Test(List<ValidationFailure> failures)

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using System.Net;
 using System.Text;
@@ -300,7 +301,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 }
                 else if (setting.Type == "checkbox")
                 {
-                    variables[name] = ((bool)value) ? ".True" : null;
+                    if (value is string stringValue && bool.TryParse(stringValue, out var result))
+                    {
+                        value = result;
+                    }
+
+                    variables[name] = (bool)value ? ".True" : null;
                 }
                 else if (setting.Type == "select")
                 {
@@ -328,12 +334,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 }
                 else
                 {
-                    throw new NotSupportedException();
+                    throw new NotSupportedException($"Type {setting.Type} is not supported.");
                 }
 
-                if (setting.Type != "password" && setting.Name != "apikey" && setting.Name != "rsskey" && indexerLogging)
+                if (setting.Type != "password" && setting.Name != "apikey" && setting.Name != "rsskey" && indexerLogging && variables.ContainsKey(name))
                 {
-                    _logger.Debug($"Setting {setting.Name} to {variables[name]}");
+                    _logger.Debug($"Setting {setting.Name} to {variables[name].ToJson()}");
                 }
             }
 
@@ -344,11 +350,13 @@ namespace NzbDrone.Core.Indexers.Cardigann
 
         public string ApplyGoTemplateText(string template, Dictionary<string, object> variables = null, TemplateTextModifier modifier = null)
         {
-            if (variables == null)
+            if (template.IsNullOrWhiteSpace() || !template.Contains("{{"))
             {
-                variables = GetBaseTemplateVariables();
+                return template;
             }
 
+            variables ??= GetBaseTemplateVariables();
+
             // handle re_replace expression
             // Example: {{ re_replace .Query.Keywords "[^a-zA-Z0-9]+" "%" }}
             var reReplaceRegex = new Regex(@"{{\s*re_replace\s+(\..+?)\s+""(.*?)""\s+""(.*?)""\s*}}");
@@ -606,10 +614,11 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     case "timeparse":
                     case "dateparse":
                         var layout = (string)filter.Args;
+
                         try
                         {
                             var date = DateTimeUtil.ParseDateTimeGoLang(data, layout);
-                            data = date.ToString(DateTimeUtil.Rfc1123ZPattern);
+                            data = date.ToString(DateTimeUtil.Rfc1123ZPattern, CultureInfo.InvariantCulture);
                         }
                         catch (InvalidDateException ex)
                         {
@@ -650,15 +659,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         break;
                     case "trim":
                         var cutset = (string)filter.Args;
-                        if (cutset != null)
-                        {
-                            data = data.Trim(cutset[0]);
-                        }
-                        else
-                        {
-                            data = data.Trim();
-                        }
-
+                        data = cutset != null ? data.Trim(cutset[0]) : data.Trim();
                         break;
                     case "prepend":
                         var prependstr = (string)filter.Args;
@@ -688,10 +689,10 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         break;
                     case "timeago":
                     case "reltime":
-                        data = DateTimeUtil.FromTimeAgo(data).ToString(DateTimeUtil.Rfc1123ZPattern);
+                        data = DateTimeUtil.FromTimeAgo(data).ToString(DateTimeUtil.Rfc1123ZPattern, CultureInfo.InvariantCulture);
                         break;
                     case "fuzzytime":
-                        data = DateTimeUtil.FromUnknown(data).ToString(DateTimeUtil.Rfc1123ZPattern);
+                        data = DateTimeUtil.FromUnknown(data).ToString(DateTimeUtil.Rfc1123ZPattern, CultureInfo.InvariantCulture);
                         break;
                     case "validfilename":
                         data = StringUtil.MakeValidFileName(data, '_', false);
@@ -739,18 +740,20 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         // for debugging
                         var debugData = data.Replace("\r", "\\r").Replace("\n", "\\n").Replace("\xA0", "\\xA0");
                         var strTag = (string)filter.Args;
-                        if (strTag != null)
-                        {
-                            strTag = string.Format("({0}):", strTag);
-                        }
-                        else
-                        {
-                            strTag = ":";
-                        }
+                        strTag = strTag != null ? $"({strTag}):" : ":";
 
-                        _logger.Debug(string.Format("CardigannIndexer ({0}): strdump{1} {2}", _definition.Id, strTag, debugData));
+                        _logger.Debug($"CardigannIndexer ({_definition.Id}): strdump{strTag} {debugData}");
+                        break;
+                    case "validate":
+                        char[] delimiters = { ',', ' ', '/', ')', '(', '.', ';', '[', ']', '"', '|', ':' };
+                        var args = (string)filter.Args;
+                        var argsList = args.ToLower().Split(delimiters, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
+                        var validList = argsList.ToList();
+                        var validIntersect = validList.Intersect(data.ToLower().Split(delimiters, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)).ToList();
+                        data = string.Join(", ", validIntersect);
                         break;
                     default:
+                        _logger.Error($"CardigannIndexer ({_definition.Id}): Unsupported field filter: {filter.Name}");
                         break;
                 }
             }
@@ -758,8 +761,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
             return data;
         }
 
-        protected Dictionary<string, string> ParseCustomHeaders(Dictionary<string, List<string>> customHeaders,
-                                                      Dictionary<string, object> variables)
+        protected Dictionary<string, string> ParseCustomHeaders(Dictionary<string, List<string>> customHeaders, Dictionary<string, object> variables)
         {
             if (customHeaders == null)
             {

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinition.cs

@@ -102,6 +102,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
         public List<ErrorBlock> Error { get; set; }
         public PageTestBlock Test { get; set; }
         public CaptchaBlock Captcha { get; set; }
+        public Dictionary<string, List<string>> Headers { get; set; }
     }
 
     public class ErrorBlock
@@ -182,20 +183,21 @@ namespace NzbDrone.Core.Indexers.Cardigann
         public string Method { get; set; }
         public BeforeBlock Before { get; set; }
         public InfohashBlock Infohash { get; set; }
+        public Dictionary<string, List<string>> Headers { get; set; }
     }
 
     public class InfohashBlock
     {
         public SelectorField Hash { get; set; }
         public SelectorField Title { get; set; }
-        public bool UseBeforeResponse { get; set; }
+        public bool Usebeforeresponse { get; set; }
     }
 
     public class SelectorField
     {
         public string Selector { get; set; }
         public string Attribute { get; set; }
-        public bool UseBeforeResponse { get; set; }
+        public bool Usebeforeresponse { get; set; }
         public List<FilterBlock> Filters { get; set; }
     }
 

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using System.Net;
 using System.Text.RegularExpressions;
@@ -72,6 +73,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 }
 
                 var parsedJson = JToken.Parse(results);
+
                 if (parsedJson == null)
                 {
                     throw new IndexerException(indexerResponse, "Error Parsing Json Response");
@@ -80,12 +82,10 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 if (search.Rows.Count != null)
                 {
                     var countVal = HandleJsonSelector(search.Rows.Count, parsedJson, variables);
-                    if (int.TryParse(countVal, out var count))
+
+                    if (int.TryParse(countVal, out var count) && count < 1)
                     {
-                        if (count < 1)
-                        {
-                            return releases;
-                        }
+                        return releases;
                     }
                 }
 
@@ -165,196 +165,188 @@ namespace NzbDrone.Core.Indexers.Cardigann
             }
             else
             {
-                try
+                IHtmlCollection<IElement> rowsDom;
+
+                if (request.SearchPath.Response != null && request.SearchPath.Response.Type.Equals("xml"))
                 {
-                    IHtmlCollection<IElement> rowsDom;
+                    var searchResultParser = new XmlParser();
+                    var searchResultDocument = searchResultParser.ParseDocument(results);
 
-                    if (request.SearchPath.Response != null && request.SearchPath.Response.Type.Equals("xml"))
+                    if (search.Preprocessingfilters != null)
                     {
-                        var searchResultParser = new XmlParser();
-                        var searchResultDocument = searchResultParser.ParseDocument(results);
-
-                        if (search.Preprocessingfilters != null)
-                        {
-                            results = ApplyFilters(results, search.Preprocessingfilters, variables);
-                            searchResultDocument = searchResultParser.ParseDocument(results);
-                            _logger.Trace(string.Format("CardigannIndexer ({0}): result after preprocessingfilters: {1}", _definition.Id, results));
-                        }
-
-                        var rowsSelector = ApplyGoTemplateText(search.Rows.Selector, variables);
-                        rowsDom = searchResultDocument.QuerySelectorAll(rowsSelector);
-                    }
-                    else
-                    {
-                        var searchResultParser = new HtmlParser();
-                        var searchResultDocument = searchResultParser.ParseDocument(results);
-
-                        if (search.Preprocessingfilters != null)
-                        {
-                            results = ApplyFilters(results, search.Preprocessingfilters, variables);
-                            searchResultDocument = searchResultParser.ParseDocument(results);
-                            _logger.Trace(string.Format("CardigannIndexer ({0}): result after preprocessingfilters: {1}", _definition.Id, results));
-                        }
-
-                        var rowsSelector = ApplyGoTemplateText(search.Rows.Selector, variables);
-                        rowsDom = searchResultDocument.QuerySelectorAll(rowsSelector);
+                        results = ApplyFilters(results, search.Preprocessingfilters, variables);
+                        searchResultDocument = searchResultParser.ParseDocument(results);
+                        _logger.Trace(string.Format("CardigannIndexer ({0}): result after preprocessingfilters: {1}", _definition.Id, results));
                     }
 
-                    var rows = new List<IElement>();
-                    foreach (var rowDom in rowsDom)
+                    var rowsSelector = ApplyGoTemplateText(search.Rows.Selector, variables);
+                    rowsDom = searchResultDocument.QuerySelectorAll(rowsSelector);
+                }
+                else
+                {
+                    var searchResultParser = new HtmlParser();
+                    var searchResultDocument = searchResultParser.ParseDocument(results);
+
+                    if (search.Preprocessingfilters != null)
                     {
-                        rows.Add(rowDom);
+                        results = ApplyFilters(results, search.Preprocessingfilters, variables);
+                        searchResultDocument = searchResultParser.ParseDocument(results);
+                        _logger.Trace(string.Format("CardigannIndexer ({0}): result after preprocessingfilters: {1}", _definition.Id, results));
                     }
 
-                    // merge following rows for After selector
-                    var after = search.Rows.After;
-                    if (after > 0)
+                    var rowsSelector = ApplyGoTemplateText(search.Rows.Selector, variables);
+                    rowsDom = searchResultDocument.QuerySelectorAll(rowsSelector);
+                }
+
+                var rows = new List<IElement>();
+                foreach (var rowDom in rowsDom)
+                {
+                    rows.Add(rowDom);
+                }
+
+                // merge following rows for After selector
+                var after = search.Rows.After;
+                if (after > 0)
+                {
+                    for (var i = 0; i < rows.Count; i += 1)
                     {
-                        for (var i = 0; i < rows.Count; i += 1)
+                        var currentRow = rows[i];
+                        for (var j = 0; j < after; j += 1)
                         {
-                            var currentRow = rows[i];
-                            for (var j = 0; j < after; j += 1)
+                            var mergeRowIndex = i + j + 1;
+                            var mergeRow = rows[mergeRowIndex];
+                            var mergeNodes = new List<INode>();
+                            foreach (var node in mergeRow.ChildNodes)
                             {
-                                var mergeRowIndex = i + j + 1;
-                                var mergeRow = rows[mergeRowIndex];
-                                var mergeNodes = new List<INode>();
-                                foreach (var node in mergeRow.ChildNodes)
-                                {
-                                    mergeNodes.Add(node);
-                                }
-
-                                currentRow.Append(mergeNodes.ToArray());
+                                mergeNodes.Add(node);
                             }
 
-                            rows.RemoveRange(i + 1, after);
+                            currentRow.Append(mergeNodes.ToArray());
                         }
+
+                        rows.RemoveRange(i + 1, after);
                     }
+                }
 
-                    foreach (var row in rows)
+                foreach (var row in rows)
+                {
+                    try
                     {
-                        try
-                        {
-                            var release = new TorrentInfo();
+                        var release = new TorrentInfo();
 
-                            // Parse fields
-                            foreach (var field in search.Fields)
+                        // Parse fields
+                        foreach (var field in search.Fields)
+                        {
+                            var fieldParts = field.Key.Split('|');
+                            var fieldName = fieldParts[0];
+                            var fieldModifiers = new List<string>();
+                            for (var i = 1; i < fieldParts.Length; i++)
                             {
-                                var fieldParts = field.Key.Split('|');
-                                var fieldName = fieldParts[0];
-                                var fieldModifiers = new List<string>();
-                                for (var i = 1; i < fieldParts.Length; i++)
+                                fieldModifiers.Add(fieldParts[i]);
+                            }
+
+                            string value = null;
+                            var variablesKey = ".Result." + fieldName;
+                            var isOptional = OptionalFields.Contains(field.Key) || fieldModifiers.Contains("optional") || field.Value.Optional;
+                            try
+                            {
+                                value = HandleSelector(field.Value, row, variables, !isOptional);
+
+                                if (isOptional && string.IsNullOrWhiteSpace(value))
                                 {
-                                    fieldModifiers.Add(fieldParts[i]);
+                                    variables[variablesKey] = null;
+                                    continue;
                                 }
 
-                                string value = null;
-                                var variablesKey = ".Result." + fieldName;
-                                var isOptional = OptionalFields.Contains(field.Key) || fieldModifiers.Contains("optional") || field.Value.Optional;
+                                variables[variablesKey] = ParseFields(value, fieldName, release, fieldModifiers, searchUrlUri);
+                            }
+                            catch (Exception ex)
+                            {
+                                if (!variables.ContainsKey(variablesKey))
+                                {
+                                    variables[variablesKey] = null;
+                                }
+
+                                if (OptionalFields.Contains(field.Key) || fieldModifiers.Contains("optional") || field.Value.Optional)
+                                {
+                                    variables[variablesKey] = null;
+                                    continue;
+                                }
+
+                                if (indexerLogging)
+                                {
+                                    _logger.Trace("Error while parsing field={0}, selector={1}, value={2}: {3}", field.Key, field.Value.Selector, value == null ? "<null>" : value, ex.Message);
+                                }
+                            }
+                        }
+
+                        var filters = search.Rows.Filters;
+                        var skipRelease = ParseRowFilters(filters, release, variables, row.ToHtmlPretty());
+
+                        if (skipRelease)
+                        {
+                            continue;
+                        }
+
+                        // if DateHeaders is set go through the previous rows and look for the header selector
+                        var dateHeaders = _definition.Search.Rows.Dateheaders;
+                        if (release.PublishDate == DateTime.MinValue && dateHeaders != null)
+                        {
+                            var prevRow = row.PreviousElementSibling;
+                            string value = null;
+                            if (prevRow == null)
+                            {
+                                // continue with parent
+                                var parent = row.ParentElement;
+                                if (parent != null)
+                                {
+                                    prevRow = parent.PreviousElementSibling;
+                                }
+                            }
+
+                            while (prevRow != null)
+                            {
+                                var curRow = prevRow;
+                                _logger.Debug(prevRow.OuterHtml);
                                 try
                                 {
-                                    value = HandleSelector(field.Value, row, variables, !isOptional);
-
-                                    if (isOptional && string.IsNullOrWhiteSpace(value))
-                                    {
-                                        variables[variablesKey] = null;
-                                        continue;
-                                    }
-
-                                    variables[variablesKey] = ParseFields(value, fieldName, release, fieldModifiers, searchUrlUri);
+                                    value = HandleSelector(dateHeaders, curRow);
+                                    break;
                                 }
-                                catch (Exception ex)
+                                catch (Exception)
                                 {
-                                    if (!variables.ContainsKey(variablesKey))
-                                    {
-                                        variables[variablesKey] = null;
-                                    }
-
-                                    if (OptionalFields.Contains(field.Key) || fieldModifiers.Contains("optional") || field.Value.Optional)
-                                    {
-                                        variables[variablesKey] = null;
-                                        continue;
-                                    }
-
-                                    if (indexerLogging)
-                                    {
-                                        _logger.Trace("Error while parsing field={0}, selector={1}, value={2}: {3}", field.Key, field.Value.Selector, value == null ? "<null>" : value, ex.Message);
-                                    }
+                                    // do nothing
                                 }
-                            }
 
-                            var filters = search.Rows.Filters;
-                            var skipRelease = ParseRowFilters(filters, release, variables, row.ToHtmlPretty());
-
-                            if (skipRelease)
-                            {
-                                continue;
-                            }
-
-                            // if DateHeaders is set go through the previous rows and look for the header selector
-                            var dateHeaders = _definition.Search.Rows.Dateheaders;
-                            if (release.PublishDate == DateTime.MinValue && dateHeaders != null)
-                            {
-                                var prevRow = row.PreviousElementSibling;
-                                string value = null;
+                                prevRow = curRow.PreviousElementSibling;
                                 if (prevRow == null)
                                 {
                                     // continue with parent
-                                    var parent = row.ParentElement;
+                                    var parent = curRow.ParentElement;
                                     if (parent != null)
                                     {
                                         prevRow = parent.PreviousElementSibling;
                                     }
                                 }
-
-                                while (prevRow != null)
-                                {
-                                    var curRow = prevRow;
-                                    _logger.Debug(prevRow.OuterHtml);
-                                    try
-                                    {
-                                        value = HandleSelector(dateHeaders, curRow);
-                                        break;
-                                    }
-                                    catch (Exception)
-                                    {
-                                        // do nothing
-                                    }
-
-                                    prevRow = curRow.PreviousElementSibling;
-                                    if (prevRow == null)
-                                    {
-                                        // continue with parent
-                                        var parent = curRow.ParentElement;
-                                        if (parent != null)
-                                        {
-                                            prevRow = parent.PreviousElementSibling;
-                                        }
-                                    }
-                                }
-
-                                if (value == null && dateHeaders.Optional == false)
-                                {
-                                    throw new CardigannException(string.Format("No date header row found for {0}", release.ToString()));
-                                }
-
-                                if (value != null)
-                                {
-                                    release.PublishDate = DateTimeUtil.FromUnknown(value);
-                                }
                             }
 
-                            releases.Add(release);
-                        }
-                        catch (Exception ex)
-                        {
-                            _logger.Error(ex, "CardigannIndexer ({0}): Error while parsing row '{1}':\n\n{2}", _definition.Id, row.ToHtmlPretty());
+                            if (value == null && dateHeaders.Optional == false)
+                            {
+                                throw new CardigannException(string.Format("No date header row found for {0}", release.ToString()));
+                            }
+
+                            if (value != null)
+                            {
+                                release.PublishDate = DateTimeUtil.FromUnknown(value);
+                            }
                         }
+
+                        releases.Add(release);
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.Error(ex, "CardigannIndexer ({0}): Error while parsing row '{1}':\n\n{2}", _definition.Id, row.ToHtmlPretty());
                     }
-                }
-                catch (Exception)
-                {
-                    // OnParseError(results, ex);
-                    throw;
                 }
             }
 
@@ -425,11 +417,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     break;
                 case "comments":
                     var commentsUrl = ResolvePath(value, searchUrlUri);
-                    if (release.CommentUrl == null)
-                    {
-                        release.CommentUrl = commentsUrl.AbsoluteUri;
-                    }
-
+                    release.CommentUrl ??= commentsUrl.AbsoluteUri;
                     value = commentsUrl.ToString();
                     break;
                 case "title":
@@ -522,7 +510,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     break;
                 case "date":
                     release.PublishDate = DateTimeUtil.FromUnknown(value);
-                    value = release.PublishDate.ToString(DateTimeUtil.Rfc1123ZPattern);
+                    value = release.PublishDate.ToString(DateTimeUtil.Rfc1123ZPattern, CultureInfo.InvariantCulture);
                     break;
                 case "files":
                     release.Files = ParseUtil.CoerceInt(value);
@@ -554,38 +542,23 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     value = release.ImdbId.ToString();
                     break;
                 case "tmdbid":
-                    var tmdbIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
-                    var tmdbIDMatch = tmdbIDRegEx.Match(value);
-                    var tmdbID = tmdbIDMatch.Groups[1].Value;
-                    release.TmdbId = (int)ParseUtil.CoerceLong(tmdbID);
+                    release.TmdbId = (int)ParseUtil.GetLongFromString(value);
                     value = release.TmdbId.ToString();
                     break;
                 case "rageid":
-                    var rageIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
-                    var rageIDMatch = rageIDRegEx.Match(value);
-                    var rageID = rageIDMatch.Groups[1].Value;
-                    release.TvRageId = (int)ParseUtil.CoerceLong(rageID);
+                    release.TvRageId = (int)ParseUtil.GetLongFromString(value);
                     value = release.TvRageId.ToString();
                     break;
                 case "traktid":
-                    var traktIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
-                    var traktIDMatch = traktIDRegEx.Match(value);
-                    var traktID = traktIDMatch.Groups[1].Value;
-                    release.TraktId = (int)ParseUtil.CoerceLong(traktID);
+                    release.TraktId = (int)ParseUtil.GetLongFromString(value);
                     value = release.TraktId.ToString();
                     break;
                 case "tvdbid":
-                    var tvdbIdRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
-                    var tvdbIdMatch = tvdbIdRegEx.Match(value);
-                    var tvdbId = tvdbIdMatch.Groups[1].Value;
-                    release.TvdbId = (int)ParseUtil.CoerceLong(tvdbId);
+                    release.TvdbId = (int)ParseUtil.GetLongFromString(value);
                     value = release.TvdbId.ToString();
                     break;
                 case "doubanid":
-                    var doubanIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
-                    var doubanIDMatch = doubanIDRegEx.Match(value);
-                    var doubanID = doubanIDMatch.Groups[1].Value;
-                    release.DoubanId = (int)ParseUtil.CoerceLong(doubanID);
+                    release.DoubanId = (int)ParseUtil.GetLongFromString(value);
                     value = release.DoubanId.ToString();
                     break;
                 case "poster":
@@ -598,8 +571,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     value = release.PosterUrl;
                     break;
                 case "genre":
+                    release.Genres ??= new List<string>();
                     char[] delimiters = { ',', ' ', '/', ')', '(', '.', ';', '[', ']', '"', '|', ':' };
-                    release.Genres = release.Genres.Union(value.Split(delimiters, System.StringSplitOptions.RemoveEmptyEntries)).ToList();
+                    release.Genres = release.Genres
+                        .Union(value.Split(delimiters, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
+                        .Select(x => x.Replace("_", " "))
+                        .ToList();
                     value = string.Join(", ", release.Genres);
                     break;
                 case "year":
@@ -645,29 +622,14 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     switch (filter.Name)
                     {
                         case "andmatch":
-                            var characterLimit = -1;
-                            if (filter.Args != null)
-                            {
-                                characterLimit = int.Parse(filter.Args);
-                            }
-
-                            var queryKeywords = variables[".Keywords"] as string;
-
+                            // See IndexerBase.FilterReleasesByQuery
                             break;
                         case "strdump":
                             // for debugging
-                            _logger.Debug(string.Format("CardigannIndexer ({0}): row strdump: {1}", _definition.Id, row.ToString()));
-                            break;
-                        case "validate":
-                            char[] delimiters = { ',', ' ', '/', ')', '(', '.', ';', '[', ']', '"', '|', ':' };
-                            var args = (string)filter.Args;
-                            var argsList = args.ToLower().Split(delimiters, StringSplitOptions.RemoveEmptyEntries);
-                            var validList = argsList.ToList();
-                            var validIntersect = validList.Intersect(row.ToString().ToLower().Split(delimiters, StringSplitOptions.RemoveEmptyEntries)).ToList();
-                            row = string.Join(", ", validIntersect);
+                            _logger.Debug($"CardigannIndexer ({_definition.Id}): row strdump: {row}");
                             break;
                         default:
-                            _logger.Error(string.Format("CardigannIndexer ({0}): Unsupported rows filter: {1}", _definition.Id, filter.Name));
+                            _logger.Error($"CardigannIndexer ({_definition.Id}): Unsupported rows filter: {filter.Name}");
                             break;
                     }
                 }

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs

@@ -4,7 +4,6 @@ using System.Collections.Specialized;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
-using System.Text;
 using System.Threading.Tasks;
 using AngleSharp.Html.Dom;
 using AngleSharp.Html.Parser;
@@ -29,11 +28,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
         protected IHtmlDocument landingResultDocument;
         protected override string SiteLink => ResolveSiteLink();
 
+        private readonly TimeSpan _rateLimit;
+
         public CardigannRequestGenerator(IConfigService configService,
                                          CardigannDefinition definition,
-                                         Logger logger)
+                                         Logger logger,
+                                         TimeSpan rateLimit)
         : base(configService, definition, logger)
         {
+            _rateLimit = rateLimit;
         }
 
         public Func<IDictionary<string, string>> GetCookies { get; set; }
@@ -56,7 +59,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
             variables[".Query.TraktID"] = searchCriteria.TraktId?.ToString() ?? null;
             variables[".Query.DoubanID"] = searchCriteria.DoubanId?.ToString() ?? null;
 
-            pageableRequests.Add(GetRequest(variables));
+            pageableRequests.Add(GetRequest(variables, searchCriteria));
 
             return pageableRequests;
         }
@@ -76,7 +79,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
             variables[".Query.Year"] = searchCriteria.Year?.ToString() ?? null;
             variables[".Query.Track"] = searchCriteria.Track;
 
-            pageableRequests.Add(GetRequest(variables));
+            pageableRequests.Add(GetRequest(variables, searchCriteria));
 
             return pageableRequests;
         }
@@ -104,7 +107,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
             variables[".Query.DoubanID"] = searchCriteria.DoubanId?.ToString() ?? null;
             variables[".Query.Episode"] = searchCriteria.EpisodeSearchString;
 
-            pageableRequests.Add(GetRequest(variables));
+            pageableRequests.Add(GetRequest(variables, searchCriteria));
 
             return pageableRequests;
         }
@@ -123,7 +126,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
             variables[".Query.Publisher"] = searchCriteria.Publisher;
             variables[".Query.Year"] = searchCriteria.Year?.ToString() ?? null;
 
-            pageableRequests.Add(GetRequest(variables));
+            pageableRequests.Add(GetRequest(variables, searchCriteria));
 
             return pageableRequests;
         }
@@ -136,7 +139,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
 
             var variables = GetQueryVariableDefaults(searchCriteria);
 
-            pageableRequests.Add(GetRequest(variables));
+            pageableRequests.Add(GetRequest(variables, searchCriteria));
 
             return pageableRequests;
         }
@@ -190,6 +193,9 @@ namespace NzbDrone.Core.Indexers.Cardigann
         {
             var login = _definition.Login;
 
+            var variables = GetBaseTemplateVariables();
+            var headers = ParseCustomHeaders(_definition.Login?.Headers ?? _definition.Search?.Headers, variables);
+
             if (login.Method == "post")
             {
                 var pairs = new Dictionary<string, string>();
@@ -218,9 +224,20 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     requestBuilder.AddFormParameter(pair.Key, pair.Value);
                 }
 
-                requestBuilder.Headers.Add("Referer", SiteLink);
+                Cookies = null;
+                if (login.Cookies != null)
+                {
+                    Cookies = CookieUtil.CookieHeaderToDictionary(string.Join("; ", login.Cookies));
+                }
 
-                var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                var request = requestBuilder
+                    .SetCookies(Cookies ?? new Dictionary<string, string>())
+                    .SetHeaders(headers ?? new Dictionary<string, string>())
+                    .SetHeader("Referer", SiteLink)
+                    .WithRateLimit(_rateLimit.TotalSeconds)
+                    .Build();
+
+                var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
                 Cookies = response.GetCookies();
 
@@ -235,13 +252,9 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 var queryCollection = new NameValueCollection();
                 var pairs = new Dictionary<string, string>();
 
-                var formSelector = login.Form;
-                if (formSelector == null)
-                {
-                    formSelector = "form";
-                }
+                var formSelector = login.Form ?? "form";
 
-                // landingResultDocument might not be initiated if the login is caused by a relogin during a query
+                // landingResultDocument might not be initiated if the login is caused by a re-login during a query
                 if (landingResultDocument == null)
                 {
                     await GetConfigurationForSetup(true);
@@ -273,11 +286,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         continue;
                     }
 
-                    var value = input.GetAttribute("value");
-                    if (value == null)
-                    {
-                        value = "";
-                    }
+                    var value = input.GetAttribute("value") ?? "";
 
                     pairs[name] = value;
                 }
@@ -356,11 +365,14 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         Encoding = _encoding
                     };
 
-                    requestBuilder.SetCookies(Cookies);
+                    var request = requestBuilder
+                        .SetCookies(Cookies)
+                        .SetHeaders(headers ?? new Dictionary<string, string>())
+                        .SetHeader("Referer", loginUrl)
+                        .WithRateLimit(_rateLimit.TotalSeconds)
+                        .Build();
 
-                    requestBuilder.Headers.Add("Referer", loginUrl);
-
-                    var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                    var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
                     var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.Content);
                     var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString();
@@ -398,7 +410,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 var enctype = form.GetAttribute("enctype");
                 if (enctype == "multipart/form-data")
                 {
-                    var headers = new Dictionary<string, string>();
                     var boundary = "---------------------------" + DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1)).TotalSeconds.ToString().Replace(".", "");
                     var bodyParts = new List<string>();
 
@@ -424,21 +435,18 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         Encoding = _encoding
                     };
 
-                    requestBuilder.Headers.Add("Referer", SiteLink);
-
-                    requestBuilder.SetCookies(Cookies);
-
                     foreach (var pair in pairs)
                     {
                         requestBuilder.AddFormParameter(pair.Key, pair.Value);
                     }
 
-                    foreach (var header in headers)
-                    {
-                        requestBuilder.SetHeader(header.Key, header.Value);
-                    }
+                    var request = requestBuilder
+                        .SetCookies(Cookies)
+                        .SetHeaders(headers ?? new Dictionary<string, string>())
+                        .SetHeader("Referer", SiteLink)
+                        .WithRateLimit(_rateLimit.TotalSeconds)
+                        .Build();
 
-                    var request = requestBuilder.Build();
                     request.SetContent(body);
 
                     loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
@@ -454,15 +462,19 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         Encoding = _encoding
                     };
 
-                    requestBuilder.SetCookies(Cookies);
-                    requestBuilder.Headers.Add("Referer", loginUrl);
-
                     foreach (var pair in pairs)
                     {
                         requestBuilder.AddFormParameter(pair.Key, pair.Value);
                     }
 
-                    loginResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                    var request = requestBuilder
+                        .SetCookies(Cookies)
+                        .SetHeaders(headers ?? new Dictionary<string, string>())
+                        .SetHeader("Referer", loginUrl)
+                        .WithRateLimit(_rateLimit.TotalSeconds)
+                        .Build();
+
+                    loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
                 }
 
                 Cookies = loginResult.GetCookies();
@@ -496,9 +508,13 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     Encoding = _encoding
                 };
 
-                requestBuilder.Headers.Add("Referer", SiteLink);
+                var request = requestBuilder
+                    .SetHeaders(headers ?? new Dictionary<string, string>())
+                    .SetHeader("Referer", SiteLink)
+                    .WithRateLimit(_rateLimit.TotalSeconds)
+                    .Build();
 
-                var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
                 Cookies = response.GetCookies();
 
@@ -521,9 +537,13 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     Encoding = _encoding
                 };
 
-                requestBuilder.Headers.Add("Referer", SiteLink);
+                var request = requestBuilder
+                    .SetHeaders(headers ?? new Dictionary<string, string>())
+                    .SetHeader("Referer", SiteLink)
+                    .WithRateLimit(_rateLimit.TotalSeconds)
+                    .Build();
 
-                var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
                 Cookies = response.GetCookies();
 
@@ -533,7 +553,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
             }
             else
             {
-                throw new NotImplementedException("Login method " + login.Method + " not implemented");
+                throw new NotImplementedException($"Login method {login.Method} not implemented");
             }
         }
 
@@ -578,15 +598,11 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 return null;
             }
 
+            var variables = GetBaseTemplateVariables();
+            var headers = ParseCustomHeaders(_definition.Login?.Headers ?? _definition.Search?.Headers, variables);
+
             var loginUrl = ResolvePath(login.Path);
 
-            Cookies = null;
-
-            if (login.Cookies != null)
-            {
-                Cookies = CookieUtil.CookieHeaderToDictionary(string.Join("; ", login.Cookies));
-            }
-
             var requestBuilder = new HttpRequestBuilder(loginUrl.AbsoluteUri)
             {
                 LogResponseContent = true,
@@ -594,14 +610,18 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 Encoding = _encoding
             };
 
-            requestBuilder.Headers.Add("Referer", SiteLink);
-
-            if (Cookies != null)
+            Cookies = null;
+            if (login.Cookies != null)
             {
-                requestBuilder.SetCookies(Cookies);
+                Cookies = CookieUtil.CookieHeaderToDictionary(string.Join("; ", login.Cookies));
             }
 
-            var request = requestBuilder.Build();
+            var request = requestBuilder
+                .SetCookies(Cookies ?? new Dictionary<string, string>())
+                .SetHeaders(headers ?? new Dictionary<string, string>())
+                .SetHeader("Referer", SiteLink)
+                .WithRateLimit(_rateLimit.TotalSeconds)
+                .Build();
 
             landingResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
@@ -634,6 +654,9 @@ namespace NzbDrone.Core.Indexers.Cardigann
         {
             var captcha = login.Captcha;
 
+            var variables = GetBaseTemplateVariables();
+            var headers = ParseCustomHeaders(_definition.Login?.Headers ?? _definition.Search?.Headers, variables);
+
             if (captcha.Type == "image")
             {
                 var captchaElement = landingResultDocument.QuerySelector(captcha.Selector);
@@ -644,8 +667,10 @@ namespace NzbDrone.Core.Indexers.Cardigann
 
                     var request = new HttpRequestBuilder(captchaUrl.ToString())
                         .SetCookies(landingResult.GetCookies())
+                        .SetHeaders(headers ?? new Dictionary<string, string>())
                         .SetHeader("Referer", loginUrl.AbsoluteUri)
                         .SetEncoding(_encoding)
+                        .WithRateLimit(_rateLimit.TotalSeconds)
                         .Build();
 
                     var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
@@ -656,10 +681,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         ImageData = response.ResponseData
                     };
                 }
-                else
-                {
-                    _logger.Debug("CardigannIndexer ({0}): No captcha image found", _definition.Id);
-                }
+
+                _logger.Debug("CardigannIndexer ({0}): No captcha image found", _definition.Id);
             }
             else
             {
@@ -689,8 +712,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
         {
             var requestLinkStr = ResolvePath(ApplyGoTemplateText(request.Path, variables)).ToString();
 
-            _logger.Debug("CardigannIndexer ({0}): handleRequest() requestLinkStr= {1}", _definition.Id, requestLinkStr);
-
             Dictionary<string, string> pairs = null;
             var queryCollection = new NameValueCollection();
 
@@ -724,25 +745,36 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 requestLinkStr += queryCollection.GetQueryString(_encoding, separator: request.Queryseparator);
             }
 
-            var httpRequest = new HttpRequestBuilder(requestLinkStr)
-                .SetCookies(Cookies ?? new Dictionary<string, string>())
-                .SetEncoding(_encoding)
-                .SetHeader("Referer", referer);
-
-            httpRequest.Method = method;
+            var httpRequestBuilder = new HttpRequestBuilder(requestLinkStr)
+            {
+                Method = method,
+                Encoding = _encoding
+            };
 
             // Add form data for POST requests
             if (method == HttpMethod.Post)
             {
                 foreach (var param in pairs)
                 {
-                    httpRequest.AddFormParameter(param.Key, param.Value);
+                    httpRequestBuilder.AddFormParameter(param.Key, param.Value);
                 }
             }
 
-            var response = await HttpClient.ExecuteProxiedAsync(httpRequest.Build(), Definition);
+            var headers = ParseCustomHeaders(_definition.Download?.Headers ?? _definition.Search?.Headers, variables);
+
+            var httpRequest = httpRequestBuilder
+                .SetCookies(Cookies ?? new Dictionary<string, string>())
+                .SetHeaders(headers ?? new Dictionary<string, string>())
+                .SetHeader("Referer", referer)
+                .WithRateLimit(_rateLimit.TotalSeconds)
+                .Build();
+
+            _logger.Debug("CardigannIndexer ({0}): handleRequest() httpRequest={1}", _definition.Id, httpRequest);
+
+            var response = await HttpClient.ExecuteProxiedAsync(httpRequest, Definition);
 
             _logger.Debug("CardigannIndexer ({0}): handleRequest() remote server returned {1}", _definition.Id, response.StatusCode);
+
             return response;
         }
 
@@ -750,11 +782,10 @@ namespace NzbDrone.Core.Indexers.Cardigann
         {
             Cookies = GetCookies();
             var method = HttpMethod.Get;
-            var headers = new Dictionary<string, string>();
 
             var variables = GetBaseTemplateVariables();
             AddTemplateVariablesFromUri(variables, link, ".DownloadUri");
-            headers = ParseCustomHeaders(_definition.Search?.Headers, variables);
+            var headers = ParseCustomHeaders(_definition.Download?.Headers ?? _definition.Search?.Headers, variables);
 
             if (_definition.Download != null)
             {
@@ -766,6 +797,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     .SetCookies(Cookies ?? new Dictionary<string, string>())
                     .SetHeaders(headers ?? new Dictionary<string, string>())
                     .SetEncoding(_encoding)
+                    .WithRateLimit(_rateLimit.TotalSeconds)
                     .Build();
 
                 request.AllowAutoRedirect = true;
@@ -791,7 +823,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 {
                     try
                     {
-                        if (!download.Infohash.UseBeforeResponse || download.Before == null || response == null)
+                        if (!download.Infohash.Usebeforeresponse || download.Before == null || response == null)
                         {
                             response = await HttpClient.ExecuteProxiedAsync(request, Definition);
                         }
@@ -799,13 +831,13 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         var hash = MatchSelector(response, download.Infohash.Hash, variables);
                         if (hash == null)
                         {
-                            throw new CardigannException($"InfoHash selectors didn't match");
+                            throw new CardigannException("InfoHash selectors didn't match hash.");
                         }
 
                         var title = MatchSelector(response, download.Infohash.Title, variables);
                         if (title == null)
                         {
-                            throw new CardigannException($"InfoHash selectors didn't match");
+                            throw new CardigannException("InfoHash selectors didn't match title.");
                         }
 
                         var magnet = MagnetLinkBuilder.BuildPublicMagnetLink(hash, title);
@@ -837,7 +869,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
 
                         try
                         {
-                            if (!selector.UseBeforeResponse || download.Before == null || response == null)
+                            if (!selector.Usebeforeresponse || download.Before == null || response == null)
                             {
                                 response = await HttpClient.ExecuteProxiedAsync(request, Definition);
                             }
@@ -856,6 +888,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                                     .SetCookies(Cookies ?? new Dictionary<string, string>())
                                     .SetHeaders(headers ?? new Dictionary<string, string>())
                                     .SetEncoding(_encoding)
+                                    .WithRateLimit(_rateLimit.TotalSeconds)
                                     .Build();
 
                                 response = await HttpClient.ExecuteProxiedAsync(testLinkRequest, Definition);
@@ -875,6 +908,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                                 .SetCookies(Cookies ?? new Dictionary<string, string>())
                                 .SetHeaders(headers ?? new Dictionary<string, string>())
                                 .SetEncoding(_encoding)
+                                .WithRateLimit(_rateLimit.TotalSeconds)
                                 .Build();
 
                             selectorDownloadRequest.Method = method;
@@ -895,6 +929,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 .SetCookies(Cookies ?? new Dictionary<string, string>())
                 .SetHeaders(headers ?? new Dictionary<string, string>())
                 .SetEncoding(_encoding)
+                .WithRateLimit(_rateLimit.TotalSeconds)
                 .Build();
 
             downloadRequest.Method = method;
@@ -907,8 +942,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
             var selectorText = ApplyGoTemplateText(selector.Selector, variables);
             var parser = new HtmlParser();
 
-            var results = response.Content;
-            var resultDocument = parser.ParseDocument(results);
+            var resultDocument = parser.ParseDocument(response.Content);
 
             var element = resultDocument.QuerySelector(selectorText);
             if (element == null)
@@ -981,8 +1015,17 @@ namespace NzbDrone.Core.Indexers.Cardigann
             return false;
         }
 
-        private IEnumerable<IndexerRequest> GetRequest(Dictionary<string, object> variables)
+        private IEnumerable<IndexerRequest> GetRequest(Dictionary<string, object> variables, SearchCriteriaBase searchCriteria)
         {
+            var limit = searchCriteria.Limit ?? 100;
+            var offset = searchCriteria.Offset ?? 0;
+
+            if (offset > 0 && limit > 0 && offset / limit > 0)
+            {
+                // Pagination doesn't work yet, this is to prevent fetching the first page multiple times.
+                yield break;
+            }
+
             var search = _definition.Search;
 
             var mappedCategories = _categories.MapTorznabCapsToTrackers((int[])variables[".Query.Categories"]);
@@ -1096,16 +1139,18 @@ namespace NzbDrone.Core.Indexers.Cardigann
 
                 _logger.Info($"Adding request: {searchUrl}");
 
-                var requestbuilder = new HttpRequestBuilder(searchUrl);
-
-                requestbuilder.Method = method;
+                var requestBuilder = new HttpRequestBuilder(searchUrl)
+                {
+                    Method = method,
+                    Encoding = _encoding
+                };
 
                 // Add FormData for searchs that POST
                 if (method == HttpMethod.Post)
                 {
                     foreach (var param in queryCollection)
                     {
-                        requestbuilder.AddFormParameter(param.Key, param.Value);
+                        requestBuilder.AddFormParameter(param.Key, param.Value);
                     }
                 }
 
@@ -1113,14 +1158,22 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 if (search.Headers != null)
                 {
                     var headers = ParseCustomHeaders(search.Headers, variables);
-                    requestbuilder.SetHeaders(headers ?? new Dictionary<string, string>());
+                    requestBuilder.SetHeaders(headers ?? new Dictionary<string, string>());
                 }
 
-                var request = new CardigannRequest(requestbuilder.SetEncoding(_encoding).Build(), variables, searchPath);
+                var request = requestBuilder
+                    .WithRateLimit(_rateLimit.TotalSeconds)
+                    .Build();
 
-                request.HttpRequest.AllowAutoRedirect = searchPath.Followredirect;
+                var cardigannRequest = new CardigannRequest(request, variables, searchPath)
+                    {
+                        HttpRequest =
+                        {
+                            AllowAutoRedirect = searchPath.Followredirect
+                        }
+                    };
 
-                yield return request;
+                yield return cardigannRequest;
             }
         }
     }

src/NzbDrone.Core/Indexers/Definitions/FunFile.cs

@@ -288,11 +288,11 @@ public class FunFileParser : IParseIndexerResponse
                 throw new Exception("Download links not found. Make sure you can download from the website.");
             }
 
-            var link = _settings.BaseUrl + qDownloadLink.GetAttribute("href");
+            var downloadUrl = _settings.BaseUrl + qDownloadLink.GetAttribute("href");
 
             var qDetailsLink = row.QuerySelector("a[href^=\"details.php?id=\"]");
             var title = qDetailsLink?.GetAttribute("title")?.Trim();
-            var details = _settings.BaseUrl + qDetailsLink?.GetAttribute("href")?.Replace("&hit=1", "");
+            var infoUrl = _settings.BaseUrl + qDetailsLink?.GetAttribute("href")?.Replace("&hit=1", "");
 
             var categoryLink = row.QuerySelector("a[href^=\"browse.php?cat=\"]")?.GetAttribute("href");
             var cat = ParseUtil.GetArgumentFromQueryString(categoryLink, "cat");
@@ -302,9 +302,9 @@ public class FunFileParser : IParseIndexerResponse
 
             var release = new TorrentInfo
             {
-                Guid = link,
-                InfoUrl = link,
-                DownloadUrl = details,
+                Guid = infoUrl,
+                InfoUrl = infoUrl,
+                DownloadUrl = downloadUrl,
                 Title = title,
                 Categories = _categories.MapTrackerCatToNewznab(cat),
                 Size = ParseUtil.GetBytes(row.Children[7].TextContent),

src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleBase.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
 using System.Threading.Tasks;
@@ -106,8 +107,23 @@ public abstract class GazelleBase<TSettings> : TorrentIndexerBase<TSettings>
         return response;
     }
 
+    protected override IDictionary<string, string> GetCookies()
+    {
+        if (Settings is GazelleUserPassOrCookieSettings cookieSettings && !string.IsNullOrWhiteSpace(cookieSettings.Cookie))
+        {
+            return CookieUtil.CookieHeaderToDictionary(cookieSettings.Cookie);
+        }
+
+        return base.GetCookies();
+    }
+
     protected override bool CheckIfLoginNeeded(HttpResponse response)
     {
+        if (Settings is GazelleUserPassOrCookieSettings cookieSettings && !string.IsNullOrWhiteSpace(cookieSettings.Cookie))
+        {
+            return false;
+        }
+
         var invalidResponses = new[] { "\"bad credentials\"", "\"groupName\":\"wrong-creds\"" };
 
         return response.HasHttpRedirect || (response.Content != null && invalidResponses.Any(response.Content.Contains));

src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleSettings.cs

@@ -4,13 +4,14 @@ using NzbDrone.Core.Validation;
 
 namespace NzbDrone.Core.Indexers.Definitions.Gazelle;
 
-public class GazelleSettingsValidator : UserPassBaseSettingsValidator<GazelleSettings>
+public class GazelleSettingsValidator<T> : UserPassBaseSettingsValidator<T>
+    where T : GazelleSettings
 {
 }
 
 public class GazelleSettings : UserPassTorrentBaseSettings
 {
-    private static readonly GazelleSettingsValidator Validator = new ();
+    private static readonly GazelleSettingsValidator<GazelleSettings> Validator = new ();
 
     public string AuthKey { get; set; }
     public string PassKey { get; set; }

src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleUserPassOrCookieSettings.cs

@@ -0,0 +1,31 @@
+using FluentValidation;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Indexers.Settings;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Indexers.Definitions.Gazelle;
+
+public class GazelleUserPassOrCookieValidator<T> : NoAuthSettingsValidator<T>
+    where T : GazelleUserPassOrCookieSettings
+{
+    public GazelleUserPassOrCookieValidator()
+    {
+        RuleFor(c => c.Username).NotEmpty().When(c => c.Cookie.IsNullOrWhiteSpace());
+        RuleFor(c => c.Password).NotEmpty().When(c => c.Cookie.IsNullOrWhiteSpace());
+        RuleFor(c => c.Cookie).NotEmpty().When(c => c.Username.IsNullOrWhiteSpace() && c.Password.IsNullOrWhiteSpace());
+    }
+}
+
+public class GazelleUserPassOrCookieSettings : GazelleSettings
+{
+    private static readonly GazelleUserPassOrCookieValidator<GazelleUserPassOrCookieSettings> Validator = new ();
+
+    [FieldDefinition(4, Label = "Cookie", HelpText = "Use the Cookie field only if 2FA is enabled for your account, leave it empty otherwise.", Privacy = PrivacyLevel.Password)]
+    public string Cookie { get; set; }
+
+    public override NzbDroneValidationResult Validate()
+    {
+        return new NzbDroneValidationResult(Validator.Validate(this));
+    }
+}

src/NzbDrone.Core/Indexers/Definitions/GreatPosterWall.cs

@@ -15,6 +15,7 @@ using NzbDrone.Core.IndexerSearch.Definitions;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser;
 using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Validation;
 
 namespace NzbDrone.Core.Indexers.Definitions;
 
@@ -118,17 +119,23 @@ public class GreatPosterWallParser : GazelleParser
 
         if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
         {
-            // Remove cookie cache
-            CookiesUpdater(null, null);
+            if (indexerResponse.HttpResponse.HasHttpRedirect)
+            {
+                if (indexerResponse.HttpResponse.RedirectUrl.ContainsIgnoreCase("login.php"))
+                {
+                    // Remove cookie cache
+                    CookiesUpdater(null, null);
+                    throw new IndexerException(indexerResponse, "We are being redirected to the login page. Most likely your session expired or was killed. Recheck your cookie or credentials and try testing the indexer.");
+                }
+
+                throw new IndexerException(indexerResponse, $"Redirected to {indexerResponse.HttpResponse.RedirectUrl} from API request");
+            }
 
             throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request");
         }
 
         if (!indexerResponse.HttpResponse.Headers.ContentType.Contains(HttpAccept.Json.Value))
         {
-            // Remove cookie cache
-            CookiesUpdater(null, null);
-
             throw new IndexerException(indexerResponse, $"Unexpected response header {indexerResponse.HttpResponse.Headers.ContentType} from API request, expected {HttpAccept.Json.Value}");
         }
 
@@ -231,10 +238,17 @@ public class GreatPosterWallParser : GazelleParser
     }
 }
 
-public class GreatPosterWallSettings : GazelleSettings
+public class GreatPosterWallSettings : GazelleUserPassOrCookieSettings
 {
+    private static readonly GazelleUserPassOrCookieValidator<GreatPosterWallSettings> Validator = new ();
+
     [FieldDefinition(6, Label = "Freeleech Only", Type = FieldType.Checkbox, HelpText = "Search freeleech torrents only")]
     public bool FreeleechOnly { get; set; }
+
+    public override NzbDroneValidationResult Validate()
+    {
+        return new NzbDroneValidationResult(Validator.Validate(this));
+    }
 }
 
 public class GreatPosterWallResponse

src/NzbDrone.Core/Indexers/Definitions/Nebulance.cs

@@ -46,12 +46,17 @@ namespace NzbDrone.Core.Indexers.Definitions
             return new NebulanceParser(Settings);
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
         {
-            // Invalidate cookies before downloading to prevent redirect to login page.
-            UpdateCookies(null, null);
+            // Avoid using cookies to prevent redirects to login page
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
+            {
+                AllowAutoRedirect = FollowRedirect
+            };
 
-            return await base.Download(link);
+            var request = requestBuilder.Build();
+
+            return Task.FromResult(request);
         }
 
         private IndexerCapabilities SetCapabilities()

src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs

@@ -112,7 +112,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                 _logger.Error("Download failed");
             }
 
-            ValidateTorrent(downloadBytes);
+            ValidateDownloadData(downloadBytes);
 
             return downloadBytes;
         }

src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net;
 using System.Threading.Tasks;
 using System.Web;
 using NLog;
@@ -9,11 +10,12 @@ using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Exceptions;
+using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser;
 using NzbDrone.Core.Validation;
 
-namespace NzbDrone.Core.Indexers.Rarbg
+namespace NzbDrone.Core.Indexers.Definitions.Rarbg
 {
     public class Rarbg : TorrentIndexerBase<RarbgSettings>
     {
@@ -24,7 +26,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
         public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
         public override IndexerPrivacy Privacy => IndexerPrivacy.Public;
         public override IndexerCapabilities Capabilities => SetCapabilities();
-        public override TimeSpan RateLimit => TimeSpan.FromSeconds(5);
+        public override TimeSpan RateLimit => TimeSpan.FromSeconds(7);
         private readonly IRarbgTokenProvider _tokenProvider;
 
         public Rarbg(IRarbgTokenProvider tokenProvider, IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
@@ -35,7 +37,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
 
         public override IIndexerRequestGenerator GetRequestGenerator()
         {
-            return new RarbgRequestGenerator(_tokenProvider) { Settings = Settings, Categories = Capabilities.Categories };
+            return new RarbgRequestGenerator(_tokenProvider, RateLimit) { Settings = Settings, Categories = Capabilities.Categories };
         }
 
         public override IParseIndexerResponse GetParser()
@@ -43,6 +45,23 @@ namespace NzbDrone.Core.Indexers.Rarbg
             return new RarbgParser(Capabilities, _logger);
         }
 
+        public static void CheckResponseByStatusCode(IndexerResponse response)
+        {
+            var responseCode = (int)response.HttpResponse.StatusCode;
+
+            switch (responseCode)
+            {
+                case (int)HttpStatusCode.TooManyRequests:
+                    throw new TooManyRequestsException(response.HttpRequest, response.HttpResponse, TimeSpan.FromMinutes(2));
+                case 520:
+                    throw new TooManyRequestsException(response.HttpRequest, response.HttpResponse, TimeSpan.FromMinutes(3));
+                case (int)HttpStatusCode.OK:
+                    break;
+                default:
+                    throw new IndexerException(response, "Indexer API call returned an unexpected StatusCode [{0}]", responseCode);
+            }
+        }
+
         private IndexerCapabilities SetCapabilities()
         {
             var caps = new IndexerCapabilities
@@ -61,7 +80,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
                 }
             };
 
-            caps.Categories.AddCategoryMapping(4, NewznabStandardCategory.XXX, "XXX (18+)");
+            // caps.Categories.AddCategoryMapping(4, NewznabStandardCategory.XXX, "XXX (18+)"); // 3x is not supported by API #11848
             caps.Categories.AddCategoryMapping(14, NewznabStandardCategory.MoviesSD, "Movies/XVID");
             caps.Categories.AddCategoryMapping(17, NewznabStandardCategory.MoviesSD, "Movies/x264");
             caps.Categories.AddCategoryMapping(18, NewznabStandardCategory.TVSD, "TV Episodes");
@@ -97,6 +116,8 @@ namespace NzbDrone.Core.Indexers.Rarbg
         {
             var response = await FetchIndexerResponse(request);
 
+            CheckResponseByStatusCode(response);
+
             // try and recover from token errors
             var jsonResponse = new HttpResponse<RarbgResponse>(response.HttpResponse);
 
@@ -106,7 +127,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
                 {
                     _logger.Debug("Invalid or expired token, refreshing token from Rarbg");
                     _tokenProvider.ExpireToken(Settings);
-                    var newToken = _tokenProvider.GetToken(Settings);
+                    var newToken = _tokenProvider.GetToken(Settings, RateLimit);
 
                     var qs = HttpUtility.ParseQueryString(request.HttpRequest.Url.Query);
                     qs.Set("token", newToken);
@@ -151,7 +172,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
                 Settings.Validate().Filter("BaseUrl").ThrowOnError();
 
                 var request = new HttpRequestBuilder(Settings.BaseUrl.Trim('/'))
-                    .Resource($"/pubapi_v2.php?get_token=get_token&app_id={BuildInfo.AppName}")
+                    .Resource($"/pubapi_v2.php?get_token=get_token&app_id=rralworP_{BuildInfo.Version}")
                     .Accept(HttpAccept.Json)
                     .Build();
 

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgParser.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Net;
 using System.Text.RegularExpressions;
 using NLog;
 using NzbDrone.Common.EnvironmentInfo;
@@ -8,7 +7,7 @@ using NzbDrone.Common.Http;
 using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.Parser.Model;
 
-namespace NzbDrone.Core.Indexers.Rarbg
+namespace NzbDrone.Core.Indexers.Definitions.Rarbg
 {
     public class RarbgParser : IParseIndexerResponse
     {
@@ -28,19 +27,8 @@ namespace NzbDrone.Core.Indexers.Rarbg
         public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
         {
             var results = new List<ReleaseInfo>();
-            var responseCode = (int)indexerResponse.HttpResponse.StatusCode;
 
-            switch (responseCode)
-            {
-                case (int)HttpStatusCode.TooManyRequests:
-                    throw new TooManyRequestsException(indexerResponse.HttpRequest, indexerResponse.HttpResponse, TimeSpan.FromMinutes(2));
-                case 520:
-                    throw new TooManyRequestsException(indexerResponse.HttpRequest, indexerResponse.HttpResponse, TimeSpan.FromMinutes(3));
-                case (int)HttpStatusCode.OK:
-                    break;
-                default:
-                    throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", responseCode);
-            }
+            Rarbg.CheckResponseByStatusCode(indexerResponse);
 
             var jsonResponse = new HttpResponse<RarbgResponse>(indexerResponse.HttpResponse);
 
@@ -84,7 +72,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
                     Title = torrent.title,
                     Size = torrent.size,
                     DownloadUrl = torrent.download,
-                    InfoUrl = $"{torrent.info_page}&app_id={BuildInfo.AppName}",
+                    InfoUrl = $"{torrent.info_page}&app_id=rralworP_{BuildInfo.Version}",
                     PublishDate = torrent.pubdate.ToUniversalTime(),
                     Seeders = torrent.seeders,
                     Peers = torrent.leechers + torrent.seeders,

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs

@@ -6,43 +6,59 @@ using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.IndexerSearch.Definitions;
 
-namespace NzbDrone.Core.Indexers.Rarbg
+namespace NzbDrone.Core.Indexers.Definitions.Rarbg
 {
     public class RarbgRequestGenerator : IIndexerRequestGenerator
     {
         private readonly IRarbgTokenProvider _tokenProvider;
+        private readonly TimeSpan _rateLimit;
 
         public RarbgSettings Settings { get; set; }
         public IndexerCapabilitiesCategories Categories { get; set; }
 
-        public RarbgRequestGenerator(IRarbgTokenProvider tokenProvider)
+        public RarbgRequestGenerator(IRarbgTokenProvider tokenProvider, TimeSpan rateLimit)
         {
             _tokenProvider = tokenProvider;
+            _rateLimit = rateLimit;
         }
 
-        private IEnumerable<IndexerRequest> GetRequest(string term, int[] categories, string imdbId = null, int? tmdbId = null, int? tvdbId = null)
+        private IEnumerable<IndexerRequest> GetRequest(bool isRssSearch, string term, int[] categories, string imdbId = null, int? tmdbId = null, int? tvdbId = null)
         {
             var requestBuilder = new HttpRequestBuilder(Settings.BaseUrl.Trim('/'))
                 .Resource("/pubapi_v2.php")
-                .AddQueryParam("mode", "search")
+                .AddQueryParam("limit", "100")
+                .AddQueryParam("token", _tokenProvider.GetToken(Settings, _rateLimit))
+                .AddQueryParam("format", "json_extended")
+                .AddQueryParam("app_id", $"rralworP_{BuildInfo.Version}")
                 .Accept(HttpAccept.Json);
 
-            if (imdbId.IsNotNullOrWhiteSpace())
+            if (isRssSearch)
             {
-                requestBuilder.AddQueryParam("search_imdb", imdbId);
+                requestBuilder
+                    .AddQueryParam("mode", "list")
+                    .WithRateLimit(31);
             }
-            else if (tmdbId.HasValue && tmdbId > 0)
+            else
             {
-                requestBuilder.AddQueryParam("search_themoviedb", tmdbId);
-            }
-            else if (tvdbId.HasValue && tvdbId > 0)
-            {
-                requestBuilder.AddQueryParam("search_tvdb", tvdbId);
-            }
+                requestBuilder.AddQueryParam("mode", "search");
 
-            if (term.IsNotNullOrWhiteSpace())
-            {
-                requestBuilder.AddQueryParam("search_string", $"{term}");
+                if (imdbId.IsNotNullOrWhiteSpace())
+                {
+                    requestBuilder.AddQueryParam("search_imdb", imdbId);
+                }
+                else if (tmdbId.HasValue && tmdbId > 0)
+                {
+                    requestBuilder.AddQueryParam("search_themoviedb", tmdbId);
+                }
+                else if (tvdbId.HasValue && tvdbId > 0)
+                {
+                    requestBuilder.AddQueryParam("search_tvdb", tvdbId);
+                }
+
+                if (term.IsNotNullOrWhiteSpace())
+                {
+                    requestBuilder.AddQueryParam("search_string", $"{term}");
+                }
             }
 
             if (!Settings.RankedOnly)
@@ -51,17 +67,13 @@ namespace NzbDrone.Core.Indexers.Rarbg
             }
 
             var cats = Categories.MapTorznabCapsToTrackers(categories);
-
-            if (cats != null && cats.Count > 0)
+            if (cats == null || !cats.Any())
             {
-                var categoryParam = string.Join(";", cats.Distinct());
-                requestBuilder.AddQueryParam("category", categoryParam);
+                // default to all, without specifying it some categories are missing (e.g. games), see #4146
+                cats = Categories.GetTrackerCategories();
             }
 
-            requestBuilder.AddQueryParam("limit", "100");
-            requestBuilder.AddQueryParam("token", _tokenProvider.GetToken(Settings));
-            requestBuilder.AddQueryParam("format", "json_extended");
-            requestBuilder.AddQueryParam("app_id", BuildInfo.AppName);
+            requestBuilder.AddQueryParam("category", string.Join(";", cats.Distinct()));
 
             yield return new IndexerRequest(requestBuilder.Build());
         }
@@ -69,35 +81,35 @@ namespace NzbDrone.Core.Indexers.Rarbg
         public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
         {
             var pageableRequests = new IndexerPageableRequestChain();
-            pageableRequests.Add(GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories, searchCriteria.FullImdbId, searchCriteria.TmdbId));
+            pageableRequests.Add(GetRequest(searchCriteria.IsRssSearch, searchCriteria.SanitizedSearchTerm, searchCriteria.Categories, searchCriteria.FullImdbId, searchCriteria.TmdbId));
             return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
         {
             var pageableRequests = new IndexerPageableRequestChain();
-            pageableRequests.Add(GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories));
+            pageableRequests.Add(GetRequest(searchCriteria.IsRssSearch, searchCriteria.SanitizedSearchTerm, searchCriteria.Categories));
             return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
         {
             var pageableRequests = new IndexerPageableRequestChain();
-            pageableRequests.Add(GetRequest(searchCriteria.SanitizedTvSearchString, searchCriteria.Categories, searchCriteria.FullImdbId, tvdbId: searchCriteria.TvdbId));
+            pageableRequests.Add(GetRequest(searchCriteria.IsRssSearch, searchCriteria.SanitizedTvSearchString, searchCriteria.Categories, searchCriteria.FullImdbId, tvdbId: searchCriteria.TvdbId));
             return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
         {
             var pageableRequests = new IndexerPageableRequestChain();
-            pageableRequests.Add(GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories));
+            pageableRequests.Add(GetRequest(searchCriteria.IsRssSearch, searchCriteria.SanitizedSearchTerm, searchCriteria.Categories));
             return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
         {
             var pageableRequests = new IndexerPageableRequestChain();
-            pageableRequests.Add(GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories));
+            pageableRequests.Add(GetRequest(searchCriteria.IsRssSearch, searchCriteria.SanitizedSearchTerm, searchCriteria.Categories));
 
             return pageableRequests;
         }

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgResponse.cs

@@ -1,7 +1,7 @@
 using System;
 using System.Collections.Generic;
 
-namespace NzbDrone.Core.Indexers.Rarbg
+namespace NzbDrone.Core.Indexers.Definitions.Rarbg
 {
     public class RarbgResponse
     {

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgSettings.cs

@@ -1,7 +1,7 @@
 using NzbDrone.Core.Annotations;
 using NzbDrone.Core.Indexers.Settings;
 
-namespace NzbDrone.Core.Indexers.Rarbg
+namespace NzbDrone.Core.Indexers.Definitions.Rarbg
 {
     public class RarbgSettings : NoAuthTorrentBaseSettings
     {

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgTokenProvider.cs

@@ -5,11 +5,11 @@ using NzbDrone.Common.Cache;
 using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Common.Http;
 
-namespace NzbDrone.Core.Indexers.Rarbg
+namespace NzbDrone.Core.Indexers.Definitions.Rarbg
 {
     public interface IRarbgTokenProvider
     {
-        string GetToken(RarbgSettings settings);
+        string GetToken(RarbgSettings settings, TimeSpan rateLimit);
         void ExpireToken(RarbgSettings settings);
     }
 
@@ -31,16 +31,18 @@ namespace NzbDrone.Core.Indexers.Rarbg
             _tokenCache.Remove(settings.BaseUrl);
         }
 
-        public string GetToken(RarbgSettings settings)
+        public string GetToken(RarbgSettings settings, TimeSpan rateLimit)
         {
             return _tokenCache.Get(settings.BaseUrl,
                 () =>
                 {
                     var requestBuilder = new HttpRequestBuilder(settings.BaseUrl.Trim('/'))
-                        .WithRateLimit(5.0)
-                        .Resource($"/pubapi_v2.php?get_token=get_token&app_id={BuildInfo.AppName}")
+                        .WithRateLimit(rateLimit.TotalSeconds)
+                        .Resource($"/pubapi_v2.php?get_token=get_token&app_id=rralworP_{BuildInfo.Version}")
                         .Accept(HttpAccept.Json);
 
+                    requestBuilder.LogResponseContent = true;
+
                     var response = _httpClient.Get<JObject>(requestBuilder.Build());
 
                     return response.Resource["token"].ToString();

src/NzbDrone.Core/Indexers/Definitions/Redacted.cs

@@ -50,6 +50,20 @@ namespace NzbDrone.Core.Indexers.Definitions
             return new RedactedParser(Settings, Capabilities.Categories);
         }
 
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
+        {
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
+            {
+                AllowAutoRedirect = FollowRedirect
+            };
+
+            var request = requestBuilder
+                .SetHeader("Authorization", Settings.Apikey)
+                .Build();
+
+            return Task.FromResult(request);
+        }
+
         private IndexerCapabilities SetCapabilities()
         {
             var caps = new IndexerCapabilities
@@ -74,30 +88,6 @@ namespace NzbDrone.Core.Indexers.Definitions
 
             return caps;
         }
-
-        public override async Task<byte[]> Download(Uri link)
-        {
-            var request = new HttpRequestBuilder(link.AbsoluteUri)
-                .SetHeader("Authorization", Settings.Apikey)
-                .Build();
-
-            var downloadBytes = Array.Empty<byte>();
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                downloadBytes = response.ResponseData;
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Download failed");
-            }
-
-            ValidateTorrent(downloadBytes);
-
-            return downloadBytes;
-        }
     }
 
     public class RedactedRequestGenerator : IIndexerRequestGenerator
@@ -188,18 +178,12 @@ namespace NzbDrone.Core.Indexers.Definitions
                 queryCats.ForEach(cat => parameters.Set($"filter_cat[{cat}]", "1"));
             }
 
-            var request = RequestBuilder()
-                .Resource($"/ajax.php?{parameters.GetQueryString()}")
-                .Build();
+            var searchUrl = _settings.BaseUrl.TrimEnd('/') + $"/ajax.php?{parameters.GetQueryString()}";
 
-            yield return new IndexerRequest(request);
-        }
+            var request = new IndexerRequest(searchUrl, HttpAccept.Json);
+            request.HttpRequest.Headers.Set("Authorization", _settings.Apikey);
 
-        private HttpRequestBuilder RequestBuilder()
-        {
-            return new HttpRequestBuilder($"{_settings.BaseUrl.TrimEnd('/')}")
-                .Accept(HttpAccept.Json)
-                .SetHeader("Authorization", _settings.Apikey);
+            yield return request;
         }
     }
 

src/NzbDrone.Core/Indexers/Definitions/SpeedApp.cs

@@ -1,7 +1,10 @@
 using System.Collections.Generic;
+using System.Linq;
 using NLog;
 using NzbDrone.Core.Configuration;
+using NzbDrone.Core.IndexerSearch.Definitions;
 using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Indexers.Definitions
 {
@@ -19,6 +22,13 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
         }
 
+        protected override IList<ReleaseInfo> CleanupReleases(IEnumerable<ReleaseInfo> releases, SearchCriteriaBase searchCriteria)
+        {
+            var cleanReleases = base.CleanupReleases(releases, searchCriteria);
+
+            return FilterReleasesByQuery(cleanReleases, searchCriteria).ToList();
+        }
+
         protected override IndexerCapabilities SetCapabilities()
         {
             var caps = new IndexerCapabilities

src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs

@@ -15,7 +15,6 @@ using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Annotations;
 using NzbDrone.Core.Configuration;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.Indexers.Settings;
 using NzbDrone.Core.IndexerSearch.Definitions;
@@ -104,70 +103,18 @@ namespace NzbDrone.Core.Indexers.Definitions
             request.HttpRequest.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
         {
-            Cookies = GetCookies();
-
-            if (link.Scheme == "magnet")
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
             {
-                ValidateMagnet(link.OriginalString);
+                AllowAutoRedirect = FollowRedirect
+            };
 
-                return Encoding.UTF8.GetBytes(link.OriginalString);
-            }
+            var request = requestBuilder
+                .SetHeader("Authorization", $"Bearer {Settings.ApiKey}")
+                .Build();
 
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-            request.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
-
-            byte[] torrentData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                torrentData = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            ValidateTorrent(torrentData);
-
-            return torrentData;
+            return Task.FromResult(request);
         }
 
         protected virtual IndexerCapabilities SetCapabilities()
@@ -276,7 +223,6 @@ namespace NzbDrone.Core.Indexers.Definitions
             var searchUrl = _settings.BaseUrl + "api/torrent?" + parameters.GetQueryString(duplicateKeysIfMulti: true);
 
             var request = new IndexerRequest(searchUrl, HttpAccept.Json);
-
             request.HttpRequest.Headers.Set("Authorization", $"Bearer {_settings.ApiKey}");
 
             yield return request;

src/NzbDrone.Core/Indexers/Definitions/SubsPlease.cs

@@ -129,7 +129,7 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(searchCriteria.RssSearch
+            pageableRequests.Add(searchCriteria.IsRssSearch
                 ? GetRssRequest()
                 : GetSearchRequests(searchCriteria.SanitizedTvSearchString));
 
@@ -147,7 +147,7 @@ namespace NzbDrone.Core.Indexers.Definitions
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(searchCriteria.RssSearch
+            pageableRequests.Add(searchCriteria.IsRssSearch
                 ? GetRssRequest()
                 : GetSearchRequests(searchCriteria.SanitizedSearchTerm));
 

src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs

@@ -1,26 +1,37 @@
 using NzbDrone.Common.Messaging;
+using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Indexers.Events
 {
     public class IndexerDownloadEvent : IEvent
     {
-        public int IndexerId { get; set; }
+        public ReleaseInfo Release { get; set; }
         public bool Successful { get; set; }
         public string Source { get; set; }
         public string Host { get; set; }
         public string Title { get; set; }
         public bool Redirect { get; set; }
         public string Url { get; set; }
+        public int DownloadClientId { get; set; }
+        public string DownloadClient { get; set; }
+        public string DownloadClientName { get; set; }
+        public string DownloadId { get; set; }
+        public GrabTrigger GrabTrigger { get; set; }
 
-        public IndexerDownloadEvent(int indexerId, bool successful, string source, string host, string title, string url, bool redirect = false)
+        public IndexerDownloadEvent(ReleaseInfo release, bool successful, string source, string host, string title, string url)
         {
-            IndexerId = indexerId;
+            Release = release;
             Successful = successful;
             Source = source;
             Host = host;
             Title = title;
-            Redirect = redirect;
             Url = url;
         }
     }
+
+    public enum GrabTrigger
+    {
+        Api,
+        Manual
+    }
 }

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -6,10 +6,12 @@ using System.Net.Http;
 using System.Text;
 using System.Threading.Tasks;
 using FluentValidation.Results;
+using MonoTorrent;
 using NLog;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Http.CloudFlare;
 using NzbDrone.Core.Indexers.Events;
 using NzbDrone.Core.Indexers.Exceptions;
@@ -58,7 +60,7 @@ namespace NzbDrone.Core.Indexers
                 return Task.FromResult(new IndexerPageableQueryResult());
             }
 
-            return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
+            return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
         }
 
         public override Task<IndexerPageableQueryResult> Fetch(MusicSearchCriteria searchCriteria)
@@ -68,7 +70,7 @@ namespace NzbDrone.Core.Indexers
                 return Task.FromResult(new IndexerPageableQueryResult());
             }
 
-            return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
+            return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
         }
 
         public override Task<IndexerPageableQueryResult> Fetch(TvSearchCriteria searchCriteria)
@@ -78,7 +80,7 @@ namespace NzbDrone.Core.Indexers
                 return Task.FromResult(new IndexerPageableQueryResult());
             }
 
-            return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
+            return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
         }
 
         public override Task<IndexerPageableQueryResult> Fetch(BookSearchCriteria searchCriteria)
@@ -88,7 +90,7 @@ namespace NzbDrone.Core.Indexers
                 return Task.FromResult(new IndexerPageableQueryResult());
             }
 
-            return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
+            return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
         }
 
         public override Task<IndexerPageableQueryResult> Fetch(BasicSearchCriteria searchCriteria)
@@ -98,7 +100,96 @@ namespace NzbDrone.Core.Indexers
                 return Task.FromResult(new IndexerPageableQueryResult());
             }
 
-            return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
+            return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
+        }
+
+        public override async Task<byte[]> Download(Uri link)
+        {
+            Cookies = GetCookies();
+
+            var request = await GetDownloadRequest(link);
+
+            if (request.Url.Scheme == "magnet")
+            {
+                ValidateMagnet(request.Url.FullUri);
+                return Encoding.UTF8.GetBytes(request.Url.FullUri);
+            }
+
+            if (request.RateLimit < RateLimit)
+            {
+                request.RateLimit = RateLimit;
+            }
+
+            byte[] fileData;
+
+            try
+            {
+                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
+                fileData = response.ResponseData;
+
+                _logger.Debug("Downloaded for release finished ({0} bytes from {1})", fileData.Length, link.AbsoluteUri);
+            }
+            catch (HttpException ex)
+            {
+                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
+                {
+                    _logger.Error(ex, "Downloading file for release failed since it no longer exists ({0})", link.AbsoluteUri);
+                    throw new ReleaseUnavailableException("Download failed", ex);
+                }
+
+                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
+                {
+                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
+                }
+                else
+                {
+                    _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
+                }
+
+                throw new ReleaseDownloadException("Download failed", ex);
+            }
+            catch (WebException ex)
+            {
+                _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
+
+                throw new ReleaseDownloadException("Download failed", ex);
+            }
+            catch (Exception)
+            {
+                _indexerStatusService.RecordFailure(Definition.Id);
+                _logger.Error("Download failed");
+                throw;
+            }
+
+            ValidateDownloadData(fileData);
+
+            return fileData;
+        }
+
+        protected virtual Task<HttpRequest> GetDownloadRequest(Uri link)
+        {
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
+            {
+                AllowAutoRedirect = FollowRedirect
+            };
+
+            if (Cookies != null)
+            {
+                requestBuilder.SetCookies(Cookies);
+            }
+
+            var request = requestBuilder.Build();
+
+            return Task.FromResult(request);
+        }
+
+        protected virtual void ValidateDownloadData(byte[] fileData)
+        {
+        }
+
+        protected void ValidateMagnet(string link)
+        {
+            MagnetLink.Parse(link);
         }
 
         protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator)
@@ -142,7 +233,7 @@ namespace NzbDrone.Core.Indexers
             _indexerStatusService.UpdateCookies(Definition.Id, cookies, expiration);
         }
 
-        protected virtual async Task<IndexerPageableQueryResult> FetchReleases(Func<IIndexerRequestGenerator, IndexerPageableRequestChain> pageableRequestChainSelector, bool isRecent = false)
+        protected virtual async Task<IndexerPageableQueryResult> FetchReleases(Func<IIndexerRequestGenerator, IndexerPageableRequestChain> pageableRequestChainSelector, SearchCriteriaBase searchCriteria, bool isRecent = false)
         {
             var releases = new List<ReleaseInfo>();
             var result = new IndexerPageableQueryResult();
@@ -276,7 +367,7 @@ namespace NzbDrone.Core.Indexers
                 _logger.Error(ex, "An error occurred while processing indexer feed. {0}", url);
             }
 
-            result.Releases = CleanupReleases(releases);
+            result.Releases = CleanupReleases(releases, searchCriteria);
 
             return result;
         }
@@ -420,6 +511,11 @@ namespace NzbDrone.Core.Indexers
                 request.RequestTimeout = TimeSpan.FromSeconds(15);
             }
 
+            if (request.RateLimit < RateLimit)
+            {
+                request.RateLimit = RateLimit;
+            }
+
             var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
 
             _eventAggregator.PublishEvent(new IndexerAuthEvent(Definition.Id, !response.HasHttpError, response.ElapsedTime));

src/NzbDrone.Core/Indexers/IndexerBase.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using FluentValidation.Results;
 using NLog;
@@ -104,7 +105,7 @@ namespace NzbDrone.Core.Indexers
 
         public abstract IndexerCapabilities GetCapabilities();
 
-        protected virtual IList<ReleaseInfo> CleanupReleases(IEnumerable<ReleaseInfo> releases)
+        protected virtual IList<ReleaseInfo> CleanupReleases(IEnumerable<ReleaseInfo> releases, SearchCriteriaBase searchCriteria)
         {
             var result = releases.ToList();
 
@@ -146,6 +147,24 @@ namespace NzbDrone.Core.Indexers
             return result.DistinctBy(v => v.Guid).ToList();
         }
 
+        protected IEnumerable<ReleaseInfo> FilterReleasesByQuery(IEnumerable<ReleaseInfo> releases, SearchCriteriaBase searchCriteria)
+        {
+            var commonWords = new[] { "and", "the", "an", "of" };
+
+            if (!searchCriteria.IsRssSearch && !searchCriteria.IsIdSearch)
+            {
+                var splitRegex = new Regex("[^\\w]+");
+
+                // split search term to individual terms for less aggressive filtering, filter common terms
+                var terms = splitRegex.Split(searchCriteria.SearchTerm).Where(t => t.IsNotNullOrWhiteSpace() && t.Length > 1 && !commonWords.ContainsIgnoreCase(t));
+
+                // check in title and description for any term searched for
+                releases = releases.Where(r => terms.Any(t => (r.Title.IsNotNullOrWhiteSpace() && r.Title.ContainsIgnoreCase(t)) || (r.Description.IsNotNullOrWhiteSpace() && r.Description.ContainsIgnoreCase(t)))).ToList();
+            }
+
+            return releases;
+        }
+
         protected virtual TSettings GetDefaultBaseUrl(TSettings settings)
         {
             var defaultLink = IndexerUrls.FirstOrDefault();

src/NzbDrone.Core/Indexers/IndexerLimitService.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using NLog;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Core.History;
 
 namespace NzbDrone.Core.Indexers
@@ -10,6 +9,8 @@ namespace NzbDrone.Core.Indexers
     {
         bool AtDownloadLimit(IndexerDefinition indexer);
         bool AtQueryLimit(IndexerDefinition indexer);
+        int CalculateRetryAfterDownloadLimit(IndexerDefinition indexer);
+        int CalculateRetryAfterQueryLimit(IndexerDefinition indexer);
     }
 
     public class IndexerLimitService : IIndexerLimitService
@@ -31,9 +32,9 @@ namespace NzbDrone.Core.Indexers
                 var grabCount = _historyService.CountSince(indexer.Id, DateTime.Now.AddHours(-24), new List<HistoryEventType> { HistoryEventType.ReleaseGrabbed });
                 var grabLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit;
 
-                if (grabCount > grabLimit)
+                if (grabCount >= grabLimit)
                 {
-                    _logger.Info("Indexer {0} has exceeded maximum grab limit for last 24 hours", indexer.Name);
+                    _logger.Info("Indexer {0} has performed {1} of possible {2} grabs in last 24 hours, exceeding the maximum grab limit", indexer.Name, grabCount, grabLimit);
 
                     return true;
                 }
@@ -51,9 +52,9 @@ namespace NzbDrone.Core.Indexers
                 var queryCount = _historyService.CountSince(indexer.Id, DateTime.Now.AddHours(-24), new List<HistoryEventType> { HistoryEventType.IndexerQuery, HistoryEventType.IndexerRss });
                 var queryLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit;
 
-                if (queryCount > queryLimit)
+                if (queryCount >= queryLimit)
                 {
-                    _logger.Info("Indexer {0} has exceeded maximum query limit for last 24 hours", indexer.Name);
+                    _logger.Info("Indexer {0} has performed {1} of possible {2} queries in last 24 hours, exceeding the maximum query limit", indexer.Name, queryCount, queryLimit);
 
                     return true;
                 }
@@ -63,5 +64,39 @@ namespace NzbDrone.Core.Indexers
 
             return false;
         }
+
+        public int CalculateRetryAfterDownloadLimit(IndexerDefinition indexer)
+        {
+            if (indexer.Id > 0 && ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit.HasValue)
+            {
+                var grabLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit.GetValueOrDefault();
+
+                var firstHistorySince = _historyService.FindFirstForIndexerSince(indexer.Id, DateTime.Now.AddHours(-24), new List<HistoryEventType> { HistoryEventType.ReleaseGrabbed }, grabLimit);
+
+                if (firstHistorySince != null)
+                {
+                    return Convert.ToInt32(firstHistorySince.Date.ToLocalTime().AddHours(24).Subtract(DateTime.Now).TotalSeconds);
+                }
+            }
+
+            return 0;
+        }
+
+        public int CalculateRetryAfterQueryLimit(IndexerDefinition indexer)
+        {
+            if (indexer.Id > 0 && ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit.HasValue)
+            {
+                var queryLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit.GetValueOrDefault();
+
+                var firstHistorySince = _historyService.FindFirstForIndexerSince(indexer.Id, DateTime.Now.AddHours(-24), new List<HistoryEventType> { HistoryEventType.IndexerQuery, HistoryEventType.IndexerRss }, queryLimit);
+
+                if (firstHistorySince != null)
+                {
+                    return Convert.ToInt32(firstHistorySince.Date.ToLocalTime().AddHours(24).Subtract(DateTime.Now).TotalSeconds);
+                }
+            }
+
+            return 0;
+        }
     }
 }

src/NzbDrone.Core/Indexers/Settings/CookieTorrentBaseSettings.cs

@@ -4,38 +4,28 @@ using NzbDrone.Core.Validation;
 
 namespace NzbDrone.Core.Indexers.Settings
 {
-    public class CookieBaseSettingsValidator : AbstractValidator<CookieTorrentBaseSettings>
+    public class CookieBaseSettingsValidator<T> : NoAuthSettingsValidator<T>
+        where T : CookieTorrentBaseSettings
     {
         public CookieBaseSettingsValidator()
         {
             RuleFor(c => c.Cookie).NotEmpty();
-            RuleFor(x => x.BaseSettings).SetValidator(new IndexerCommonSettingsValidator());
-            RuleFor(x => x.TorrentBaseSettings).SetValidator(new IndexerTorrentSettingsValidator());
         }
     }
 
-    public class CookieTorrentBaseSettings : ITorrentIndexerSettings
+    public class CookieTorrentBaseSettings : NoAuthTorrentBaseSettings
     {
-        private static readonly CookieBaseSettingsValidator Validator = new ();
+        private static readonly CookieBaseSettingsValidator<CookieTorrentBaseSettings> Validator = new ();
 
         public CookieTorrentBaseSettings()
         {
             Cookie = "";
         }
 
-        [FieldDefinition(1, Label = "Base Url", HelpText = "Select which baseurl Prowlarr will use for requests to the site", Type = FieldType.Select, SelectOptionsProviderAction = "getUrls")]
-        public string BaseUrl { get; set; }
-
-        [FieldDefinition(2, Label = "Cookie", HelpText = "Site Cookie", Privacy = PrivacyLevel.Password, Type = FieldType.Password)]
+        [FieldDefinition(2, Label = "Cookie", HelpText = "Site Cookie", Privacy = PrivacyLevel.Password)]
         public string Cookie { get; set; }
 
-        [FieldDefinition(10)]
-        public IndexerBaseSettings BaseSettings { get; set; } = new ();
-
-        [FieldDefinition(11)]
-        public IndexerTorrentBaseSettings TorrentBaseSettings { get; set; } = new ();
-
-        public virtual NzbDroneValidationResult Validate()
+        public override NzbDroneValidationResult Validate()
         {
             return new NzbDroneValidationResult(Validator.Validate(this));
         }

src/NzbDrone.Core/Indexers/Settings/NoAuthTorrentBaseSettings.cs

@@ -9,8 +9,8 @@ namespace NzbDrone.Core.Indexers.Settings
     {
         public NoAuthSettingsValidator()
         {
-            RuleFor(x => x.BaseSettings).SetValidator(new IndexerCommonSettingsValidator());
-            RuleFor(x => x.TorrentBaseSettings).SetValidator(new IndexerTorrentSettingsValidator());
+            RuleFor(c => c.BaseSettings).SetValidator(new IndexerCommonSettingsValidator());
+            RuleFor(c => c.TorrentBaseSettings).SetValidator(new IndexerTorrentSettingsValidator());
         }
     }
 
@@ -21,10 +21,10 @@ namespace NzbDrone.Core.Indexers.Settings
         [FieldDefinition(1, Label = "Base Url", Type = FieldType.Select, SelectOptionsProviderAction = "getUrls", HelpText = "Select which baseurl Prowlarr will use for requests to the site")]
         public string BaseUrl { get; set; }
 
-        [FieldDefinition(10)]
+        [FieldDefinition(20)]
         public IndexerBaseSettings BaseSettings { get; set; } = new ();
 
-        [FieldDefinition(11)]
+        [FieldDefinition(21)]
         public IndexerTorrentBaseSettings TorrentBaseSettings { get; set; } = new ();
 
         public virtual NzbDroneValidationResult Validate()

src/NzbDrone.Core/Indexers/Settings/UserPassTorrentBaseSettings.cs

@@ -4,19 +4,17 @@ using NzbDrone.Core.Validation;
 
 namespace NzbDrone.Core.Indexers.Settings
 {
-    public class UserPassBaseSettingsValidator<T> : AbstractValidator<T>
+    public class UserPassBaseSettingsValidator<T> : NoAuthSettingsValidator<T>
         where T : UserPassTorrentBaseSettings
     {
         public UserPassBaseSettingsValidator()
         {
             RuleFor(c => c.Username).NotEmpty();
             RuleFor(c => c.Password).NotEmpty();
-            RuleFor(x => x.BaseSettings).SetValidator(new IndexerCommonSettingsValidator());
-            RuleFor(x => x.TorrentBaseSettings).SetValidator(new IndexerTorrentSettingsValidator());
         }
     }
 
-    public class UserPassTorrentBaseSettings : ITorrentIndexerSettings
+    public class UserPassTorrentBaseSettings : NoAuthTorrentBaseSettings
     {
         private static readonly UserPassBaseSettingsValidator<UserPassTorrentBaseSettings> Validator = new ();
 
@@ -26,22 +24,13 @@ namespace NzbDrone.Core.Indexers.Settings
             Password = "";
         }
 
-        [FieldDefinition(1, Label = "Base Url", HelpText = "Select which baseurl Prowlarr will use for requests to the site", Type = FieldType.Select, SelectOptionsProviderAction = "getUrls")]
-        public string BaseUrl { get; set; }
-
         [FieldDefinition(2, Label = "Username", HelpText = "Site Username", Privacy = PrivacyLevel.UserName)]
         public string Username { get; set; }
 
         [FieldDefinition(3, Label = "Password", HelpText = "Site Password", Privacy = PrivacyLevel.Password, Type = FieldType.Password)]
         public string Password { get; set; }
 
-        [FieldDefinition(10)]
-        public IndexerBaseSettings BaseSettings { get; set; } = new ();
-
-        [FieldDefinition(11)]
-        public IndexerTorrentBaseSettings TorrentBaseSettings { get; set; } = new ();
-
-        public virtual NzbDroneValidationResult Validate()
+        public override NzbDroneValidationResult Validate()
         {
             return new NzbDroneValidationResult(Validator.Validate(this));
         }

src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs

@@ -1,12 +1,7 @@
-using System;
-using System.Net;
 using System.Text;
-using System.Threading.Tasks;
 using MonoTorrent;
 using NLog;
-using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Messaging.Events;
 
 namespace NzbDrone.Core.Indexers
@@ -19,85 +14,15 @@ namespace NzbDrone.Core.Indexers
         {
         }
 
-        public override async Task<byte[]> Download(Uri link)
-        {
-            Cookies = GetCookies();
-
-            if (link.Scheme == "magnet")
-            {
-                ValidateMagnet(link.OriginalString);
-
-                return Encoding.UTF8.GetBytes(link.OriginalString);
-            }
-
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-
-            byte[] torrentData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                torrentData = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            ValidateTorrent(torrentData);
-
-            return torrentData;
-        }
-
-        protected void ValidateMagnet(string link)
-        {
-            MagnetLink.Parse(link);
-        }
-
-        protected void ValidateTorrent(byte[] torrentData)
+        protected override void ValidateDownloadData(byte[] fileData)
         {
             try
             {
-                Torrent.Load(torrentData);
+                Torrent.Load(fileData);
             }
             catch
             {
-                _logger.Trace("Invalid torrent file contents: {0}", Encoding.ASCII.GetString(torrentData));
+                _logger.Trace("Invalid torrent file contents: {0}", Encoding.ASCII.GetString(fileData));
                 throw;
             }
         }

src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs

@@ -1,11 +1,6 @@
-using System;
-using System.Net;
-using System.Threading.Tasks;
 using NLog;
-using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Download;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Messaging.Events;
 
 namespace NzbDrone.Core.Indexers
@@ -21,64 +16,9 @@ namespace NzbDrone.Core.Indexers
             _nzbValidationService = nzbValidationService;
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override void ValidateDownloadData(byte[] fileData)
         {
-            Cookies = GetCookies();
-
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-
-            byte[] nzbData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                nzbData = response.ResponseData;
-
-                _logger.Debug("Downloaded nzb for release finished ({0} bytes from {1})", nzbData.Length, link.AbsoluteUri);
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading nzb file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading nzb failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading nzb failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading nzb failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading nzb failed");
-                throw;
-            }
-
-            _nzbValidationService.Validate(nzbData);
-
-            return nzbData;
+            _nzbValidationService.Validate(fileData);
         }
     }
 }

src/NzbDrone.Core/Localization/Core/en.json

@@ -185,6 +185,7 @@
   "IgnoredAddresses": "Ignored Addresses",
   "IllRestartLater": "I'll restart later",
   "IncludeHealthWarningsHelpText": "Include Health Warnings",
+  "IncludeManualGrabsHelpText": "Include Manual Grabs made within Prowlarr",
   "Indexer": "Indexer",
   "IndexerAlreadySetup": "At least one instance of indexer is already setup",
   "IndexerAuth": "Indexer Auth",
@@ -272,7 +273,8 @@
   "Ok": "Ok",
   "OnApplicationUpdate": "On Application Update",
   "OnApplicationUpdateHelpText": "On Application Update",
-  "OnGrab": "On Grab",
+  "OnGrab": "On Release Grab",
+  "OnGrabHelpText": "On Release Grab",
   "OnHealthIssue": "On Health Issue",
   "OnHealthIssueHelpText": "On Health Issue",
   "OpenBrowserOnStart": "Open browser on start",

src/NzbDrone.Core/Notifications/Apprise/Apprise.cs

@@ -17,6 +17,11 @@ namespace NzbDrone.Core.Notifications.Apprise
             _proxy = proxy;
         }
 
+        public override void OnGrab(GrabMessage message)
+        {
+            _proxy.SendNotification(Settings, RELEASE_GRABBED_TITLE_BRANDED, $"{message.Message}");
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
             _proxy.SendNotification(Settings, HEALTH_ISSUE_TITLE_BRANDED, $"{healthCheck.Message}");

src/NzbDrone.Core/Notifications/Boxcar/Boxcar.cs

@@ -15,6 +15,12 @@ namespace NzbDrone.Core.Notifications.Boxcar
 
         public override string Link => "https://boxcar.io/client";
         public override string Name => "Boxcar";
+
+        public override void OnGrab(GrabMessage message)
+        {
+            _proxy.SendNotification(RELEASE_GRABBED_TITLE, message.Message, Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck message)
         {
             _proxy.SendNotification(HEALTH_ISSUE_TITLE, message.Message, Settings);

src/NzbDrone.Core/Notifications/Discord/Discord.cs

@@ -19,6 +19,65 @@ namespace NzbDrone.Core.Notifications.Discord
         public override string Name => "Discord";
         public override string Link => "https://support.discordapp.com/hc/en-us/articles/228383668-Intro-to-Webhooks";
 
+        public override void OnGrab(GrabMessage message)
+        {
+            var embed = new Embed
+                        {
+                            Author = new DiscordAuthor
+                            {
+                                Name = Settings.Author.IsNullOrWhiteSpace() ? Environment.MachineName : Settings.Author,
+                                IconUrl = "https://raw.githubusercontent.com/Prowlarr/Prowlarr/develop/Logo/256.png"
+                            },
+                            Title = RELEASE_GRABBED_TITLE,
+                            Description = message.Message,
+                            Timestamp = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"),
+                            Color = message.Successful ? (int)DiscordColors.Success : (int)DiscordColors.Danger,
+                            Fields = new List<DiscordField>()
+                        };
+
+            foreach (var field in Settings.GrabFields)
+            {
+                var discordField = new DiscordField();
+
+                switch ((DiscordGrabFieldType)field)
+                {
+                    case DiscordGrabFieldType.Release:
+                        discordField.Name = "Release";
+                        discordField.Value = string.Format("```{0}```", message.Release.Title);
+                        break;
+                    case DiscordGrabFieldType.Indexer:
+                        discordField.Name = "Indexer";
+                        discordField.Value = message.Release.Indexer ?? string.Empty;
+                        break;
+                    case DiscordGrabFieldType.DownloadClient:
+                        discordField.Name = "Download Client";
+                        discordField.Value = message.DownloadClientName ?? string.Empty;
+                        break;
+                    case DiscordGrabFieldType.GrabTrigger:
+                        discordField.Name = "Grab Trigger";
+                        discordField.Value = message.GrabTrigger.ToString() ?? string.Empty;
+                        break;
+                    case DiscordGrabFieldType.Source:
+                        discordField.Name = "Source";
+                        discordField.Value = message.Source ?? string.Empty;
+                        break;
+                    case DiscordGrabFieldType.Host:
+                        discordField.Name = "Host";
+                        discordField.Value = message.Host ?? string.Empty;
+                        break;
+                }
+
+                if (discordField.Name.IsNotNullOrWhiteSpace() && discordField.Value.IsNotNullOrWhiteSpace())
+                {
+                    embed.Fields.Add(discordField);
+                }
+            }
+
+            var payload = CreatePayload(null, new List<Embed> { embed });
+
+            _proxy.SendPayload(payload, Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
             var attachments = new List<Embed>

src/NzbDrone.Core/Notifications/Discord/DiscordFieldType.cs

@@ -2,32 +2,11 @@ namespace NzbDrone.Core.Notifications.Discord
 {
     public enum DiscordGrabFieldType
     {
-        Overview,
-        Rating,
-        Genres,
-        Quality,
-        Group,
-        Size,
-        Links,
         Release,
-        Poster,
-        Fanart
-    }
-
-    public enum DiscordImportFieldType
-    {
-        Overview,
-        Rating,
-        Genres,
-        Quality,
-        Codecs,
-        Group,
-        Size,
-        Languages,
-        Subtitles,
-        Links,
-        Release,
-        Poster,
-        Fanart
+        Indexer,
+        DownloadClient,
+        GrabTrigger,
+        Source,
+        Host
     }
 }

src/NzbDrone.Core/Notifications/Discord/DiscordSettings.cs

@@ -20,7 +20,6 @@ namespace NzbDrone.Core.Notifications.Discord
         {
             //Set Default Fields
             GrabFields = new List<int> { 0, 1, 2, 3, 5, 6, 7, 8, 9 };
-            ImportFields = new List<int> { 0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12 };
         }
 
         private static readonly DiscordSettingsValidator Validator = new DiscordSettingsValidator();
@@ -40,9 +39,6 @@ namespace NzbDrone.Core.Notifications.Discord
         [FieldDefinition(4, Label = "On Grab Fields", Advanced = true, SelectOptions = typeof(DiscordGrabFieldType), HelpText = "Change the fields that are passed in for this 'on grab' notification", Type = FieldType.TagSelect)]
         public IEnumerable<int> GrabFields { get; set; }
 
-        [FieldDefinition(5, Label = "On Import Fields", Advanced = true, SelectOptions = typeof(DiscordImportFieldType), HelpText = "Change the fields that are passed for this 'on import' notification", Type = FieldType.TagSelect)]
-        public IEnumerable<int> ImportFields { get; set; }
-
         public NzbDroneValidationResult Validate()
         {
             return new NzbDroneValidationResult(Validator.Validate(this));

src/NzbDrone.Core/Notifications/Email/Email.cs

@@ -26,6 +26,11 @@ namespace NzbDrone.Core.Notifications.Email
 
         public override string Link => null;
 
+        public override void OnGrab(GrabMessage message)
+        {
+            SendEmail(Settings, RELEASE_GRABBED_TITLE_BRANDED, message.Message);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck message)
         {
             SendEmail(Settings, HEALTH_ISSUE_TITLE_BRANDED, message.Message);

src/NzbDrone.Core/Notifications/Gotify/Gotify.cs

@@ -19,6 +19,11 @@ namespace NzbDrone.Core.Notifications.Gotify
         public override string Name => "Gotify";
         public override string Link => "https://gotify.net/";
 
+        public override void OnGrab(GrabMessage message)
+        {
+            _proxy.SendNotification(RELEASE_GRABBED_TITLE, message.Message, Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
             _proxy.SendNotification(HEALTH_ISSUE_TITLE, healthCheck.Message, Settings);

src/NzbDrone.Core/Notifications/GrabMessage.cs

@@ -0,0 +1,25 @@
+using NzbDrone.Core.Indexers.Events;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.Notifications
+{
+    public class GrabMessage
+    {
+        public ReleaseInfo Release { get; set; }
+
+        public bool Successful { get; set; }
+        public string Host { get; set; }
+        public string Source { get; set; }
+        public GrabTrigger GrabTrigger { get; set; }
+        public bool Redirect { get; set; }
+        public string Message { get; set; }
+        public string DownloadClientType { get; set; }
+        public string DownloadClientName { get; set; }
+        public string DownloadId { get; set; }
+
+        public override string ToString()
+        {
+            return Message;
+        }
+    }
+}

src/NzbDrone.Core/Notifications/INotification.cs

@@ -6,9 +6,11 @@ namespace NzbDrone.Core.Notifications
     {
         string Link { get; }
 
+        void OnGrab(GrabMessage grabMessage);
         void OnHealthIssue(HealthCheck.HealthCheck healthCheck);
         void OnApplicationUpdate(ApplicationUpdateMessage updateMessage);
         void ProcessQueue();
+        bool SupportsOnGrab { get; }
         bool SupportsOnHealthIssue { get; }
         bool SupportsOnApplicationUpdate { get; }
     }

src/NzbDrone.Core/Notifications/Join/Join.cs

@@ -17,6 +17,11 @@ namespace NzbDrone.Core.Notifications.Join
 
         public override string Link => "https://joaoapps.com/join/";
 
+        public override void OnGrab(GrabMessage message)
+        {
+            _proxy.SendNotification(RELEASE_GRABBED_TITLE_BRANDED, message.Message, Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck message)
         {
             _proxy.SendNotification(HEALTH_ISSUE_TITLE_BRANDED, message.Message, Settings);

src/NzbDrone.Core/Notifications/Notifiarr/Notifiarr.cs

@@ -19,6 +19,12 @@ namespace NzbDrone.Core.Notifications.Notifiarr
 
         public override string Link => "https://notifiarr.com";
         public override string Name => "Notifiarr";
+
+        public override void OnGrab(GrabMessage grabMessage)
+        {
+            _proxy.SendNotification(BuildGrabPayload(grabMessage), Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
             _proxy.SendNotification(BuildHealthPayload(healthCheck), Settings);

src/NzbDrone.Core/Notifications/NotificationBase.cs

@@ -8,9 +8,11 @@ namespace NzbDrone.Core.Notifications
     public abstract class NotificationBase<TSettings> : INotification
         where TSettings : IProviderConfig, new()
     {
+        protected const string RELEASE_GRABBED_TITLE = "Release Grabbed";
         protected const string HEALTH_ISSUE_TITLE = "Health Check Failure";
         protected const string APPLICATION_UPDATE_TITLE = "Application Updated";
 
+        protected const string RELEASE_GRABBED_TITLE_BRANDED = "Prowlarr - " + RELEASE_GRABBED_TITLE;
         protected const string HEALTH_ISSUE_TITLE_BRANDED = "Prowlarr - " + HEALTH_ISSUE_TITLE;
         protected const string APPLICATION_UPDATE_TITLE_BRANDED = "Prowlarr - " + APPLICATION_UPDATE_TITLE;
 
@@ -27,6 +29,10 @@ namespace NzbDrone.Core.Notifications
 
         public abstract string Link { get; }
 
+        public virtual void OnGrab(GrabMessage grabMessage)
+        {
+        }
+
         public virtual void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
         }
@@ -39,6 +45,7 @@ namespace NzbDrone.Core.Notifications
         {
         }
 
+        public bool SupportsOnGrab => HasConcreteImplementation("OnGrab");
         public bool SupportsOnHealthIssue => HasConcreteImplementation("OnHealthIssue");
         public bool SupportsOnApplicationUpdate => HasConcreteImplementation("OnApplicationUpdate");
 

src/NzbDrone.Core/Notifications/NotificationDefinition.cs

@@ -6,10 +6,13 @@ namespace NzbDrone.Core.Notifications
     {
         public bool OnHealthIssue { get; set; }
         public bool OnApplicationUpdate { get; set; }
+        public bool OnGrab { get; set; }
+        public bool SupportsOnGrab { get; set; }
+        public bool IncludeManualGrabs { get; set; }
         public bool SupportsOnHealthIssue { get; set; }
         public bool IncludeHealthWarnings { get; set; }
         public bool SupportsOnApplicationUpdate { get; set; }
 
-        public override bool Enable => OnHealthIssue || OnApplicationUpdate;
+        public override bool Enable => OnHealthIssue || OnApplicationUpdate || OnGrab;
     }
 }

src/NzbDrone.Core/Notifications/NotificationFactory.cs

@@ -9,6 +9,7 @@ namespace NzbDrone.Core.Notifications
 {
     public interface INotificationFactory : IProviderFactory<INotification, NotificationDefinition>
     {
+        List<INotification> OnGrabEnabled();
         List<INotification> OnHealthIssueEnabled();
         List<INotification> OnApplicationUpdateEnabled();
     }
@@ -20,6 +21,11 @@ namespace NzbDrone.Core.Notifications
         {
         }
 
+        public List<INotification> OnGrabEnabled()
+        {
+            return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnGrab).ToList();
+        }
+
         public List<INotification> OnHealthIssueEnabled()
         {
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthIssue).ToList();
@@ -34,6 +40,7 @@ namespace NzbDrone.Core.Notifications
         {
             base.SetProviderCharacteristics(provider, definition);
 
+            definition.SupportsOnGrab = provider.SupportsOnGrab;
             definition.SupportsOnHealthIssue = provider.SupportsOnHealthIssue;
             definition.SupportsOnApplicationUpdate = provider.SupportsOnApplicationUpdate;
         }

src/NzbDrone.Core/Notifications/NotificationService.cs

@@ -1,7 +1,12 @@
 using System;
+using System.Drawing.Drawing2D;
 using NLog;
+using NzbDrone.Common.Extensions;
 using NzbDrone.Core.HealthCheck;
+using NzbDrone.Core.Indexers.Events;
+using NzbDrone.Core.Indexers.PassThePopcorn;
 using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.Update.History.Events;
 
 namespace NzbDrone.Core.Notifications
@@ -9,7 +14,8 @@ namespace NzbDrone.Core.Notifications
     public class NotificationService
         : IHandle<HealthCheckFailedEvent>,
           IHandleAsync<HealthCheckCompleteEvent>,
-          IHandle<UpdateInstalledEvent>
+          IHandle<UpdateInstalledEvent>,
+          IHandle<IndexerDownloadEvent>
     {
         private readonly INotificationFactory _notificationFactory;
         private readonly Logger _logger;
@@ -35,6 +41,43 @@ namespace NzbDrone.Core.Notifications
             return false;
         }
 
+        private bool ShouldHandleOnGrab(GrabMessage message, bool includeManual)
+        {
+            if (message.GrabTrigger == GrabTrigger.Api)
+            {
+                return true;
+            }
+
+            if (message.GrabTrigger == GrabTrigger.Manual && includeManual)
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        private string GetMessage(ReleaseInfo release, GrabTrigger grabTrigger, string source, string downloadClient)
+        {
+            var message = string.Format("{0} grabbed by {1} from {2}",
+                                    release.Title,
+                                    source,
+                                    release.Indexer);
+
+            if (grabTrigger == GrabTrigger.Manual)
+            {
+                message = string.Format("{0} manually grabbed in Prowlarr from {1}",
+                                    release.Title,
+                                    release.Indexer);
+            }
+
+            if (downloadClient.IsNotNullOrWhiteSpace())
+            {
+                message += $" and sent to {downloadClient}";
+            }
+
+            return message;
+        }
+
         public void Handle(HealthCheckFailedEvent message)
         {
             // Don't send health check notifications during the start up grace period,
@@ -99,5 +142,37 @@ namespace NzbDrone.Core.Notifications
                 }
             }
         }
+
+        public void Handle(IndexerDownloadEvent message)
+        {
+            var grabMessage = new GrabMessage
+            {
+                Release = message.Release,
+                Source = message.Source,
+                Host = message.Host,
+                Successful = message.Successful,
+                DownloadClientName = message.DownloadClientName,
+                DownloadClientType = message.DownloadClient,
+                DownloadId = message.DownloadId,
+                Redirect = message.Redirect,
+                GrabTrigger = message.GrabTrigger,
+                Message = GetMessage(message.Release, message.GrabTrigger, message.Source, message.DownloadClientName)
+            };
+
+            foreach (var notification in _notificationFactory.OnGrabEnabled())
+            {
+                try
+                {
+                    if (ShouldHandleOnGrab(grabMessage, ((NotificationDefinition)notification.Definition).IncludeManualGrabs))
+                    {
+                        notification.OnGrab(grabMessage);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    _logger.Error(ex, "Unable to send OnGrab notification to {0}", notification.Definition.Name);
+                }
+            }
+        }
     }
 }

src/NzbDrone.Core/Notifications/Ntfy/Ntfy.cs

@@ -17,6 +17,11 @@ namespace NzbDrone.Core.Notifications.Ntfy
 
         public override string Link => "https://ntfy.sh/";
 
+        public override void OnGrab(GrabMessage message)
+        {
+            _proxy.SendNotification(RELEASE_GRABBED_TITLE_BRANDED, message.Message, Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck message)
         {
             _proxy.SendNotification(HEALTH_ISSUE_TITLE_BRANDED, message.Message, Settings);

src/NzbDrone.Core/Notifications/Prowl/Prowl.cs

@@ -16,6 +16,11 @@ namespace NzbDrone.Core.Notifications.Prowl
         public override string Link => "https://www.prowlapp.com/";
         public override string Name => "Prowl";
 
+        public override void OnGrab(GrabMessage message)
+        {
+            _prowlProxy.SendNotification(RELEASE_GRABBED_TITLE, message.Message, Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
             _prowlProxy.SendNotification(HEALTH_ISSUE_TITLE, healthCheck.Message, Settings);

src/NzbDrone.Core/Notifications/PushBullet/PushBullet.cs

@@ -19,6 +19,11 @@ namespace NzbDrone.Core.Notifications.PushBullet
         public override string Name => "Pushbullet";
         public override string Link => "https://www.pushbullet.com/";
 
+        public override void OnGrab(GrabMessage message)
+        {
+            _proxy.SendNotification(RELEASE_GRABBED_TITLE_BRANDED, message.Message, Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
             _proxy.SendNotification(HEALTH_ISSUE_TITLE_BRANDED, healthCheck.Message, Settings);

src/NzbDrone.Core/Notifications/Pushover/Pushover.cs

@@ -16,6 +16,11 @@ namespace NzbDrone.Core.Notifications.Pushover
         public override string Name => "Pushover";
         public override string Link => "https://pushover.net/";
 
+        public override void OnGrab(GrabMessage message)
+        {
+            _proxy.SendNotification(RELEASE_GRABBED_TITLE, message.Message, Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
             _proxy.SendNotification(HEALTH_ISSUE_TITLE, healthCheck.Message, Settings);

src/NzbDrone.Core/Notifications/SendGrid/SendGrid.cs

@@ -19,6 +19,11 @@ namespace NzbDrone.Core.Notifications.SendGrid
         public override string Name => "SendGrid";
         public override string Link => "https://sendgrid.com/";
 
+        public override void OnGrab(GrabMessage message)
+        {
+            _proxy.SendNotification(RELEASE_GRABBED_TITLE, message.Message, Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
             _proxy.SendNotification(HEALTH_ISSUE_TITLE, healthCheck.Message, Settings);

src/NzbDrone.Core/Notifications/Simplepush/Simplepush.cs

@@ -16,6 +16,11 @@ namespace NzbDrone.Core.Notifications.Simplepush
         public override string Name => "Simplepush";
         public override string Link => "https://simplepush.io/";
 
+        public override void OnGrab(GrabMessage message)
+        {
+            _proxy.SendNotification(RELEASE_GRABBED_TITLE, message.Message, Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
             _proxy.SendNotification(HEALTH_ISSUE_TITLE, healthCheck.Message, Settings);

src/NzbDrone.Core/Notifications/Telegram/Telegram.cs

@@ -16,6 +16,11 @@ namespace NzbDrone.Core.Notifications.Telegram
         public override string Name => "Telegram";
         public override string Link => "https://telegram.org/";
 
+        public override void OnGrab(GrabMessage message)
+        {
+            _proxy.SendNotification(RELEASE_GRABBED_TITLE, message.Message, Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
             _proxy.SendNotification(HEALTH_ISSUE_TITLE, healthCheck.Message, Settings);

src/NzbDrone.Core/Notifications/Twitter/Twitter.cs

@@ -2,6 +2,7 @@ using System.Collections.Generic;
 using FluentValidation.Results;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Exceptions;
+using NzbDrone.Core.HealthCheck;
 using NzbDrone.Core.Validation;
 
 namespace NzbDrone.Core.Notifications.Twitter
@@ -18,6 +19,11 @@ namespace NzbDrone.Core.Notifications.Twitter
         public override string Name => "Twitter";
         public override string Link => "https://twitter.com/";
 
+        public override void OnGrab(GrabMessage message)
+        {
+            _twitterService.SendNotification($"Release Grabbed: {message.Message}", Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
             _twitterService.SendNotification($"Health Issue: {healthCheck.Message}", Settings);

src/NzbDrone.Core/Notifications/Webhook/Webhook.cs

@@ -18,6 +18,11 @@ namespace NzbDrone.Core.Notifications.Webhook
 
         public override string Link => "https://wiki.servarr.com/prowlarr/settings#connect";
 
+        public override void OnGrab(GrabMessage grabMessage)
+        {
+            _proxy.SendWebhook(BuildGrabPayload(grabMessage), Settings);
+        }
+
         public override void OnHealthIssue(HealthCheck.HealthCheck healthCheck)
         {
             _proxy.SendWebhook(BuildHealthPayload(healthCheck), Settings);

src/NzbDrone.Core/Notifications/Webhook/WebhookBase.cs

@@ -14,6 +14,22 @@ namespace NzbDrone.Core.Notifications.Webhook
             _configFileProvider = configFileProvider;
         }
 
+        public WebhookGrabPayload BuildGrabPayload(GrabMessage message)
+        {
+            return new WebhookGrabPayload
+            {
+                EventType = WebhookEventType.Grab,
+                InstanceName = _configFileProvider.InstanceName,
+                Release = new WebhookRelease(message.Release),
+                Trigger = message.GrabTrigger,
+                Source = message.Source,
+                Host = message.Host,
+                DownloadClient = message.DownloadClientName,
+                DownloadClientType = message.DownloadClientType,
+                DownloadId = message.DownloadId
+            };
+        }
+
         protected WebhookHealthPayload BuildHealthPayload(HealthCheck.HealthCheck healthCheck)
         {
             return new WebhookHealthPayload

src/NzbDrone.Core/Notifications/Webhook/WebhookGrabPayload.cs

@@ -0,0 +1,15 @@
+using NzbDrone.Core.Indexers.Events;
+
+namespace NzbDrone.Core.Notifications.Webhook
+{
+    public class WebhookGrabPayload : WebhookPayload
+    {
+        public WebhookRelease Release { get; set; }
+        public GrabTrigger Trigger { get; set; }
+        public string Source { get; set; }
+        public string Host { get; set; }
+        public string DownloadClient { get; set; }
+        public string DownloadClientType { get; set; }
+        public string DownloadId { get; set; }
+    }
+}

src/NzbDrone.Core/Notifications/Webhook/WebhookRelease.cs

@@ -0,0 +1,22 @@
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.Notifications.Webhook
+{
+    public class WebhookRelease
+    {
+        public WebhookRelease()
+        {
+        }
+
+        public WebhookRelease(ReleaseInfo release)
+        {
+            ReleaseTitle = release.Title;
+            Indexer = release.Indexer;
+            Size = release.Size;
+        }
+
+        public string ReleaseTitle { get; set; }
+        public string Indexer { get; set; }
+        public long? Size { get; set; }
+    }
+}

src/NzbDrone.Core/Parser/DateTimeUtil.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Globalization;
+using System.Linq;
 using System.Text.RegularExpressions;
+using NzbDrone.Common.Extensions;
 
 namespace NzbDrone.Core.Parser
 {
@@ -120,6 +122,18 @@ namespace NzbDrone.Core.Parser
             try
             {
                 str = str.Trim();
+
+                if (DateTime.TryParseExact(str, Rfc1123ZPattern, CultureInfo.InvariantCulture, DateTimeStyles.None, out var parsedDate))
+                {
+                    return parsedDate;
+                }
+
+                // try parsing the str as an unix timestamp
+                if (str.IsAllDigits() && long.TryParse(str, out var unixTimeStamp))
+                {
+                    return UnixTimestampToDateTime(unixTimeStamp);
+                }
+
                 if (str.ToLower().Contains("now"))
                 {
                     return DateTime.UtcNow;
@@ -212,14 +226,6 @@ namespace NzbDrone.Core.Parser
                     return dt;
                 }
 
-                // try parsing the str as an unix timestamp
-                if (long.TryParse(str, out var unixTimeStamp))
-                {
-                    return UnixTimestampToDateTime(unixTimeStamp);
-                }
-
-                // it wasn't a timestamp, continue....
-
                 // add missing year
                 match = _MissingYearRegexp.Match(str);
                 if (match.Success)
@@ -250,69 +256,73 @@ namespace NzbDrone.Core.Parser
         public static DateTime ParseDateTimeGoLang(string date, string layout)
         {
             date = date.Trim();
-            var pattern = layout;
 
-            // year
-            pattern = pattern.Replace("2006", "yyyy");
-            pattern = pattern.Replace("06", "yy");
+            var commonStandardFormats = new[] { "y", "h", "d" };
 
-            // month
-            pattern = pattern.Replace("January", "MMMM");
-            pattern = pattern.Replace("Jan", "MMM");
-            pattern = pattern.Replace("01", "MM");
+            if (commonStandardFormats.Any(layout.ContainsIgnoreCase) && DateTime.TryParseExact(date, layout, CultureInfo.InvariantCulture, DateTimeStyles.None, out var parsedDate))
+            {
+                return parsedDate;
+            }
 
-            // day
-            pattern = pattern.Replace("Monday", "dddd");
-            pattern = pattern.Replace("Mon", "ddd");
-            pattern = pattern.Replace("02", "dd");
-            pattern = pattern.Replace("2", "d");
+            var format = layout
 
-            // hours/minutes/seconds
-            pattern = pattern.Replace("05", "ss");
+                // year
+                .Replace("2006", "yyyy")
+                .Replace("06", "yy")
 
-            pattern = pattern.Replace("15", "HH");
-            pattern = pattern.Replace("03", "hh");
-            pattern = pattern.Replace("3", "h");
+                // month
+                .Replace("January", "MMMM")
+                .Replace("Jan", "MMM")
+                .Replace("01", "MM")
 
-            pattern = pattern.Replace("04", "mm");
-            pattern = pattern.Replace("4", "m");
+                // day
+                .Replace("Monday", "dddd")
+                .Replace("Mon", "ddd")
+                .Replace("02", "dd")
+                .Replace("2", "d")
 
-            pattern = pattern.Replace("5", "s");
+                // hours/minutes/seconds
+                .Replace("05", "ss")
+                .Replace("15", "HH")
+                .Replace("03", "hh")
+                .Replace("3", "h")
+                .Replace("04", "mm")
+                .Replace("4", "m")
+                .Replace("5", "s")
 
-            // month again
-            pattern = pattern.Replace("1", "M");
+                // month again
+                .Replace("1", "M")
 
-            // fractional seconds
-            pattern = pattern.Replace(".0000", "ffff");
-            pattern = pattern.Replace(".000", "fff");
-            pattern = pattern.Replace(".00", "ff");
-            pattern = pattern.Replace(".0", "f");
+                // fractional seconds
+                .Replace(".0000", "ffff")
+                .Replace(".000", "fff")
+                .Replace(".00", "ff")
+                .Replace(".0", "f")
+                .Replace(".9999", "FFFF")
+                .Replace(".999", "FFF")
+                .Replace(".99", "FF")
+                .Replace(".9", "F")
 
-            pattern = pattern.Replace(".9999", "FFFF");
-            pattern = pattern.Replace(".999", "FFF");
-            pattern = pattern.Replace(".99", "FF");
-            pattern = pattern.Replace(".9", "F");
+                // AM/PM
+                .Replace("PM", "tt")
+                .Replace("pm", "tt") // not sure if this works
 
-            // AM/PM
-            pattern = pattern.Replace("PM", "tt");
-            pattern = pattern.Replace("pm", "tt"); // not sure if this works
-
-            // timezones
-            // these might need further tuning
-            pattern = pattern.Replace("Z07:00", "'Z'zzz");
-            pattern = pattern.Replace("Z07", "'Z'zz");
-            pattern = pattern.Replace("Z07:00", "'Z'zzz");
-            pattern = pattern.Replace("Z07", "'Z'zz");
-            pattern = pattern.Replace("-07:00", "zzz");
-            pattern = pattern.Replace("-07", "zz");
+                // timezones
+                // these might need further tuning
+                .Replace("Z07:00", "'Z'zzz")
+                .Replace("Z07", "'Z'zz")
+                .Replace("Z07:00", "'Z'zzz")
+                .Replace("Z07", "'Z'zz")
+                .Replace("-07:00", "zzz")
+                .Replace("-07", "zz");
 
             try
             {
-                return DateTime.ParseExact(date, pattern, CultureInfo.InvariantCulture);
+                return DateTime.ParseExact(date, format, CultureInfo.InvariantCulture);
             }
             catch (FormatException ex)
             {
-                throw new InvalidDateException($"Error while parsing DateTime \"{date}\", using layout \"{layout}\" ({pattern}): {ex.Message}");
+                throw new InvalidDateException($"Error while parsing DateTime \"{date}\", using layout \"{layout}\" ({format}): {ex.Message}", ex);
             }
         }