diff --git a/src/NzbDrone.Common/Http/CookieUtil.cs b/src/NzbDrone.Common/Http/CookieUtil.cs new file mode 100644 index 000000000..5b57c06a1 --- /dev/null +++ b/src/NzbDrone.Common/Http/CookieUtil.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace NzbDrone.Common.Http +{ + public static class CookieUtil + { + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie + // NOTE: we are not checking non-ascii characters and we should + private static readonly Regex _CookieRegex = new Regex(@"([^\(\)<>@,;:\\""/\[\]\?=\{\}\s]+)=([^,;\\""\s]+)"); + private static readonly char[] InvalidKeyChars = { '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '=', '{', '}', ' ', '\t', '\n' }; + private static readonly char[] InvalidValueChars = { '"', ',', ';', '\\', ' ', '\t', '\n' }; + + public static Dictionary CookieHeaderToDictionary(string cookieHeader) + { + var cookieDictionary = new Dictionary(); + if (cookieHeader == null) + { + return cookieDictionary; + } + + var matches = _CookieRegex.Match(cookieHeader); + while (matches.Success) + { + if (matches.Groups.Count > 2) + { + cookieDictionary[matches.Groups[1].Value] = matches.Groups[2].Value; + } + + matches = matches.NextMatch(); + } + + return cookieDictionary; + } + + public static string CookieDictionaryToHeader(Dictionary cookieDictionary) + { + if (cookieDictionary == null) + { + return ""; + } + + foreach (var kv in cookieDictionary) + { + if (kv.Key.IndexOfAny(InvalidKeyChars) > -1 || kv.Value.IndexOfAny(InvalidValueChars) > -1) + { + throw new FormatException($"The cookie '{kv.Key}={kv.Value}' is malformed."); + } + } + + return string.Join("; ", cookieDictionary.Select(kv => kv.Key + "=" + kv.Value)); + } + } +} diff --git a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs index 84b82a676..d94b330f1 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs @@ -143,7 +143,7 @@ namespace NzbDrone.Common.Http.Dispatchers } } - return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode); + return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), httpWebResponse.Cookies, data, httpWebResponse.StatusCode); } public void DownloadFile(string url, string fileName) diff --git a/src/NzbDrone.Common/Http/HttpResponse.cs b/src/NzbDrone.Common/Http/HttpResponse.cs index e23a1cc9a..85ebe9ffc 100644 --- a/src/NzbDrone.Common/Http/HttpResponse.cs +++ b/src/NzbDrone.Common/Http/HttpResponse.cs @@ -11,18 +11,20 @@ namespace NzbDrone.Common.Http { private static readonly Regex RegexSetCookie = new Regex("^(.*?)=(.*?)(?:;|$)", RegexOptions.Compiled); - public HttpResponse(HttpRequest request, HttpHeader headers, byte[] binaryData, HttpStatusCode statusCode = HttpStatusCode.OK) + public HttpResponse(HttpRequest request, HttpHeader headers, CookieCollection cookies, byte[] binaryData, HttpStatusCode statusCode = HttpStatusCode.OK) { Request = request; Headers = headers; + Cookies = cookies; ResponseData = binaryData; StatusCode = statusCode; } - public HttpResponse(HttpRequest request, HttpHeader headers, string content, HttpStatusCode statusCode = HttpStatusCode.OK) + public HttpResponse(HttpRequest request, HttpHeader headers, CookieCollection cookies, string content, HttpStatusCode statusCode = HttpStatusCode.OK) { Request = request; Headers = headers; + Cookies = cookies; ResponseData = Headers.GetEncodingFromContentType().GetBytes(content); _content = content; StatusCode = statusCode; @@ -30,6 +32,7 @@ namespace NzbDrone.Common.Http public HttpRequest Request { get; private set; } public HttpHeader Headers { get; private set; } + public CookieCollection Cookies { get; private set; } public HttpStatusCode StatusCode { get; private set; } public byte[] ResponseData { get; private set; } @@ -65,14 +68,9 @@ namespace NzbDrone.Common.Http { var result = new Dictionary(); - var setCookieHeaders = GetCookieHeaders(); - foreach (var cookie in setCookieHeaders) + foreach (Cookie cookie in Cookies) { - var match = RegexSetCookie.Match(cookie); - if (match.Success) - { - result[match.Groups[1].Value] = match.Groups[2].Value; - } + result[cookie.Name] = cookie.Value; } return result; @@ -95,7 +93,7 @@ namespace NzbDrone.Common.Http where T : new() { public HttpResponse(HttpResponse response) - : base(response.Request, response.Headers, response.ResponseData, response.StatusCode) + : base(response.Request, response.Headers, response.Cookies, response.ResponseData, response.StatusCode) { Resource = Json.Deserialize(response.Content); } diff --git a/src/NzbDrone.Core.Test/HealthCheck/Checks/SystemTimeCheckFixture.cs b/src/NzbDrone.Core.Test/HealthCheck/Checks/SystemTimeCheckFixture.cs index 74cf3a0b4..bb49eb0f6 100644 --- a/src/NzbDrone.Core.Test/HealthCheck/Checks/SystemTimeCheckFixture.cs +++ b/src/NzbDrone.Core.Test/HealthCheck/Checks/SystemTimeCheckFixture.cs @@ -1,4 +1,5 @@ using System; +using System.Net; using System.Text; using Moq; using NUnit.Framework; @@ -31,7 +32,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks Mocker.GetMock() .Setup(s => s.Execute(It.IsAny())) - .Returns(r => new HttpResponse(r, new HttpHeader(), Encoding.ASCII.GetBytes(json))); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), Encoding.ASCII.GetBytes(json))); } [Test] diff --git a/src/NzbDrone.Core.Test/IndexerTests/BasicRssParserFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/BasicRssParserFixture.cs index 82120f888..de839834b 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/BasicRssParserFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/BasicRssParserFixture.cs @@ -1,4 +1,5 @@ -using System.Linq; +using System.Linq; +using System.Net; using System.Text; using FluentAssertions; using NUnit.Framework; @@ -40,7 +41,7 @@ namespace NzbDrone.Core.Test.IndexerTests private IndexerResponse CreateResponse(string url, string content) { var httpRequest = new HttpRequest(url); - var httpResponse = new HttpResponse(httpRequest, new HttpHeader(), Encoding.UTF8.GetBytes(content)); + var httpResponse = new HttpResponse(httpRequest, new HttpHeader(), new CookieCollection(), Encoding.UTF8.GetBytes(content)); return new IndexerResponse(new IndexerRequest(httpRequest), httpResponse); } diff --git a/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs index ca0e0f182..0b8c8a407 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Net; using FluentAssertions; using Moq; using NUnit.Framework; @@ -32,7 +33,7 @@ namespace NzbDrone.Core.Test.IndexerTests.FileListTests Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), recentFeed)); var releases = Subject.Fetch(new MovieSearchCriteria { Categories = new int[] { 2000 } }).Releases; diff --git a/src/NzbDrone.Core.Test/IndexerTests/HDBitsTests/HDBitsFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/HDBitsTests/HDBitsFixture.cs index 9fc936814..6224c35c8 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/HDBitsTests/HDBitsFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/HDBitsTests/HDBitsFixture.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Net; using System.Text; using FluentAssertions; using Moq; @@ -44,7 +45,7 @@ namespace NzbDrone.Core.Test.IndexerTests.HDBitsTests Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.POST))) - .Returns(r => new HttpResponse(r, new HttpHeader(), responseJson)); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), responseJson)); var torrents = Subject.Fetch(_movieSearchCriteria).Releases; @@ -73,7 +74,7 @@ namespace NzbDrone.Core.Test.IndexerTests.HDBitsTests Mocker.GetMock() .Setup(v => v.Execute(It.IsAny())) - .Returns(r => new HttpResponse(r, new HttpHeader(), Encoding.UTF8.GetBytes(responseJson))); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), Encoding.UTF8.GetBytes(responseJson))); var torrents = Subject.Fetch(_movieSearchCriteria).Releases; diff --git a/src/NzbDrone.Core.Test/IndexerTests/IPTorrentsTests/IPTorrentsFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/IPTorrentsTests/IPTorrentsFixture.cs index fbd9afbda..84a758eae 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/IPTorrentsTests/IPTorrentsFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/IPTorrentsTests/IPTorrentsFixture.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Net; using FluentAssertions; using Moq; using NUnit.Framework; @@ -90,7 +91,7 @@ namespace NzbDrone.Core.Test.IndexerTests.IPTorrentsTests Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), recentFeed)); var releases = Subject.Fetch(new MovieSearchCriteria { Categories = new int[] { 2000 } }).Releases; diff --git a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabCapabilitiesProviderFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabCapabilitiesProviderFixture.cs index bd9d316bb..fce49a5a4 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabCapabilitiesProviderFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabCapabilitiesProviderFixture.cs @@ -1,4 +1,5 @@ using System; +using System.Net; using System.Xml; using FluentAssertions; using Moq; @@ -30,7 +31,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests { Mocker.GetMock() .Setup(o => o.Get(It.IsAny())) - .Returns(r => new HttpResponse(r, new HttpHeader(), caps)); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), caps)); } [Test] diff --git a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs index bb9f2145b..94db26fc4 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Net; using FluentAssertions; using Moq; using NUnit.Framework; @@ -42,7 +43,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), recentFeed)); var releases = Subject.Fetch(new MovieSearchCriteria { Categories = new int[] { 2000 }, Limit = 100, Offset = 0 }).Releases; diff --git a/src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs index fabb4c292..9ef21a3ac 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Net; using FluentAssertions; using Moq; using NUnit.Framework; @@ -32,7 +33,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NyaaTests Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), recentFeed)); var releases = Subject.Fetch(new MovieSearchCriteria()).Releases; diff --git a/src/NzbDrone.Core.Test/IndexerTests/PTPTests/PTPFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/PTPTests/PTPFixture.cs index f60a5bf2f..9b8d58e77 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/PTPTests/PTPFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/PTPTests/PTPFixture.cs @@ -1,4 +1,5 @@ using System.Linq; +using System.Net; using FluentAssertions; using Moq; using NUnit.Framework; @@ -36,11 +37,11 @@ namespace NzbDrone.Core.Test.IndexerTests.PTPTests Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.POST))) - .Returns(r => new HttpResponse(r, new HttpHeader(), authStream.ToString())); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), authStream.ToString())); Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader { ContentType = HttpAccept.Json.Value }, responseJson)); + .Returns(r => new HttpResponse(r, new HttpHeader { ContentType = HttpAccept.Json.Value }, new CookieCollection(), responseJson)); var torrents = Subject.Fetch(new MovieSearchCriteria()).Releases; diff --git a/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs index 3bcd81ae9..8348572a8 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Net; using FluentAssertions; using Moq; using NUnit.Framework; @@ -37,7 +38,7 @@ namespace NzbDrone.Core.Test.IndexerTests.RarbgTests Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), recentFeed)); var releases = Subject.Fetch(new MovieSearchCriteria { Categories = new int[] { 2000 } }).Releases; @@ -64,7 +65,7 @@ namespace NzbDrone.Core.Test.IndexerTests.RarbgTests { Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), "{ error_code: 20, error: \"some message\" }")); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), "{ error_code: 20, error: \"some message\" }")); var releases = Subject.Fetch(new MovieSearchCriteria { Categories = new int[] { 2000 } }).Releases; @@ -76,7 +77,7 @@ namespace NzbDrone.Core.Test.IndexerTests.RarbgTests { Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), "{ error_code: 25, error: \"some message\" }")); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), "{ error_code: 25, error: \"some message\" }")); var releases = Subject.Fetch(new MovieSearchCriteria { Categories = new int[] { 2000 } }).Releases; diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs index e4baf3756..c237a2898 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Net; using FluentAssertions; using Moq; using NUnit.Framework; @@ -37,7 +38,7 @@ namespace NzbDrone.Core.Test.IndexerTests.TorrentRssIndexerTests Mocker.GetMock() .Setup(o => o.Execute(It.IsAny())) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), recentFeed)); } [Test] diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs index f8e0d0b3c..79fb7fdfb 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs @@ -1,3 +1,4 @@ +using System.Net; using FluentAssertions; using Moq; using NUnit.Framework; @@ -26,7 +27,7 @@ namespace NzbDrone.Core.Test.IndexerTests.TorrentRssIndexerTests Mocker.GetMock() .Setup(o => o.Execute(It.IsAny())) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), recentFeed)); } [Test] diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs index 7c3747b93..9e1fe4b31 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Net; using FluentAssertions; using Moq; using NUnit.Framework; @@ -43,7 +44,7 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), recentFeed)); var releases = Subject.Fetch(new MovieSearchCriteria()).Releases; @@ -72,7 +73,7 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), recentFeed)); var releases = Subject.Fetch(new MovieSearchCriteria()).Releases; @@ -102,7 +103,7 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + .Returns(r => new HttpResponse(r, new HttpHeader(), new CookieCollection(), recentFeed)); var releases = Subject.Fetch(new MovieSearchCriteria()).Releases; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs index 6c0cd5fe9..6ab922804 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs @@ -24,7 +24,10 @@ namespace NzbDrone.Core.Indexers.Cardigann { return new CardigannRequestGenerator(_definitionService.GetDefinition(Settings.DefinitionFile), Settings, - _logger); + _logger) + { + HttpClient = _httpClient + }; } public override IParseIndexerResponse GetParser() diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs index fed8b3002..166add8f9 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs @@ -222,7 +222,7 @@ namespace NzbDrone.Core.Indexers.Cardigann _logger.Debug($"{name} got value {value.ToJson()}"); - if (setting.Type == "text") + if (setting.Type == "text" || setting.Type == "password") { variables[name] = value; } @@ -232,13 +232,13 @@ namespace NzbDrone.Core.Indexers.Cardigann } else if (setting.Type == "select") { - _logger.Debug($"setting options: {setting.Options.ToJson()}"); + _logger.Debug($"Setting options: {setting.Options.ToJson()}"); var sorted = setting.Options.OrderBy(x => x.Key).ToList(); var selected = sorted[(int)(long)value]; - _logger.Debug($"selected option: {selected.ToJson()}"); + _logger.Debug($"Selected option: {selected.ToJson()}"); - variables[name] = selected.Value; + variables[name] = selected.Key; } else if (setting.Type == "info") { @@ -249,7 +249,7 @@ namespace NzbDrone.Core.Indexers.Cardigann throw new NotSupportedException(); } - _logger.Debug($"Setting {setting.Name} to {variables[name]}"); + _logger.Debug($"Setting {setting.Name} to {(setting.Type == "password" ? "Redacted" : variables[name])}"); } return variables; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannConfigException.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannConfigException.cs new file mode 100644 index 000000000..e8057e991 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannConfigException.cs @@ -0,0 +1,24 @@ +using NzbDrone.Common.Exceptions; +using NzbDrone.Core.Indexers.Cardigann; + +namespace NzbDrone.Core.Indexers.Definitions.Cardigann +{ + public class CardigannConfigException : NzbDroneException + { + private readonly CardigannDefinition _configuration; + + public CardigannConfigException(CardigannDefinition config, string message, params object[] args) + : base(message, args) + { + _configuration = config; + } + + public CardigannConfigException(CardigannDefinition config, string message) + : base(message) + { + _configuration = config; + } + + public CardigannDefinition Configuration => _configuration; + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs index 6257e444f..1c0bac5bd 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs @@ -333,8 +333,8 @@ namespace NzbDrone.Core.Indexers.Cardigann continue; } - _logger.Error("Error while parsing field={0}, selector={1}, value={2}: {3}", field.Key, field.Value.Selector, value == null ? "" : value, ex.Message); - throw; + //Parser errors usually happen on every result and are costly to performance, trace only + _logger.Trace("Error while parsing field={0}, selector={1}, value={2}: {3}", field.Key, field.Value.Selector, value == null ? "" : value, ex.Message); } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs index 344ba0548..0d81c4783 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs @@ -1,15 +1,25 @@ using System; using System.Collections.Generic; +using System.Collections.Specialized; using System.Linq; using System.Net; +using AngleSharp.Html.Dom; +using AngleSharp.Html.Parser; +using Newtonsoft.Json.Linq; using NLog; using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers.Definitions.Cardigann; using NzbDrone.Core.IndexerSearch.Definitions; namespace NzbDrone.Core.Indexers.Cardigann { public class CardigannRequestGenerator : CardigannBase, IIndexerRequestGenerator { + public IHttpClient HttpClient { get; set; } + public IDictionary Cookies { get; set; } + protected HttpResponse landingResult; + protected IHtmlDocument landingResultDocument; + public CardigannRequestGenerator(CardigannDefinition definition, CardigannSettings settings, Logger logger) @@ -144,8 +154,617 @@ namespace NzbDrone.Core.Indexers.Cardigann return variables; } + private void Authenticate() + { + var login = _definition.Login; + + if (login == null || TestLogin()) + { + return; + } + + if (login.Method == "post") + { + var pairs = new Dictionary(); + + foreach (var input in login.Inputs) + { + var value = ApplyGoTemplateText(input.Value); + pairs.Add(input.Key, value); + } + + var loginUrl = ResolvePath(login.Path).ToString(); + + CookiesUpdater(null, null); + + var requestBuilder = new HttpRequestBuilder(loginUrl) + { + LogResponseContent = true, + Method = HttpMethod.POST, + AllowAutoRedirect = true, + SuppressHttpError = true + }; + + requestBuilder.Headers.Add("Referer", SiteLink); + + var response = HttpClient.Execute(requestBuilder.Build()); + + Cookies = response.GetCookies(); + + CheckForError(response, login.Error); + + CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30)); + } + else if (login.Method == "form") + { + var loginUrl = ResolvePath(login.Path).ToString(); + + var queryCollection = new NameValueCollection(); + var pairs = new Dictionary(); + + var formSelector = login.Form; + if (formSelector == null) + { + formSelector = "form"; + } + + // landingResultDocument might not be initiated if the login is caused by a relogin during a query + if (landingResultDocument == null) + { + GetConfigurationForSetup(true); + } + + var form = landingResultDocument.QuerySelector(formSelector); + if (form == null) + { + throw new CardigannConfigException(_definition, string.Format("Login failed: No form found on {0} using form selector {1}", loginUrl, formSelector)); + } + + var inputs = form.QuerySelectorAll("input"); + if (inputs == null) + { + throw new CardigannConfigException(_definition, string.Format("Login failed: No inputs found on {0} using form selector {1}", loginUrl, formSelector)); + } + + var submitUrlstr = form.GetAttribute("action"); + if (login.Submitpath != null) + { + submitUrlstr = login.Submitpath; + } + + foreach (var input in inputs) + { + var name = input.GetAttribute("name"); + if (name == null) + { + continue; + } + + var value = input.GetAttribute("value"); + if (value == null) + { + value = ""; + } + + pairs[name] = value; + } + + foreach (var input in login.Inputs) + { + var value = ApplyGoTemplateText(input.Value); + var inputKey = input.Key; + if (login.Selectors) + { + var inputElement = landingResultDocument.QuerySelector(input.Key); + if (inputElement == null) + { + throw new CardigannConfigException(_definition, string.Format("Login failed: No input found using selector {0}", input.Key)); + } + + inputKey = inputElement.GetAttribute("name"); + } + + pairs[inputKey] = value; + } + + // selector inputs + if (login.Selectorinputs != null) + { + foreach (var selectorinput in login.Selectorinputs) + { + string value = null; + try + { + value = HandleSelector(selectorinput.Value, landingResultDocument.FirstElementChild); + pairs[selectorinput.Key] = value; + } + catch (Exception ex) + { + throw new Exception(string.Format("Error while parsing selector input={0}, selector={1}, value={2}: {3}", selectorinput.Key, selectorinput.Value.Selector, value, ex.Message)); + } + } + } + + // getselector inputs + if (login.Getselectorinputs != null) + { + foreach (var selectorinput in login.Getselectorinputs) + { + string value = null; + try + { + value = HandleSelector(selectorinput.Value, landingResultDocument.FirstElementChild); + queryCollection[selectorinput.Key] = value; + } + catch (Exception ex) + { + throw new Exception(string.Format("Error while parsing get selector input={0}, selector={1}, value={2}: {3}", selectorinput.Key, selectorinput.Value.Selector, value, ex.Message)); + } + } + } + + if (queryCollection.Count > 0) + { + submitUrlstr += "?" + queryCollection.GetQueryString(); + } + + var submitUrl = ResolvePath(submitUrlstr, new Uri(loginUrl)); + + // automatically solve simpleCaptchas, if used + var simpleCaptchaPresent = landingResultDocument.QuerySelector("script[src*=\"simpleCaptcha\"]"); + if (simpleCaptchaPresent != null) + { + var captchaUrl = ResolvePath("simpleCaptcha.php?numImages=1"); + + var requestBuilder = new HttpRequestBuilder(captchaUrl.ToString()) + { + LogResponseContent = true, + Method = HttpMethod.GET + }; + + requestBuilder.Headers.Add("Referer", loginUrl); + + var simpleCaptchaResult = HttpClient.Execute(requestBuilder.Build()); + + var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.Content); + var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString(); + pairs["captchaSelection"] = captchaSelection; + pairs["submitme"] = "X"; + } + + if (login.Captcha != null) + { + var captcha = login.Captcha; + if (captcha.Type == "image") + { + _settings.ExtraFieldData.TryGetValue("CaptchaText", out var captchaText); + if (captchaText != null) + { + var input = captcha.Input; + if (login.Selectors) + { + var inputElement = landingResultDocument.QuerySelector(captcha.Input); + if (inputElement == null) + { + throw new CardigannConfigException(_definition, string.Format("Login failed: No captcha input found using {0}", captcha.Input)); + } + + input = inputElement.GetAttribute("name"); + } + + pairs[input] = (string)captchaText; + } + } + + if (captcha.Type == "text") + { + _settings.ExtraFieldData.TryGetValue("CaptchaAnswer", out var captchaAnswer); + if (captchaAnswer != null) + { + var input = captcha.Input; + if (login.Selectors) + { + var inputElement = landingResultDocument.QuerySelector(captcha.Input); + if (inputElement == null) + { + throw new CardigannConfigException(_definition, string.Format("Login failed: No captcha input found using {0}", captcha.Input)); + } + + input = inputElement.GetAttribute("name"); + } + + pairs[input] = (string)captchaAnswer; + } + } + } + + // clear landingResults/Document, otherwise we might use an old version for a new relogin (if GetConfigurationForSetup() wasn't called before) + landingResult = null; + landingResultDocument = null; + + HttpResponse loginResult = null; + var enctype = form.GetAttribute("enctype"); + if (enctype == "multipart/form-data") + { + var headers = new Dictionary(); + var boundary = "---------------------------" + DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1)).TotalSeconds.ToString().Replace(".", ""); + var bodyParts = new List(); + + foreach (var pair in pairs) + { + var part = "--" + boundary + "\r\n" + + "Content-Disposition: form-data; name=\"" + pair.Key + "\"\r\n" + + "\r\n" + + pair.Value; + bodyParts.Add(part); + } + + bodyParts.Add("--" + boundary + "--"); + + headers.Add("Content-Type", "multipart/form-data; boundary=" + boundary); + var body = string.Join("\r\n", bodyParts); + + var requestBuilder = new HttpRequestBuilder(submitUrl.ToString()) + { + LogResponseContent = true, + Method = HttpMethod.POST, + AllowAutoRedirect = true + }; + + requestBuilder.Headers.Add("Referer", SiteLink); + + requestBuilder.SetCookies(Cookies); + + foreach (var pair in pairs) + { + requestBuilder.AddFormParameter(pair.Key, pair.Value); + } + + foreach (var header in headers) + { + requestBuilder.SetHeader(header.Key, header.Value); + } + + var request = requestBuilder.Build(); + request.SetContent(body); + + loginResult = HttpClient.Execute(request); + } + else + { + var requestBuilder = new HttpRequestBuilder(submitUrl.ToString()) + { + LogResponseContent = true, + Method = HttpMethod.POST, + AllowAutoRedirect = true, + SuppressHttpError = true + }; + + requestBuilder.SetCookies(Cookies); + requestBuilder.Headers.Add("Referer", loginUrl); + + foreach (var pair in pairs) + { + requestBuilder.AddFormParameter(pair.Key, pair.Value); + } + + loginResult = HttpClient.Execute(requestBuilder.Build()); + } + + Cookies = loginResult.GetCookies(); + CheckForError(loginResult, login.Error); + CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30)); + } + else if (login.Method == "cookie") + { + CookiesUpdater(null, null); + _settings.ExtraFieldData.TryGetValue("cookie", out var cookies); + CookiesUpdater(CookieUtil.CookieHeaderToDictionary((string)cookies), DateTime.Now + TimeSpan.FromDays(30)); + } + else if (login.Method == "get") + { + var queryCollection = new NameValueCollection(); + foreach (var input in login.Inputs) + { + var value = ApplyGoTemplateText(input.Value); + queryCollection.Add(input.Key, value); + } + + var loginUrl = ResolvePath(login.Path + "?" + queryCollection.GetQueryString()).ToString(); + + CookiesUpdater(null, null); + + var requestBuilder = new HttpRequestBuilder(loginUrl) + { + LogResponseContent = true, + Method = HttpMethod.GET, + SuppressHttpError = true + }; + + requestBuilder.Headers.Add("Referer", SiteLink); + + var response = HttpClient.Execute(requestBuilder.Build()); + + Cookies = response.GetCookies(); + + CheckForError(response, login.Error); + + CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30)); + } + else if (login.Method == "oneurl") + { + var oneUrl = ApplyGoTemplateText(login.Inputs["oneurl"]); + var loginUrl = ResolvePath(login.Path + oneUrl).ToString(); + + CookiesUpdater(null, null); + + var requestBuilder = new HttpRequestBuilder(loginUrl) + { + LogResponseContent = true, + Method = HttpMethod.GET, + SuppressHttpError = true + }; + + requestBuilder.Headers.Add("Referer", SiteLink); + + var response = HttpClient.Execute(requestBuilder.Build()); + + Cookies = response.GetCookies(); + + CheckForError(response, login.Error); + + CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30)); + } + else + { + throw new NotImplementedException("Login method " + login.Method + " not implemented"); + } + } + + protected bool CheckForError(HttpResponse loginResult, IList errorBlocks) + { + if (loginResult.StatusCode == HttpStatusCode.Unauthorized) + { + throw new HttpException(loginResult); + } + + if (errorBlocks == null) + { + return true; + } + + var resultParser = new HtmlParser(); + var resultDocument = resultParser.ParseDocument(loginResult.Content); + foreach (var error in errorBlocks) + { + var selection = resultDocument.QuerySelector(error.Selector); + if (selection != null) + { + var errorMessage = selection.TextContent; + if (error.Message != null) + { + errorMessage = HandleSelector(error.Message, resultDocument.FirstElementChild); + } + + throw new CardigannConfigException(_definition, string.Format("Error: {0}", errorMessage.Trim())); + } + } + + return true; + } + + public void GetConfigurationForSetup(bool automaticlogin) + { + var login = _definition.Login; + + if (login == null || login.Method != "form") + { + return; + } + + var loginUrl = ResolvePath(login.Path); + + Cookies = null; + + if (login.Cookies != null) + { + Cookies = CookieUtil.CookieHeaderToDictionary(string.Join("; ", login.Cookies)); + } + + var requestBuilder = new HttpRequestBuilder(loginUrl.AbsoluteUri) + { + LogResponseContent = true, + Method = HttpMethod.GET + }; + + requestBuilder.Headers.Add("Referer", SiteLink); + + if (Cookies != null) + { + requestBuilder.SetCookies(Cookies); + } + + landingResult = HttpClient.Execute(requestBuilder.Build()); + + Cookies = landingResult.GetCookies(); + + // Some sites have a temporary redirect before the login page, we need to process it. + //if (_definition.Followredirect) + //{ + // await FollowIfRedirect(landingResult, loginUrl.AbsoluteUri, overrideCookies: landingResult.Cookies, accumulateCookies: true); + //} + var hasCaptcha = false; + var htmlParser = new HtmlParser(); + landingResultDocument = htmlParser.ParseDocument(landingResult.Content); + + if (login.Captcha != null) + { + var captcha = login.Captcha; + if (captcha.Type == "image") + { + var captchaElement = landingResultDocument.QuerySelector(captcha.Selector); + if (captchaElement != null) + { + hasCaptcha = true; + + //TODO Bubble this to UI when we get a captcha so that user can action it + //Jackett does this by inserting image or question into the extrasettings which then show up in the add modal + // + //var captchaUrl = ResolvePath(captchaElement.GetAttribute("src"), loginUrl); + //var captchaImageData = RequestWithCookiesAsync(captchaUrl.ToString(), landingResult.GetCookies, referer: loginUrl.AbsoluteUri); + // var CaptchaImage = new ImageItem { Name = "Captcha Image" }; + //var CaptchaText = new StringItem { Name = "Captcha Text" }; + //CaptchaImage.Value = captchaImageData.ContentBytes; + //configData.AddDynamic("CaptchaImage", CaptchaImage); + //configData.AddDynamic("CaptchaText", CaptchaText); + } + else + { + _logger.Debug(string.Format("CardigannIndexer ({0}): No captcha image found", _definition.Id)); + } + } + else if (captcha.Type == "text") + { + var captchaElement = landingResultDocument.QuerySelector(captcha.Selector); + if (captchaElement != null) + { + hasCaptcha = true; + + //var captchaChallenge = new DisplayItem(captchaElement.TextContent) { Name = "Captcha Challenge" }; + //var captchaAnswer = new StringItem { Name = "Captcha Answer" }; + + //configData.AddDynamic("CaptchaChallenge", captchaChallenge); + //configData.AddDynamic("CaptchaAnswer", captchaAnswer); + } + else + { + _logger.Debug(string.Format("CardigannIndexer ({0}): No captcha image found", _definition.Id)); + } + } + else + { + throw new NotImplementedException(string.Format("Captcha type \"{0}\" is not implemented", captcha.Type)); + } + } + + if (hasCaptcha && automaticlogin) + { + _logger.Error(string.Format("CardigannIndexer ({0}): Found captcha during automatic login, aborting", _definition.Id)); + return; + } + + return; + } + + protected bool TestLogin() + { + var login = _definition.Login; + + if (login == null || login.Test == null) + { + return false; + } + + // test if login was successful + var loginTestUrl = ResolvePath(login.Test.Path).ToString(); + + // var headers = ParseCustomHeaders(_definition.Search?.Headers, GetBaseTemplateVariables()); + var requestBuilder = new HttpRequestBuilder(loginTestUrl) + { + LogResponseContent = true, + Method = HttpMethod.GET, + SuppressHttpError = true + }; + + if (Cookies != null) + { + requestBuilder.SetCookies(Cookies); + } + + var testResult = HttpClient.Execute(requestBuilder.Build()); + + if (testResult.HasHttpRedirect) + { + var errormessage = "Login Failed, got redirected."; + var domainHint = GetRedirectDomainHint(testResult); + if (domainHint != null) + { + errormessage += " Try changing the indexer URL to " + domainHint + "."; + } + + _logger.Debug(errormessage); + return false; + } + + if (login.Test.Selector != null) + { + var testResultParser = new HtmlParser(); + var testResultDocument = testResultParser.ParseDocument(testResult.Content); + var selection = testResultDocument.QuerySelectorAll(login.Test.Selector); + if (selection.Length == 0) + { + _logger.Debug(string.Format("Login failed: Selector \"{0}\" didn't match", login.Test.Selector)); + return false; + } + } + + return true; + } + + protected string GetRedirectDomainHint(string requestUrl, string redirectUrl) + { + if (requestUrl.StartsWith(SiteLink) && !redirectUrl.StartsWith(SiteLink)) + { + var uri = new Uri(redirectUrl); + return uri.Scheme + "://" + uri.Host + "/"; + } + + return null; + } + + protected string GetRedirectDomainHint(HttpResponse result) => GetRedirectDomainHint(result.Request.Url.ToString(), result.Headers.GetSingleValue("Location")); + + protected bool CheckIfLoginIsNeeded(HttpResponse response, IHtmlDocument document) + { + if (response.HasHttpRedirect) + { + var domainHint = GetRedirectDomainHint(response); + if (domainHint != null) + { + var errormessage = "Got redirected to another domain. Try changing the indexer URL to " + domainHint + "."; + + throw new Exception(errormessage); + } + + return true; + } + + if (_definition.Login == null || _definition.Login.Test == null) + { + return false; + } + + if (_definition.Login.Test.Selector != null) + { + var selection = document.QuerySelectorAll(_definition.Login.Test.Selector); + if (selection.Length == 0) + { + return true; + } + } + + return false; + } + private IEnumerable GetRequest(Dictionary variables) { + Cookies = GetCookies(); + + if (Cookies == null || !Cookies.Any()) + { + Authenticate(); + } + var search = _definition.Search; var mappedCategories = MapTorznabCapsToTrackers((int[])variables[".Query.Categories"]); @@ -270,6 +889,14 @@ namespace NzbDrone.Core.Indexers.Cardigann } } + if (Cookies != null) + { + foreach (var cookie in Cookies) + { + request.HttpRequest.Cookies.Add(cookie.Key, cookie.Value); + } + } + request.HttpRequest.Method = method; yield return request; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/DateTimeUtil.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/DateTimeUtil.cs index 562375713..66ea2b63c 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/DateTimeUtil.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/DateTimeUtil.cs @@ -102,13 +102,17 @@ namespace NzbDrone.Core.Indexers.Cardigann // http://www.codeproject.com/Articles/33298/C-Date-Time-Parser public static DateTime FromFuzzyTime(string str, string format = null) { - /*var dtFormat = format == "UK" ? - DateTimeRoutines.DateTimeRoutines.DateTimeFormat.UkDate : - DateTimeRoutines.DateTimeRoutines.DateTimeFormat.UsaDate; + //var dtFormat = format == "UK" ? + // DateTimeRoutines.DateTimeRoutines.DateTimeFormat.UkDate : + // DateTimeRoutines.DateTimeRoutines.DateTimeFormat.UsaDate; - if (DateTimeRoutines.DateTimeRoutines.TryParseDateOrTime( - str, dtFormat, out DateTimeRoutines.DateTimeRoutines.ParsedDateTime dt)) - return dt.DateTime;*/ + //if (DateTimeRoutines.DateTimeRoutines.TryParseDateOrTime( + // str, dtFormat, out DateTimeRoutines.DateTimeRoutines.ParsedDateTime dt)) + // return dt.DateTime; + if (DateTime.TryParse(str, out var dateTimeParsed)) + { + return dateTimeParsed; + } throw new Exception("FromFuzzyTime parsing failed"); } diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 1d6ffc30e..d8a6a51e3 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -12,7 +12,6 @@ using NzbDrone.Core.Http.CloudFlare; using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Parser.Model; -using NzbDrone.Core.ThingiProvider; namespace NzbDrone.Core.Indexers { @@ -43,11 +42,12 @@ namespace NzbDrone.Core.Indexers public override IndexerPageableQueryResult Fetch(MovieSearchCriteria searchCriteria) { + //TODO: Re-Enable when All Indexer Caps are fixed and tests don't fail //if (!SupportsSearch) //{ - // return new List(); + // return new IndexerPageableQueryResult(); //} - return FetchReleases(g => g.GetSearchRequests(searchCriteria)); + return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria)); } public override IndexerPageableQueryResult Fetch(MusicSearchCriteria searchCriteria) @@ -57,7 +57,7 @@ namespace NzbDrone.Core.Indexers return new IndexerPageableQueryResult(); } - return FetchReleases(g => g.GetSearchRequests(searchCriteria)); + return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria)); } public override IndexerPageableQueryResult Fetch(TvSearchCriteria searchCriteria) @@ -67,7 +67,7 @@ namespace NzbDrone.Core.Indexers return new IndexerPageableQueryResult(); } - return FetchReleases(g => g.GetSearchRequests(searchCriteria)); + return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria)); } public override IndexerPageableQueryResult Fetch(BookSearchCriteria searchCriteria) @@ -77,7 +77,7 @@ namespace NzbDrone.Core.Indexers return new IndexerPageableQueryResult(); } - return FetchReleases(g => g.GetSearchRequests(searchCriteria)); + return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria)); } public override IndexerPageableQueryResult Fetch(BasicSearchCriteria searchCriteria) @@ -87,13 +87,11 @@ namespace NzbDrone.Core.Indexers return new IndexerPageableQueryResult(); } - return FetchReleases(g => g.GetSearchRequests(searchCriteria)); + return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria)); } - protected IndexerPageableRequestChain GetRequestChain(SearchCriteriaBase searchCriteria = null) + protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator) { - var generator = GetRequestGenerator(); - //A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page. generator.GetCookies = () => { @@ -107,14 +105,12 @@ namespace NzbDrone.Core.Indexers return cookies; }; - var requests = generator.GetSearchRequests(searchCriteria as MovieSearchCriteria); - generator.CookiesUpdater = (cookies, expiration) => { _indexerStatusService.UpdateCookies(Definition.Id, cookies, expiration); }; - return requests; + return generator; } protected virtual IndexerPageableQueryResult FetchReleases(Func pageableRequestChainSelector, bool isRecent = false) @@ -373,7 +369,11 @@ namespace NzbDrone.Core.Indexers { _indexerStatusService.UpdateCookies(Definition.Id, cookies, expiration); }; + var generator = GetRequestGenerator(); + + generator = SetCookieFunctions(generator); + var firstRequest = generator.GetSearchRequests(new BasicSearchCriteria { SearchType = "search" }).GetAllTiers().FirstOrDefault()?.FirstOrDefault(); if (firstRequest == null) diff --git a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs index 9254a5b3e..cc18c4c97 100644 --- a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs +++ b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs @@ -32,12 +32,12 @@ namespace NzbDrone.Core.Indexers public IDictionary GetIndexerCookies(int indexerId) { - return GetProviderStatus(indexerId).Cookies; + return GetProviderStatus(indexerId)?.Cookies ?? null; } public DateTime GetIndexerCookiesExpirationDate(int indexerId) { - return GetProviderStatus(indexerId).CookiesExpirationDate ?? DateTime.Now + TimeSpan.FromDays(12); + return GetProviderStatus(indexerId)?.CookiesExpirationDate ?? DateTime.Now + TimeSpan.FromDays(12); } public void UpdateRssSyncStatus(int indexerId, ReleaseInfo releaseInfo) @@ -54,12 +54,15 @@ namespace NzbDrone.Core.Indexers public void UpdateCookies(int indexerId, IDictionary cookies, DateTime? expiration) { - lock (_syncRoot) + if (indexerId > 0) { - var status = GetProviderStatus(indexerId); - status.Cookies = cookies; - status.CookiesExpirationDate = expiration; - _providerStatusRepository.Upsert(status); + lock (_syncRoot) + { + var status = GetProviderStatus(indexerId); + status.Cookies = cookies; + status.CookiesExpirationDate = expiration; + _providerStatusRepository.Upsert(status); + } } } } diff --git a/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs b/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs index 189552847..beae8724b 100644 --- a/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs +++ b/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs @@ -35,20 +35,16 @@ namespace Prowlarr.Api.V1.Indexers if (definition.Implementation == typeof(Cardigann).Name) { - Console.WriteLine("mapping cardigann def"); - var extraFields = definition.ExtraFields?.Select((x, i) => MapField(x, i)).ToList() ?? new List(); resource.Fields.AddRange(extraFields); var settings = (CardigannSettings)definition.Settings; - Console.WriteLine($"Got {settings.ExtraFieldData.Count} fields"); foreach (var setting in settings.ExtraFieldData) { var field = extraFields.FirstOrDefault(x => x.Name == setting.Key); if (field != null) { - Console.WriteLine($"setting {setting.Key} to {setting.Value}"); field.Value = setting.Value; } } @@ -79,8 +75,6 @@ namespace Prowlarr.Api.V1.Indexers if (resource.Implementation == typeof(Cardigann).Name) { - Console.WriteLine("mapping cardigann resource"); - var standardFields = base.ToResource(definition).Fields.Select(x => x.Name).ToList(); var settings = (CardigannSettings)definition.Settings; @@ -105,7 +99,6 @@ namespace Prowlarr.Api.V1.Indexers private Field MapField(SettingsField fieldAttribute, int order) { - Console.WriteLine($"Adding field {fieldAttribute.Name}"); var field = new Field { Name = fieldAttribute.Name,