tedkulp/Prowlarr
1
0
Fork 0
mirror of https://github.com/Prowlarr/Prowlarr.git synced 2026-04-24 22:55:21 -04:00
Code Issues Packages Projects Releases Wiki Activity

New: Support for indexers with image CAPTCHAs

Browse Source
This commit is contained in:
ta264
2021-03-17 21:30:11 +00:00
committed by Qstick
parent 95d5e0d347
commit 0fa526a1af
15 changed files with 314 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/NzbDrone.Core/Indexers/Definitions/Cardigann/Captcha.cs
+9
View File
@@ -0,0 +1,9 @@
namespace NzbDrone.Core.Indexers.Definitions.Cardigann
{
public class Captcha
{
public string Type { get; set; } = "image";
public string ContentType { get; set; }
public byte[] ImageData { get; set; }
}
}
src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs
+49 -10
View File
@@ -1,7 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.IndexerVersions;
@@ -13,6 +16,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
public class Cardigann : HttpIndexerBase<CardigannSettings>
{
private readonly IIndexerDefinitionUpdateService _definitionService;
private readonly ICached<CardigannRequestGenerator> _generatorCache;
public override string Name => "Cardigann";
public override string BaseUrl => "";
@@ -23,21 +27,24 @@ namespace NzbDrone.Core.Indexers.Cardigann
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new CardigannRequestGenerator(_configService,
_definitionService.GetDefinition(Settings.DefinitionFile),
Settings,
_logger)
{
HttpClient = _httpClient
};
return _generatorCache.Get(Settings.DefinitionFile, () =>
new CardigannRequestGenerator(_configService,
_definitionService.GetDefinition(Settings.DefinitionFile),
_logger)
{
HttpClient = _httpClient,
Settings = Settings
});
}
public override IParseIndexerResponse GetParser()
{
return new CardigannParser(_configService,
_definitionService.GetDefinition(Settings.DefinitionFile),
Settings,
_logger);
_definitionService.GetDefinition(Settings.DefinitionFile),
_logger)
{
Settings = Settings
};
}
public override IEnumerable<ProviderDefinition> DefaultDefinitions
@@ -55,10 +62,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
IHttpClient httpClient,
IIndexerStatusService indexerStatusService,
IConfigService configService,
ICacheManager cacheManager,
Logger logger)
: base(httpClient, indexerStatusService, configService, logger)
{
_definitionService = definitionService;
_generatorCache = cacheManager.GetRollingCache<CardigannRequestGenerator>(GetType(), "CardigannGeneratorCache", TimeSpan.FromMinutes(5));
}
private IndexerDefinition GetDefinition(CardigannMetaDefinition definition)
@@ -71,6 +80,16 @@ namespace NzbDrone.Core.Indexers.Cardigann
var settings = definition.Settings ?? defaultSettings;
if (definition.Login?.Captcha != null)
{
settings.Add(new SettingsField
{
Name = "cardigannCaptcha",
Type = "cardigannCaptcha",
Label = "CAPTCHA"
});
}
return new IndexerDefinition
{
Enable = true,
@@ -93,6 +112,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
SetCookieFunctions(generator);
generator.Settings = Settings;
return generator.CheckIfLoginIsNeeded(httpResponse);
}
@@ -102,6 +123,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
SetCookieFunctions(generator);
generator.Settings = Settings;
await generator.DoLogin();
}
@@ -113,5 +136,21 @@ namespace NzbDrone.Core.Indexers.Cardigann
return;
}
}
public override object RequestAction(string action, IDictionary<string, string> query)
{
if (action == "checkCaptcha")
{
var generator = (CardigannRequestGenerator)GetRequestGenerator();
var result = generator.GetConfigurationForSetup(false).GetAwaiter().GetResult();
return new
{
captchaRequest = result
};
}
return null;
}
}
}
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs
+6 -4
View File
@@ -18,7 +18,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
public class CardigannBase
{
protected readonly CardigannDefinition _definition;
protected readonly CardigannSettings _settings;
protected readonly Logger _logger;
protected readonly Encoding _encoding;
protected readonly IConfigService _configService;
@@ -48,14 +47,14 @@ namespace NzbDrone.Core.Indexers.Cardigann
protected static readonly Regex _LogicFunctionRegex = new Regex(
$@"\b({string.Join("|", _SupportedLogicFunctions.Select(Regex.Escape))})(?:\s+(\(?\.[^\)\s]+\)?|""[^""]+"")){{2,}}");
public CardigannSettings Settings { get; set; }
public CardigannBase(IConfigService configService,
CardigannDefinition definition,
CardigannSettings settings,
Logger logger)
{
_configService = configService;
_definition = definition;
_settings = settings;
_encoding = Encoding.GetEncoding(definition.Encoding);
_logger = logger;
@@ -224,7 +223,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
foreach (var setting in _definition.Settings)
{
var name = ".Config." + setting.Name;
var value = _settings.ExtraFieldData.GetValueOrDefault(setting.Name, setting.Default);
var value = Settings.ExtraFieldData.GetValueOrDefault(setting.Name, setting.Default);
if (setting.Type != "password" && indexerLogging)
{
@@ -260,6 +259,9 @@ namespace NzbDrone.Core.Indexers.Cardigann
{
variables[name] = value;
}
else if (setting.Type == "cardigannCaptcha")
{
}
else
{
throw new NotSupportedException();
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs
+1
View File
@@ -15,5 +15,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
public List<string> Legacylinks { get; set; }
public List<SettingsField> Settings { get; set; }
public string Sha { get; set; }
public LoginBlock Login { get; set; }
}
}
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs
+1 -2
View File
@@ -20,9 +20,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
public CardigannParser(IConfigService configService,
CardigannDefinition definition,
CardigannSettings settings,
Logger logger)
: base(configService, definition, settings, logger)
: base(configService, definition, logger)
{
}
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs
+53 -138
View File
@@ -25,9 +25,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
public CardigannRequestGenerator(IConfigService configService,
CardigannDefinition definition,
CardigannSettings settings,
Logger logger)
: base(configService, definition, settings, logger)
: base(configService, definition, logger)
{
}
@@ -181,7 +180,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
LogResponseContent = true,
Method = HttpMethod.POST,
AllowAutoRedirect = true,
SuppressHttpError = true
SuppressHttpError = true,
};
foreach (var pair in pairs)
@@ -339,46 +338,22 @@ namespace NzbDrone.Core.Indexers.Cardigann
if (login.Captcha != null)
{
var captcha = login.Captcha;
if (captcha.Type == "image")
Settings.ExtraFieldData.TryGetValue("CAPTCHA", out var captchaText);
if (captchaText != null)
{
_settings.ExtraFieldData.TryGetValue("CaptchaText", out var captchaText);
if (captchaText != null)
var input = captcha.Input;
if (login.Selectors)
{
var input = captcha.Input;
if (login.Selectors)
var inputElement = landingResultDocument.QuerySelector(captcha.Input);
if (inputElement == null)
{
var inputElement = landingResultDocument.QuerySelector(captcha.Input);
if (inputElement == null)
{
throw new CardigannConfigException(_definition, string.Format("Login failed: No captcha input found using {0}", captcha.Input));
}
input = inputElement.GetAttribute("name");
throw new CardigannConfigException(_definition, string.Format("Login failed: No captcha input found using {0}", captcha.Input));
}
pairs[input] = (string)captchaText;
input = inputElement.GetAttribute("name");
}
}
if (captcha.Type == "text")
{
_settings.ExtraFieldData.TryGetValue("CaptchaAnswer", out var captchaAnswer);
if (captchaAnswer != null)
{
var input = captcha.Input;
if (login.Selectors)
{
var inputElement = landingResultDocument.QuerySelector(captcha.Input);
if (inputElement == null)
{
throw new CardigannConfigException(_definition, string.Format("Login failed: No captcha input found using {0}", captcha.Input));
}
input = inputElement.GetAttribute("name");
}
pairs[input] = (string)captchaAnswer;
}
pairs[input] = (string)captchaText;
}
}
@@ -462,7 +437,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
else if (login.Method == "cookie")
{
CookiesUpdater(null, null);
_settings.ExtraFieldData.TryGetValue("cookie", out var cookies);
Settings.ExtraFieldData.TryGetValue("cookie", out var cookies);
CookiesUpdater(CookieUtil.CookieHeaderToDictionary((string)cookies), DateTime.Now + TimeSpan.FromDays(30));
}
else if (login.Method == "get")
@@ -557,13 +532,13 @@ namespace NzbDrone.Core.Indexers.Cardigann
return true;
}
public async Task GetConfigurationForSetup(bool automaticlogin)
public async Task<Captcha> GetConfigurationForSetup(bool automaticlogin)
{
var login = _definition.Login;
if (login == null || login.Method != "form")
{
return;
return null;
}
var loginUrl = ResolvePath(login.Path);
@@ -588,7 +563,9 @@ namespace NzbDrone.Core.Indexers.Cardigann
requestBuilder.SetCookies(Cookies);
}
landingResult = await HttpClient.ExecuteAsync(requestBuilder.Build());
var request = requestBuilder.Build();
landingResult = await HttpClient.ExecuteAsync(request);
Cookies = landingResult.GetCookies();
@@ -597,122 +574,60 @@ namespace NzbDrone.Core.Indexers.Cardigann
//{
// await FollowIfRedirect(landingResult, loginUrl.AbsoluteUri, overrideCookies: landingResult.Cookies, accumulateCookies: true);
//}
var hasCaptcha = false;
var htmlParser = new HtmlParser();
landingResultDocument = htmlParser.ParseDocument(landingResult.Content);
Captcha captcha = null;
if (login.Captcha != null)
{
var captcha = login.Captcha;
if (captcha.Type == "image")
captcha = await GetCaptcha(login);
}
if (captcha != null && automaticlogin)
{
_logger.Error(string.Format("CardigannIndexer ({0}): Found captcha during automatic login, aborting", _definition.Id));
}
return captcha;
}
private async Task<Captcha> GetCaptcha(LoginBlock login)
{
var captcha = login.Captcha;
if (captcha.Type == "image")
{
var captchaElement = landingResultDocument.QuerySelector(captcha.Selector);
if (captchaElement != null)
{
var captchaElement = landingResultDocument.QuerySelector(captcha.Selector);
if (captchaElement != null)
{
hasCaptcha = true;
var loginUrl = ResolvePath(login.Path);
var captchaUrl = ResolvePath(captchaElement.GetAttribute("src"), loginUrl);
//TODO Bubble this to UI when we get a captcha so that user can action it
//Jackett does this by inserting image or question into the extrasettings which then show up in the add modal
//
//var captchaUrl = ResolvePath(captchaElement.GetAttribute("src"), loginUrl);
//var captchaImageData = RequestWithCookiesAsync(captchaUrl.ToString(), landingResult.GetCookies, referer: loginUrl.AbsoluteUri);
// var CaptchaImage = new ImageItem { Name = "Captcha Image" };
//var CaptchaText = new StringItem { Name = "Captcha Text" };
//CaptchaImage.Value = captchaImageData.ContentBytes;
//configData.AddDynamic("CaptchaImage", CaptchaImage);
//configData.AddDynamic("CaptchaText", CaptchaText);
}
else
{
_logger.Debug(string.Format("CardigannIndexer ({0}): No captcha image found", _definition.Id));
}
}
else if (captcha.Type == "text")
{
var captchaElement = landingResultDocument.QuerySelector(captcha.Selector);
if (captchaElement != null)
{
hasCaptcha = true;
var request = new HttpRequestBuilder(captchaUrl.ToString())
.SetCookies(landingResult.GetCookies())
.SetHeader("Referrer", loginUrl.AbsoluteUri)
.Build();
//var captchaChallenge = new DisplayItem(captchaElement.TextContent) { Name = "Captcha Challenge" };
//var captchaAnswer = new StringItem { Name = "Captcha Answer" };
var response = await HttpClient.ExecuteAsync(request);
//configData.AddDynamic("CaptchaChallenge", captchaChallenge);
//configData.AddDynamic("CaptchaAnswer", captchaAnswer);
}
else
return new Captcha
{
_logger.Debug(string.Format("CardigannIndexer ({0}): No captcha image found", _definition.Id));
}
ContentType = response.Headers.ContentType,
ImageData = response.ResponseData
};
}
else
{
throw new NotImplementedException(string.Format("Captcha type \"{0}\" is not implemented", captcha.Type));
_logger.Debug(string.Format("CardigannIndexer ({0}): No captcha image found", _definition.Id));
}
}
if (hasCaptcha && automaticlogin)
else
{
_logger.Error(string.Format("CardigannIndexer ({0}): Found captcha during automatic login, aborting", _definition.Id));
return;
throw new NotImplementedException(string.Format("Captcha type \"{0}\" is not implemented", captcha.Type));
}
return;
}
protected async Task<bool> TestLogin()
{
var login = _definition.Login;
if (login == null || login.Test == null)
{
return false;
}
// test if login was successful
var loginTestUrl = ResolvePath(login.Test.Path).ToString();
// var headers = ParseCustomHeaders(_definition.Search?.Headers, GetBaseTemplateVariables());
var requestBuilder = new HttpRequestBuilder(loginTestUrl)
{
LogResponseContent = true,
Method = HttpMethod.GET,
SuppressHttpError = true
};
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var testResult = await HttpClient.ExecuteAsync(requestBuilder.Build());
if (testResult.HasHttpRedirect)
{
var errormessage = "Login Failed, got redirected.";
var domainHint = GetRedirectDomainHint(testResult);
if (domainHint != null)
{
errormessage += " Try changing the indexer URL to " + domainHint + ".";
}
_logger.Debug(errormessage);
return false;
}
if (login.Test.Selector != null)
{
var testResultParser = new HtmlParser();
var testResultDocument = testResultParser.ParseDocument(testResult.Content);
var selection = testResultDocument.QuerySelectorAll(login.Test.Selector);
if (selection.Length == 0)
{
_logger.Debug(string.Format("Login failed: Selector \"{0}\" didn't match", login.Test.Selector));
return false;
}
}
return true;
return null;
}
protected string GetRedirectDomainHint(string requestUrl, string redirectUrl)