tedkulp/Prowlarr
1
0
Fork 0
mirror of https://github.com/Prowlarr/Prowlarr.git synced 2026-04-25 22:59:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fixed: (Anidub/Animedia) Use rate limit in sub-requests

Browse Source
This commit is contained in:
Bogdan
2023-02-06 13:25:27 +02:00
parent f8082047a5
commit 94c45541ae
2 changed files with 135 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/NzbDrone.Core/Indexers/Definitions/Anidub.cs
+66 -64
View File
@@ -39,12 +39,12 @@ namespace NzbDrone.Core.Indexers.Definitions
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new AnidubRequestGenerator { Settings = Settings, Capabilities = Capabilities };
return new AnidubRequestGenerator(Settings);
}
public override IParseIndexerResponse GetParser()
{
return new AnidubParser(Settings, Capabilities.Categories) { HttpClient = _httpClient, Logger = _logger };
return new AnidubParser(Settings, Capabilities.Categories, RateLimit, _httpClient, _logger);
}
protected override async Task DoLogin()
@@ -135,21 +135,25 @@ namespace NzbDrone.Core.Indexers.Definitions
public class AnidubRequestGenerator : IIndexerRequestGenerator
{
public UserPassTorrentBaseSettings Settings { get; set; }
public IndexerCapabilities Capabilities { get; set; }
private readonly UserPassTorrentBaseSettings _settings;
private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories)
public AnidubRequestGenerator(UserPassTorrentBaseSettings settings)
{
var requestUrl = string.Empty;
_settings = settings;
}
private IEnumerable<IndexerRequest> GetPagedRequests(string term)
{
string requestUrl;
var isSearch = !string.IsNullOrWhiteSpace(term);
if (isSearch)
{
requestUrl = string.Format("{0}/index.php?do=search", Settings.BaseUrl.TrimEnd('/'));
requestUrl = $"{_settings.BaseUrl.TrimEnd('/')}/index.php?do=search";
}
else
{
requestUrl = Settings.BaseUrl;
requestUrl = _settings.BaseUrl;
}
var request = new IndexerRequest(requestUrl, HttpAccept.Html);
@@ -194,7 +198,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));
return pageableRequests;
}
@@ -203,7 +207,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedTvSearchString), searchCriteria.Categories));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedTvSearchString}"));
return pageableRequests;
}
@@ -212,7 +216,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));
return pageableRequests;
}
@@ -221,7 +225,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));
return pageableRequests;
}
@@ -230,7 +234,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));
return pageableRequests;
}
@@ -243,33 +247,37 @@ namespace NzbDrone.Core.Indexers.Definitions
{
private readonly UserPassTorrentBaseSettings _settings;
private readonly IndexerCapabilitiesCategories _categories;
public IIndexerHttpClient HttpClient { get; set; }
public Logger Logger { get; set; }
private readonly TimeSpan _rateLimit;
private readonly IIndexerHttpClient _httpClient;
private readonly Logger _logger;
private static Dictionary<string, string> CategoriesMap => new Dictionary<string, string>
{
{ "/anime_tv/full", "14" },
{ "/anime_tv/anime_ongoing", "10" },
{ "/anime_tv/shonen", "11" },
{ "/anime_tv", "2" },
{ "/xxx", "13" },
{ "/manga", "15" },
{ "/ost", "16" },
{ "/podcast", "17" },
{ "/anime_movie", "3" },
{ "/anime_ova/anime_ona", "5" },
{ "/anime_ova", "4" },
{ "/dorama/japan_dorama", "6" },
{ "/dorama/korea_dorama", "7" },
{ "/dorama/china_dorama", "8" },
{ "/dorama", "9" },
{ "/anons_ongoing", "12" }
};
private static Dictionary<string, string> CategoriesMap => new ()
{
{ "/anime_tv/full", "14" },
{ "/anime_tv/anime_ongoing", "10" },
{ "/anime_tv/shonen", "11" },
{ "/anime_tv", "2" },
{ "/xxx", "13" },
{ "/manga", "15" },
{ "/ost", "16" },
{ "/podcast", "17" },
{ "/anime_movie", "3" },
{ "/anime_ova/anime_ona", "5" },
{ "/anime_ova", "4" },
{ "/dorama/japan_dorama", "6" },
{ "/dorama/korea_dorama", "7" },
{ "/dorama/china_dorama", "8" },
{ "/dorama", "9" },
{ "/anons_ongoing", "12" }
};
public AnidubParser(UserPassTorrentBaseSettings settings, IndexerCapabilitiesCategories categories)
public AnidubParser(UserPassTorrentBaseSettings settings, IndexerCapabilitiesCategories categories, TimeSpan rateLimit, IIndexerHttpClient httpClient, Logger logger)
{
_settings = settings;
_categories = categories;
_rateLimit = rateLimit;
_httpClient = httpClient;
_logger = logger;
}
private static string GetTitle(AngleSharp.Html.Dom.IHtmlDocument content, AngleSharp.Dom.IElement tabNode)
@@ -314,9 +322,9 @@ namespace NzbDrone.Core.Indexers.Definitions
private static int GetReleaseLeechers(AngleSharp.Dom.IElement tabNode)
{
const string LeechersSelector = ".list.down > .li_swing_m";
const string leechersSelector = ".list.down > .li_swing_m";
var leechersStr = tabNode.QuerySelector(LeechersSelector).TextContent;
var leechersStr = tabNode.QuerySelector(leechersSelector).TextContent;
int.TryParse(leechersStr, out var leechers);
return leechers;
}
@@ -332,18 +340,18 @@ namespace NzbDrone.Core.Indexers.Definitions
private static int GetReleaseGrabs(AngleSharp.Dom.IElement tabNode)
{
const string GrabsSelector = ".list.down > .li_download_m";
const string grabsSelector = ".list.down > .li_download_m";
var grabsStr = tabNode.QuerySelector(GrabsSelector).TextContent;
var grabsStr = tabNode.QuerySelector(grabsSelector).TextContent;
int.TryParse(grabsStr, out var grabs);
return grabs;
}
private static string GetDateFromDocument(AngleSharp.Html.Dom.IHtmlDocument content)
{
const string DateSelector = ".story_inf > li:nth-child(2)";
const string dateSelector = ".story_inf > li:nth-child(2)";
var domDate = content.QuerySelector(DateSelector).LastChild;
var domDate = content.QuerySelector(dateSelector).LastChild;
if (domDate?.NodeName != "#text")
{
@@ -384,16 +392,16 @@ namespace NzbDrone.Core.Indexers.Definitions
return utcDate.AddHours(-russianStandardTimeDiff);
}
Logger.Warn($"[AniDub] Date time couldn't be parsed on. Date text: {dateText}");
_logger.Warn($"[AniDub] Date time couldn't be parsed on. Date text: {dateText}");
return DateTime.UtcNow;
}
private static long GetReleaseSize(AngleSharp.Dom.IElement tabNode)
{
const string SizeSelector = ".list.down > .red";
const string sizeSelector = ".list.down > .red";
var sizeStr = tabNode.QuerySelector(SizeSelector).TextContent;
var sizeStr = tabNode.QuerySelector(sizeSelector).TextContent;
return ParseUtil.GetBytes(sizeStr);
}
@@ -433,11 +441,11 @@ namespace NzbDrone.Core.Indexers.Definitions
var release = new TorrentInfo
{
Title = GetTitle(dom, t),
InfoUrl = indexerResponse.Request.Url.ToString(),
InfoUrl = indexerResponse.Request.Url.FullUri,
DownloadVolumeFactor = 0,
UploadVolumeFactor = 1,
Guid = indexerResponse.Request.Url.ToString() + t.Id,
Guid = indexerResponse.Request.Url.FullUri + t.Id,
Seeders = GetReleaseSeeders(t),
Peers = GetReleaseSeeders(t) + GetReleaseLeechers(t),
Grabs = GetReleaseGrabs(t),
@@ -459,36 +467,30 @@ namespace NzbDrone.Core.Indexers.Definitions
var parser = new HtmlParser();
var dom = parser.ParseDocument(indexerResponse.Content);
var domQuery = string.Empty;
if (indexerResponse.Request.Url.Query.Contains("do=search"))
{
domQuery = ".searchitem > h3 > a";
}
else
{
domQuery = "#dle-content > .story > .story_h > .lcol > h2 > a";
}
var links = dom.QuerySelectorAll(domQuery);
var links = dom.QuerySelectorAll(".searchitem > h3 > a[href], #dle-content > .story > .story_h > .lcol > h2 > a[href]");
foreach (var link in links)
{
var url = link.GetAttribute("href");
var releaseRequest = new IndexerRequest(url, HttpAccept.Html);
var releaseResponse = new IndexerResponse(releaseRequest, HttpClient.Execute(releaseRequest.HttpRequest));
var releaseRequest = new HttpRequestBuilder(url)
.WithRateLimit(_rateLimit.TotalSeconds)
.SetHeader("Referer", _settings.BaseUrl)
.Accept(HttpAccept.Html)
.Build();
var releaseIndexerRequest = new IndexerRequest(releaseRequest);
var releaseResponse = new IndexerResponse(releaseIndexerRequest, _httpClient.Execute(releaseIndexerRequest.HttpRequest));
// Throw common http errors here before we try to parse
if (releaseResponse.HttpResponse.HasHttpError)
{
if (releaseResponse.HttpResponse.StatusCode == HttpStatusCode.TooManyRequests)
{
throw new TooManyRequestsException(releaseRequest.HttpRequest, releaseResponse.HttpResponse);
}
else
{
throw new IndexerException(releaseResponse, "Http error code: " + releaseResponse.HttpResponse.StatusCode);
throw new TooManyRequestsException(releaseResponse.HttpRequest, releaseResponse.HttpResponse);
}
throw new IndexerException(releaseResponse, $"HTTP Error - {releaseResponse.HttpResponse.StatusCode}. {url}");
}
torrentInfos.AddRange(ParseRelease(releaseResponse));