Cleanup Search UI, Newznab Caps API

This commit is contained in:
Qstick
2020-10-20 13:46:58 -04:00
parent f290afa68c
commit 84cbfe870f
105 changed files with 562 additions and 791 deletions
@@ -0,0 +1,33 @@
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
namespace NzbDrone.Core.Indexers.AwesomeHD
{
public class AwesomeHD : HttpIndexerBase<AwesomeHDSettings>
{
public override string Name => "AwesomeHD";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override bool SupportsRss => true;
public override bool SupportsSearch => true;
public override int PageSize => 50;
public AwesomeHD(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, indexerStatusService, configService, logger)
{
}
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new AwesomeHDRequestGenerator() { Settings = Settings };
}
public override IParseIndexerResponse GetParser()
{
return new AwesomeHDRssParser(Settings);
}
}
}
@@ -0,0 +1,41 @@
using System;
using System.Collections.Generic;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.AwesomeHD
{
public class AwesomeHDRequestGenerator : IIndexerRequestGenerator
{
public AwesomeHDSettings Settings { get; set; }
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetRequest(null));
return pageableRequests;
}
public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetRequest(searchCriteria.ImdbId));
return pageableRequests;
}
public Func<IDictionary<string, string>> GetCookies { get; set; }
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
private IEnumerable<IndexerRequest> GetRequest(string searchParameters)
{
if (searchParameters != null)
{
yield return new IndexerRequest($"{Settings.BaseUrl.Trim().TrimEnd('/')}/searchapi.php?action=imdbsearch&passkey={Settings.Passkey.Trim()}&imdb={searchParameters}", HttpAccept.Rss);
}
else
{
yield return new IndexerRequest($"{Settings.BaseUrl.Trim().TrimEnd('/')}/searchapi.php?action=latestmovies&passkey={Settings.Passkey.Trim()}", HttpAccept.Rss);
}
}
}
}
@@ -0,0 +1,168 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Xml;
using System.Xml.Linq;
using NzbDrone.Common.Http;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.AwesomeHD
{
public class AwesomeHDRssParser : IParseIndexerResponse
{
private readonly AwesomeHDSettings _settings;
public AwesomeHDRssParser(AwesomeHDSettings settings)
{
_settings = settings;
}
public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var torrentInfos = new List<ReleaseInfo>();
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new IndexerException(indexerResponse,
"Unexpected response status {0} code from API request",
indexerResponse.HttpResponse.StatusCode);
}
try
{
var xdoc = XDocument.Parse(indexerResponse.Content);
var searchResults = xdoc.Descendants("searchresults").Select(x => new
{
AuthKey = x.Element("authkey").Value,
}).FirstOrDefault();
var torrents = xdoc.Descendants("torrent")
.Select(x => new
{
Id = x.Element("id").Value,
Name = x.Element("name").Value,
Year = x.Element("year").Value,
GroupId = x.Element("groupid").Value,
Time = DateTime.Parse(x.Element("time").Value),
UserId = x.Element("userid").Value,
Size = long.Parse(x.Element("size").Value),
Snatched = x.Element("snatched").Value,
Seeders = x.Element("seeders").Value,
Leechers = x.Element("leechers").Value,
ReleaseGroup = x.Element("releasegroup").Value,
Resolution = x.Element("resolution").Value,
Media = x.Element("media").Value,
Format = x.Element("format").Value,
Encoding = x.Element("encoding").Value,
AudioFormat = x.Element("audioformat").Value,
AudioBitrate = x.Element("audiobitrate").Value,
AudioChannels = x.Element("audiochannels").Value,
Subtitles = x.Element("subtitles").Value,
EncodeStatus = x.Element("encodestatus").Value,
Freeleech = x.Element("freeleech").Value,
Internal = x.Element("internal").Value == "1",
UserRelease = x.Element("userrelease").Value == "1",
ImdbId = x.Element("imdb").Value
}).ToList();
foreach (var torrent in torrents)
{
var id = torrent.Id;
var title = $"{torrent.Name}.{torrent.Year}.{torrent.Resolution}.{torrent.Media}.{torrent.Encoding}.{torrent.AudioFormat}-{torrent.ReleaseGroup}";
if (torrent.Encoding.ToLower() == "x265")
{
//Per AHD staff they only allow HDR x265 encodes (https://github.com/Prowlarr/Prowlarr/issues/4386)
title = $"{torrent.Name}.{torrent.Year}.{torrent.Resolution}.{torrent.Media}.HDR.{torrent.Encoding}.{torrent.AudioFormat}-{torrent.ReleaseGroup}";
}
IndexerFlags flags = 0;
if (torrent.Freeleech == "0.00")
{
flags |= IndexerFlags.G_Freeleech;
}
if (torrent.Freeleech == "0.25")
{
flags |= IndexerFlags.G_Freeleech75;
}
if (torrent.Freeleech == "0.75")
{
flags |= IndexerFlags.G_Freeleech25;
}
if (torrent.Freeleech == "0.50")
{
flags |= IndexerFlags.G_Halfleech;
}
if (torrent.Internal)
{
flags |= IndexerFlags.AHD_Internal;
}
if (torrent.UserRelease)
{
flags |= IndexerFlags.AHD_UserRelease;
}
var imdbId = 0;
if (torrent.ImdbId.Length > 2)
{
imdbId = int.Parse(torrent.ImdbId.Substring(2));
}
torrentInfos.Add(new TorrentInfo()
{
Guid = string.Format("AwesomeHD-{0}", id),
Title = title,
Size = torrent.Size,
DownloadUrl = GetDownloadUrl(id, searchResults.AuthKey, _settings.Passkey),
InfoUrl = GetInfoUrl(torrent.GroupId, id),
Seeders = int.Parse(torrent.Seeders),
Peers = int.Parse(torrent.Leechers) + int.Parse(torrent.Seeders),
PublishDate = torrent.Time.ToUniversalTime(),
ImdbId = imdbId,
IndexerFlags = flags,
});
}
}
catch (XmlException)
{
throw new IndexerException(indexerResponse,
"An error occurred while processing feed, feed invalid");
}
return torrentInfos.OrderByDescending(o => ((dynamic)o).Seeders).ToArray();
}
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
private string GetDownloadUrl(string torrentId, string authKey, string passKey)
{
var url = new HttpUri(_settings.BaseUrl)
.CombinePath("/torrents.php")
.AddQueryParam("action", "download")
.AddQueryParam("id", torrentId)
.AddQueryParam("authkey", authKey)
.AddQueryParam("torrent_pass", passKey);
return url.FullUri;
}
private string GetInfoUrl(string groupId, string torrentId)
{
var url = new HttpUri(_settings.BaseUrl)
.CombinePath("/torrents.php")
.AddQueryParam("id", groupId)
.AddQueryParam("torrentid", torrentId);
return url.FullUri;
}
}
}
@@ -0,0 +1,51 @@
using System.Collections.Generic;
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.AwesomeHD
{
public class AwesomeHDSettingsValidator : AbstractValidator<AwesomeHDSettings>
{
public AwesomeHDSettingsValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.Passkey).NotEmpty();
}
}
public class AwesomeHDSettings : ITorrentIndexerSettings
{
private static readonly AwesomeHDSettingsValidator Validator = new AwesomeHDSettingsValidator();
public AwesomeHDSettings()
{
BaseUrl = "https://awesome-hd.me";
MinimumSeeders = 0;
MultiLanguages = new List<int>();
RequiredFlags = new List<int>();
}
[FieldDefinition(0, Label = "API URL", Advanced = true, HelpText = "Do not change this unless you know what you're doing. Since you Passkey will be sent to that host.")]
public string BaseUrl { get; set; }
[FieldDefinition(1, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
public IEnumerable<int> MultiLanguages { get; set; }
[FieldDefinition(2, Label = "Passkey", Privacy = PrivacyLevel.ApiKey)]
public string Passkey { get; set; }
[FieldDefinition(3, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
public int MinimumSeeders { get; set; }
[FieldDefinition(4, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", HelpLink = "https://github.com/Prowlarr/Prowlarr/wiki/Indexer-Flags#1-required-flags", Advanced = true)]
public IEnumerable<int> RequiredFlags { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}
@@ -0,0 +1,30 @@
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
namespace NzbDrone.Core.Indexers.FileList
{
public class FileList : HttpIndexerBase<FileListSettings>
{
public override string Name => "FileList.io";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override bool SupportsRss => true;
public override bool SupportsSearch => true;
public FileList(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, indexerStatusService, configService, logger)
{
}
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new FileListRequestGenerator() { Settings = Settings };
}
public override IParseIndexerResponse GetParser()
{
return new FileListParser(Settings);
}
}
}
@@ -0,0 +1,24 @@
using System;
using Newtonsoft.Json;
namespace NzbDrone.Core.Indexers.FileList
{
public class FileListTorrent
{
public string Id { get; set; }
public string Name { get; set; }
public long Size { get; set; }
public int Leechers { get; set; }
public int Seeders { get; set; }
[JsonProperty(PropertyName = "times_completed")]
public uint TimesCompleted { get; set; }
public uint Comments { get; set; }
public uint Files { get; set; }
[JsonProperty(PropertyName = "imdb")]
public string ImdbId { get; set; }
[JsonProperty(PropertyName = "freeleech")]
public bool FreeLeech { get; set; }
[JsonProperty(PropertyName = "upload_date")]
public DateTime UploadDate { get; set; }
}
}
@@ -0,0 +1,89 @@
using System;
using System.Collections.Generic;
using System.Net;
using Newtonsoft.Json;
using NzbDrone.Common.Http;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.FileList
{
public class FileListParser : IParseIndexerResponse
{
private readonly FileListSettings _settings;
public FileListParser(FileListSettings settings)
{
_settings = settings;
}
public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var torrentInfos = new List<ReleaseInfo>();
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new IndexerException(indexerResponse,
"Unexpected response status {0} code from API request",
indexerResponse.HttpResponse.StatusCode);
}
var queryResults = JsonConvert.DeserializeObject<List<FileListTorrent>>(indexerResponse.Content);
foreach (var result in queryResults)
{
var id = result.Id;
IndexerFlags flags = 0;
if (result.FreeLeech)
{
flags |= IndexerFlags.G_Freeleech;
}
var imdbId = 0;
if (result.ImdbId != null && result.ImdbId.Length > 2)
{
imdbId = int.Parse(result.ImdbId.Substring(2));
}
torrentInfos.Add(new TorrentInfo()
{
Guid = string.Format("FileList-{0}", id),
Title = result.Name,
Size = result.Size,
DownloadUrl = GetDownloadUrl(id),
InfoUrl = GetInfoUrl(id),
Seeders = result.Seeders,
Peers = result.Leechers + result.Seeders,
PublishDate = result.UploadDate,
ImdbId = imdbId,
IndexerFlags = flags
});
}
return torrentInfos.ToArray();
}
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
private string GetDownloadUrl(string torrentId)
{
var url = new HttpUri(_settings.BaseUrl)
.CombinePath("/download.php")
.AddQueryParam("id", torrentId)
.AddQueryParam("passkey", _settings.Passkey);
return url.FullUri;
}
private string GetInfoUrl(string torrentId)
{
var url = new HttpUri(_settings.BaseUrl)
.CombinePath("/details.php")
.AddQueryParam("id", torrentId);
return url.FullUri;
}
}
}
@@ -0,0 +1,52 @@
using System;
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.FileList
{
public class FileListRequestGenerator : IIndexerRequestGenerator
{
public FileListSettings Settings { get; set; }
public Func<IDictionary<string, string>> GetCookies { get; set; }
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetRequest("latest-torrents", ""));
return pageableRequests;
}
public virtual IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace())
{
pageableRequests.Add(GetRequest("search-torrents", string.Format("&type=imdb&query={0}", searchCriteria.ImdbId)));
}
else
{
foreach (var queryTitle in searchCriteria.QueryTitles)
{
var titleYearSearchQuery = string.Format("{0}", queryTitle);
pageableRequests.Add(GetRequest("search-torrents", string.Format("&type=name&query={0}", titleYearSearchQuery.Trim())));
}
}
return pageableRequests;
}
private IEnumerable<IndexerRequest> GetRequest(string searchType, string parameters)
{
var categoriesQuery = string.Join(",", Settings.Categories.Distinct());
var baseUrl = string.Format("{0}/api.php?action={1}&category={2}&username={3}&passkey={4}{5}", Settings.BaseUrl.TrimEnd('/'), searchType, categoriesQuery, Settings.Username.Trim(), Settings.Passkey.Trim(), parameters);
yield return new IndexerRequest(baseUrl, HttpAccept.Json);
}
}
}
@@ -0,0 +1,90 @@
using System.Collections.Generic;
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.FileList
{
public class FileListSettingsValidator : AbstractValidator<FileListSettings>
{
public FileListSettingsValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.Username).NotEmpty();
RuleFor(c => c.Passkey).NotEmpty();
}
}
public class FileListSettings : ITorrentIndexerSettings
{
private static readonly FileListSettingsValidator Validator = new FileListSettingsValidator();
public FileListSettings()
{
BaseUrl = "https://filelist.io";
MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
Categories = new int[]
{
(int)FileListCategories.Movie_HD,
(int)FileListCategories.Movie_SD,
(int)FileListCategories.Movie_4K
};
MultiLanguages = new List<int>();
RequiredFlags = new List<int>();
}
[FieldDefinition(0, Label = "Username", Privacy = PrivacyLevel.UserName)]
public string Username { get; set; }
[FieldDefinition(1, Label = "Passkey", Privacy = PrivacyLevel.ApiKey)]
public string Passkey { get; set; }
[FieldDefinition(2, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
public IEnumerable<int> MultiLanguages { get; set; }
[FieldDefinition(3, Label = "API URL", Advanced = true, HelpText = "Do not change this unless you know what you're doing. Since your API key will be sent to that host.")]
public string BaseUrl { get; set; }
[FieldDefinition(4, Label = "Categories", Type = FieldType.Select, SelectOptions = typeof(FileListCategories), Advanced = true, HelpText = "Categories for use in search and feeds. If unspecified, all options are used.")]
public IEnumerable<int> Categories { get; set; }
[FieldDefinition(5, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
public int MinimumSeeders { get; set; }
[FieldDefinition(6, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", HelpLink = "https://github.com/Prowlarr/Prowlarr/wiki/Indexer-Flags#1-required-flags", Advanced = true)]
public IEnumerable<int> RequiredFlags { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
public enum FileListCategories
{
[FieldOption]
Movie_SD = 1,
[FieldOption]
Movie_DVD = 2,
[FieldOption]
Movie_DVDRO = 3,
[FieldOption]
Movie_HD = 4,
[FieldOption]
Movie_HDRO = 19,
[FieldOption]
Movie_BluRay = 20,
[FieldOption]
Movie_BluRay4K = 26,
[FieldOption]
Movie_3D = 25,
[FieldOption]
Movie_4K = 6,
[FieldOption]
Xxx = 7
}
}
@@ -0,0 +1,30 @@
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
namespace NzbDrone.Core.Indexers.HDBits
{
public class HDBits : HttpIndexerBase<HDBitsSettings>
{
public override string Name => "HDBits";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override int PageSize => 30;
public HDBits(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, indexerStatusService, configService, logger)
{
}
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new HDBitsRequestGenerator() { Settings = Settings };
}
public override IParseIndexerResponse GetParser()
{
return new HDBitsParser(Settings);
}
}
}
@@ -0,0 +1,132 @@
using System;
using Newtonsoft.Json;
namespace NzbDrone.Core.Indexers.HDBits
{
public class TorrentQuery
{
[JsonProperty(Required = Required.Always)]
public string Username { get; set; }
[JsonProperty(Required = Required.Always)]
public string Passkey { get; set; }
public string Hash { get; set; }
public string Search { get; set; }
public int[] Category { get; set; }
public int[] Codec { get; set; }
public int[] Medium { get; set; }
public int? Origin { get; set; }
[JsonProperty(PropertyName = "imdb")]
public ImdbInfo ImdbInfo { get; set; }
[JsonProperty(PropertyName = "tvdb")]
public TvdbInfo TvdbInfo { get; set; }
[JsonProperty(PropertyName = "file_in_torrent")]
public string FileInTorrent { get; set; }
[JsonProperty(PropertyName = "snatched_only")]
public bool? SnatchedOnly { get; set; }
public int? Limit { get; set; }
public int? Page { get; set; }
public TorrentQuery Clone()
{
return MemberwiseClone() as TorrentQuery;
}
}
public class HDBitsResponse
{
[JsonProperty(Required = Required.Always)]
public StatusCode Status { get; set; }
public string Message { get; set; }
public object Data { get; set; }
}
public class TorrentQueryResponse
{
public string Id { get; set; }
public string Hash { get; set; }
public int Leechers { get; set; }
public int Seeders { get; set; }
public string Name { get; set; }
[JsonProperty(PropertyName = "times_completed")]
public uint TimesCompleted { get; set; }
public long Size { get; set; }
[JsonProperty(PropertyName = "utadded")]
public long UtAdded { get; set; }
public DateTime Added { get; set; }
public uint Comments { get; set; }
[JsonProperty(PropertyName = "numfiles")]
public uint NumFiles { get; set; }
[JsonProperty(PropertyName = "filename")]
public string FileName { get; set; }
[JsonProperty(PropertyName = "freeleech")]
public string FreeLeech { get; set; }
[JsonProperty(PropertyName = "type_category")]
public int TypeCategory { get; set; }
[JsonProperty(PropertyName = "type_codec")]
public int TypeCodec { get; set; }
[JsonProperty(PropertyName = "type_medium")]
public int TypeMedium { get; set; }
[JsonProperty(PropertyName = "type_origin")]
public int TypeOrigin { get; set; }
[JsonProperty(PropertyName = "imdb")]
public ImdbInfo ImdbInfo { get; set; }
[JsonProperty(PropertyName = "tvdb")]
public TvdbInfo TvdbInfo { get; set; }
}
public class ImdbInfo
{
public int Id { get; set; }
public string EnglishTitle { get; set; }
public string OriginalTitle { get; set; }
public int? Year { get; set; }
public string[] Genres { get; set; }
public float? Rating { get; set; }
}
public class TvdbInfo
{
public int? Id { get; set; }
public int? Season { get; set; }
public int? Episode { get; set; }
}
public enum StatusCode
{
Success = 0,
Failure = 1,
SslRequired = 2,
JsonMalformed = 3,
AuthDataMissing = 4,
AuthFailed = 5,
MissingRequiredParameters = 6,
InvalidParameter = 7,
ImdbImportFail = 8,
ImdbTvNotAllowed = 9
}
}
@@ -0,0 +1,9 @@
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.HDBits
{
public class HDBitsInfo : TorrentInfo
{
public bool? Internal { get; set; }
}
}
@@ -0,0 +1,109 @@
using System;
using System.Collections.Generic;
using System.Net;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NzbDrone.Common.Http;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.HDBits
{
public class HDBitsParser : IParseIndexerResponse
{
private readonly HDBitsSettings _settings;
public HDBitsParser(HDBitsSettings settings)
{
_settings = settings;
}
public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var torrentInfos = new List<ReleaseInfo>();
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new IndexerException(indexerResponse,
"Unexpected response status {0} code from API request",
indexerResponse.HttpResponse.StatusCode);
}
var jsonResponse = JsonConvert.DeserializeObject<HDBitsResponse>(indexerResponse.Content);
if (jsonResponse.Status != StatusCode.Success)
{
throw new IndexerException(indexerResponse,
"HDBits API request returned status code {0}: {1}",
jsonResponse.Status,
jsonResponse.Message ?? string.Empty);
}
var responseData = jsonResponse.Data as JArray;
if (responseData == null)
{
throw new IndexerException(indexerResponse,
"Indexer API call response missing result data");
}
var queryResults = responseData.ToObject<TorrentQueryResponse[]>();
foreach (var result in queryResults)
{
var id = result.Id;
var internalRelease = result.TypeOrigin == 1 ? true : false;
IndexerFlags flags = 0;
if (result.FreeLeech == "yes")
{
flags |= IndexerFlags.G_Freeleech;
}
if (internalRelease)
{
flags |= IndexerFlags.HDB_Internal;
}
torrentInfos.Add(new HDBitsInfo()
{
Guid = string.Format("HDBits-{0}", id),
Title = result.Name,
Size = result.Size,
InfoHash = result.Hash,
DownloadUrl = GetDownloadUrl(id),
InfoUrl = GetInfoUrl(id),
Seeders = result.Seeders,
Peers = result.Leechers + result.Seeders,
PublishDate = result.Added.ToUniversalTime(),
Internal = internalRelease,
ImdbId = result.ImdbInfo?.Id ?? 0,
IndexerFlags = flags
});
}
return torrentInfos.ToArray();
}
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
private string GetDownloadUrl(string torrentId)
{
var url = new HttpUri(_settings.BaseUrl)
.CombinePath("/download.php")
.AddQueryParam("id", torrentId)
.AddQueryParam("passkey", _settings.ApiKey);
return url.FullUri;
}
private string GetInfoUrl(string torrentId)
{
var url = new HttpUri(_settings.BaseUrl)
.CombinePath("/details.php")
.AddQueryParam("id", torrentId);
return url.FullUri;
}
}
}
@@ -0,0 +1,80 @@
using System;
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.HDBits
{
public class HDBitsRequestGenerator : IIndexerRequestGenerator
{
public HDBitsSettings Settings { get; set; }
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetRequest(new TorrentQuery()));
return pageableRequests;
}
public virtual IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
var query = new TorrentQuery();
if (TryAddSearchParameters(query, searchCriteria))
{
pageableRequests.Add(GetRequest(query));
}
return pageableRequests;
}
private bool TryAddSearchParameters(TorrentQuery query, SearchCriteriaBase searchCriteria)
{
if (searchCriteria.ImdbId.IsNullOrWhiteSpace())
{
return false;
}
var imdbId = int.Parse(searchCriteria.ImdbId.Substring(2));
if (imdbId != 0)
{
query.ImdbInfo = query.ImdbInfo ?? new ImdbInfo();
query.ImdbInfo.Id = imdbId;
return true;
}
return false;
}
public Func<IDictionary<string, string>> GetCookies { get; set; }
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
private IEnumerable<IndexerRequest> GetRequest(TorrentQuery query)
{
var request = new HttpRequestBuilder(Settings.BaseUrl)
.Resource("/api/torrents")
.Build();
request.Method = HttpMethod.POST;
const string appJson = "application/json";
request.Headers.Accept = appJson;
request.Headers.ContentType = appJson;
query.Username = Settings.Username;
query.Passkey = Settings.ApiKey;
query.Category = Settings.Categories.ToArray();
query.Codec = Settings.Codecs.ToArray();
query.Medium = Settings.Mediums.ToArray();
request.SetContent(query.ToJson());
yield return new IndexerRequest(request);
}
}
}
@@ -0,0 +1,97 @@
using System.Collections.Generic;
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.HDBits
{
public class HDBitsSettingsValidator : AbstractValidator<HDBitsSettings>
{
public HDBitsSettingsValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.ApiKey).NotEmpty();
}
}
public class HDBitsSettings : ITorrentIndexerSettings
{
private static readonly HDBitsSettingsValidator Validator = new HDBitsSettingsValidator();
public HDBitsSettings()
{
BaseUrl = "https://hdbits.org";
MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
Categories = new int[] { (int)HdBitsCategory.Movie };
Codecs = System.Array.Empty<int>();
Mediums = System.Array.Empty<int>();
MultiLanguages = new List<int>();
RequiredFlags = new List<int>();
}
[FieldDefinition(0, Label = "Username", Privacy = PrivacyLevel.UserName)]
public string Username { get; set; }
[FieldDefinition(1, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
public IEnumerable<int> MultiLanguages { get; set; }
[FieldDefinition(2, Label = "API Key", Privacy = PrivacyLevel.ApiKey)]
public string ApiKey { get; set; }
[FieldDefinition(3, Label = "API URL", Advanced = true, HelpText = "Do not change this unless you know what you're doing. Since your API key will be sent to that host.")]
public string BaseUrl { get; set; }
[FieldDefinition(4, Label = "Categories", Type = FieldType.TagSelect, SelectOptions = typeof(HdBitsCategory), Advanced = true, HelpText = "Options: Movie, TV, Documentary, Music, Sport, Audio, XXX, MiscDemo. If unspecified, all options are used.")]
public IEnumerable<int> Categories { get; set; }
[FieldDefinition(5, Label = "Codecs", Type = FieldType.TagSelect, SelectOptions = typeof(HdBitsCodec), Advanced = true, HelpText = "Options: h264, Mpeg2, VC1, Xvid. If unspecified, all options are used.")]
public IEnumerable<int> Codecs { get; set; }
[FieldDefinition(6, Label = "Mediums", Type = FieldType.TagSelect, SelectOptions = typeof(HdBitsMedium), Advanced = true, HelpText = "Options: BluRay, Encode, Capture, Remux, WebDL. If unspecified, all options are used.")]
public IEnumerable<int> Mediums { get; set; }
[FieldDefinition(7, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
public int MinimumSeeders { get; set; }
[FieldDefinition(8, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", HelpLink = "https://github.com/Prowlarr/Prowlarr/wiki/Indexer-Flags#1-required-flags", Advanced = true)]
public IEnumerable<int> RequiredFlags { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
public enum HdBitsCategory
{
Movie = 1,
Tv = 2,
Documentary = 3,
Music = 4,
Sport = 5,
Audio = 6,
Xxx = 7,
MiscDemo = 8
}
public enum HdBitsCodec
{
H264 = 1,
Mpeg2 = 2,
Vc1 = 3,
Xvid = 4,
HEVC = 5
}
public enum HdBitsMedium
{
Bluray = 1,
Encode = 3,
Capture = 4,
Remux = 5,
WebDl = 6
}
}
@@ -0,0 +1,32 @@
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
namespace NzbDrone.Core.Indexers.IPTorrents
{
public class IPTorrents : HttpIndexerBase<IPTorrentsSettings>
{
public override string Name => "IP Torrents";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override bool SupportsSearch => false;
public override int PageSize => 0;
public IPTorrents(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, indexerStatusService, configService, logger)
{
}
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new IPTorrentsRequestGenerator() { Settings = Settings };
}
public override IParseIndexerResponse GetParser()
{
return new TorrentRssParser() { ParseSizeInDescription = true };
}
}
}
@@ -0,0 +1,34 @@
using System;
using System.Collections.Generic;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.IPTorrents
{
public class IPTorrentsRequestGenerator : IIndexerRequestGenerator
{
public IPTorrentsSettings Settings { get; set; }
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetRssRequests());
return pageableRequests;
}
public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
{
return new IndexerPageableRequestChain();
}
private IEnumerable<IndexerRequest> GetRssRequests()
{
yield return new IndexerRequest(Settings.BaseUrl, HttpAccept.Rss);
}
public Func<IDictionary<string, string>> GetCookies { get; set; }
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
}
}
@@ -0,0 +1,55 @@
using System.Collections.Generic;
using System.Text.RegularExpressions;
using FluentValidation;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.IPTorrents
{
public class IPTorrentsSettingsValidator : AbstractValidator<IPTorrentsSettings>
{
public IPTorrentsSettingsValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.BaseUrl).Matches(@"(?:/|t\.)rss\?.+$");
RuleFor(c => c.BaseUrl).Matches(@"(?:/|t\.)rss\?.+;download(?:;|$)")
.WithMessage("Use Direct Download Url (;download)")
.When(v => v.BaseUrl.IsNotNullOrWhiteSpace() && Regex.IsMatch(v.BaseUrl, @"(?:/|t\.)rss\?.+$"));
}
}
public class IPTorrentsSettings : ITorrentIndexerSettings
{
private static readonly IPTorrentsSettingsValidator Validator = new IPTorrentsSettingsValidator();
public IPTorrentsSettings()
{
BaseUrl = string.Empty;
MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
MultiLanguages = new List<int>();
RequiredFlags = new List<int>();
}
[FieldDefinition(0, Label = "Feed URL", HelpText = "The full RSS feed url generated by IPTorrents, using only the categories you selected (HD, SD, x264, etc ...)")]
public string BaseUrl { get; set; }
[FieldDefinition(1, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
public IEnumerable<int> MultiLanguages { get; set; }
[FieldDefinition(2, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
public int MinimumSeeders { get; set; }
[FieldDefinition(3, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", HelpLink = "https://github.com/Prowlarr/Prowlarr/wiki/Indexer-Flags#1-required-flags", Advanced = true)]
public IEnumerable<int> RequiredFlags { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}
@@ -0,0 +1,164 @@
using System;
using System.Collections.Generic;
using System.Linq;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.ThingiProvider;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Newznab
{
public class Newznab : HttpIndexerBase<NewznabSettings>
{
private readonly INewznabCapabilitiesProvider _capabilitiesProvider;
public override string Name => "Newznab";
public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override int PageSize => _capabilitiesProvider.GetCapabilities(Settings).DefaultPageSize;
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new NewznabRequestGenerator(_capabilitiesProvider)
{
PageSize = PageSize,
Settings = Settings
};
}
public override IParseIndexerResponse GetParser()
{
return new NewznabRssParser(Settings);
}
public override IEnumerable<ProviderDefinition> DefaultDefinitions
{
get
{
yield return GetDefinition("DOGnzb", GetSettings("https://api.dognzb.cr"));
yield return GetDefinition("DrunkenSlug", GetSettings("https://api.drunkenslug.com"));
yield return GetDefinition("Nzb-Tortuga", GetSettings("https://www.nzb-tortuga.com"));
yield return GetDefinition("Nzb.su", GetSettings("https://api.nzb.su"));
yield return GetDefinition("NZBCat", GetSettings("https://nzb.cat"));
yield return GetDefinition("NZBFinder.ws", GetSettings("https://nzbfinder.ws", categories: new[] { 2030, 2040, 2045, 2050, 2060, 2070, 2080, 2090 }));
yield return GetDefinition("NZBgeek", GetSettings("https://api.nzbgeek.info"));
yield return GetDefinition("nzbplanet.net", GetSettings("https://api.nzbplanet.net"));
yield return GetDefinition("omgwtfnzbs", GetSettings("https://api.omgwtfnzbs.me", categories: new[] { 2000, 2020, 2030, 2040, 2045, 2050, 2070 }));
yield return GetDefinition("OZnzb.com", GetSettings("https://api.oznzb.com"));
yield return GetDefinition("SimplyNZBs", GetSettings("https://simplynzbs.com"));
yield return GetDefinition("Tabula Rasa", GetSettings("https://www.tabula-rasa.pw", apiPath: @"/api/v1/api"));
yield return GetDefinition("Usenet Crawler", GetSettings("https://www.usenet-crawler.com"));
}
}
public Newznab(INewznabCapabilitiesProvider capabilitiesProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, indexerStatusService, configService, logger)
{
_capabilitiesProvider = capabilitiesProvider;
}
private IndexerDefinition GetDefinition(string name, NewznabSettings settings)
{
return new IndexerDefinition
{
EnableRss = false,
EnableAutomaticSearch = false,
EnableInteractiveSearch = false,
Name = name,
Implementation = GetType().Name,
Settings = settings,
Protocol = DownloadProtocol.Usenet,
Privacy = IndexerPrivacy.Private,
SupportsRss = SupportsRss,
SupportsSearch = SupportsSearch
};
}
private NewznabSettings GetSettings(string url, string apiPath = null, int[] categories = null)
{
var settings = new NewznabSettings { BaseUrl = url };
if (categories != null)
{
settings.Categories = categories;
}
if (apiPath.IsNotNullOrWhiteSpace())
{
settings.ApiPath = apiPath;
}
return settings;
}
protected override void Test(List<ValidationFailure> failures)
{
base.Test(failures);
if (failures.HasErrors())
{
return;
}
failures.AddIfNotNull(TestCapabilities());
}
protected static List<int> CategoryIds(List<NewznabCategory> categories)
{
var l = categories.Select(c => c.Id).ToList();
foreach (var category in categories)
{
if (category.Subcategories != null)
{
l.AddRange(CategoryIds(category.Subcategories));
}
}
return l;
}
protected virtual ValidationFailure TestCapabilities()
{
try
{
var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
var notSupported = Settings.Categories.Except(CategoryIds(capabilities.Categories));
if (notSupported.Any())
{
_logger.Warn($"{Definition.Name} does not support the following categories: {string.Join(", ", notSupported)}. You should probably remove them.");
if (notSupported.Count() == Settings.Categories.Count())
{
return new ValidationFailure(string.Empty, $"This indexer does not support any of the selected categories! (You may need to turn on advanced settings to see them)");
}
}
if (capabilities.SupportedSearchParameters != null && capabilities.SupportedSearchParameters.Contains("q"))
{
return null;
}
if (capabilities.SupportedMovieSearchParameters != null &&
new[] { "q", "imdbid" }.Any(v => capabilities.SupportedMovieSearchParameters.Contains(v)) &&
new[] { "imdbtitle", "imdbyear" }.All(v => capabilities.SupportedMovieSearchParameters.Contains(v)))
{
return null;
}
return new ValidationFailure(string.Empty, "This indexer does not support searching for movies :(. Tell your indexer staff to enable this or force add the indexer by disabling search, adding the indexer and then enabling it again.");
}
catch (Exception ex)
{
_logger.Warn(ex, "Unable to connect to indexer: " + ex.Message);
return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log for more details");
}
}
}
}
@@ -0,0 +1,33 @@
using System.Collections.Generic;
namespace NzbDrone.Core.Indexers.Newznab
{
public class NewznabCapabilities
{
public int DefaultPageSize { get; set; }
public int MaxPageSize { get; set; }
public string[] SupportedSearchParameters { get; set; }
public string[] SupportedMovieSearchParameters { get; set; }
public bool SupportsAggregateIdSearch { get; set; }
public List<NewznabCategory> Categories { get; set; }
public NewznabCapabilities()
{
DefaultPageSize = 100;
MaxPageSize = 100;
SupportedSearchParameters = new[] { "q" };
SupportedMovieSearchParameters = new[] { "q", "imdbid", "imdbtitle", "imdbyear" };
SupportsAggregateIdSearch = false;
Categories = new List<NewznabCategory>();
}
}
public class NewznabCategory
{
public int Id { get; set; }
public string Name { get; set; }
public string Description { get; set; }
public List<NewznabCategory> Subcategories { get; set; }
}
}
@@ -0,0 +1,166 @@
using System;
using System.Collections.Generic;
using System.Xml;
using System.Xml.Linq;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Common.Serializer;
namespace NzbDrone.Core.Indexers.Newznab
{
public interface INewznabCapabilitiesProvider
{
NewznabCapabilities GetCapabilities(NewznabSettings settings);
}
public class NewznabCapabilitiesProvider : INewznabCapabilitiesProvider
{
private readonly ICached<NewznabCapabilities> _capabilitiesCache;
private readonly IHttpClient _httpClient;
private readonly Logger _logger;
public NewznabCapabilitiesProvider(ICacheManager cacheManager, IHttpClient httpClient, Logger logger)
{
_capabilitiesCache = cacheManager.GetCache<NewznabCapabilities>(GetType());
_httpClient = httpClient;
_logger = logger;
}
public NewznabCapabilities GetCapabilities(NewznabSettings indexerSettings)
{
var key = indexerSettings.ToJson();
var capabilities = _capabilitiesCache.Get(key, () => FetchCapabilities(indexerSettings), TimeSpan.FromDays(7));
return capabilities;
}
private NewznabCapabilities FetchCapabilities(NewznabSettings indexerSettings)
{
var capabilities = new NewznabCapabilities();
var url = string.Format("{0}{1}?t=caps", indexerSettings.BaseUrl.TrimEnd('/'), indexerSettings.ApiPath.TrimEnd('/'));
if (indexerSettings.ApiKey.IsNotNullOrWhiteSpace())
{
url += "&apikey=" + indexerSettings.ApiKey;
}
var request = new HttpRequest(url, HttpAccept.Rss);
HttpResponse response;
try
{
response = _httpClient.Get(request);
}
catch (Exception ex)
{
_logger.Debug(ex, "Failed to get Newznab API capabilities from {0}", indexerSettings.BaseUrl);
throw;
}
try
{
capabilities = ParseCapabilities(response);
}
catch (XmlException ex)
{
ex.WithData(response, 128 * 1024);
_logger.Trace("Unexpected Response content ({0} bytes): {1}", response.ResponseData.Length, response.Content);
_logger.Debug(ex, "Failed to parse newznab api capabilities for {0}", indexerSettings.BaseUrl);
throw;
}
catch (Exception ex)
{
ex.WithData(response, 128 * 1024);
_logger.Trace("Unexpected Response content ({0} bytes): {1}", response.ResponseData.Length, response.Content);
}
return capabilities;
}
private NewznabCapabilities ParseCapabilities(HttpResponse response)
{
var capabilities = new NewznabCapabilities();
var xDoc = XDocument.Parse(response.Content);
if (xDoc == null)
{
throw new XmlException("Invalid XML").WithData(response);
}
NewznabRssParser.CheckError(xDoc, new IndexerResponse(new IndexerRequest(response.Request), response));
var xmlRoot = xDoc.Element("caps");
if (xmlRoot == null)
{
throw new XmlException("Unexpected XML").WithData(response);
}
var xmlLimits = xmlRoot.Element("limits");
if (xmlLimits != null)
{
capabilities.DefaultPageSize = int.Parse(xmlLimits.Attribute("default").Value);
capabilities.MaxPageSize = int.Parse(xmlLimits.Attribute("max").Value);
}
var xmlSearching = xmlRoot.Element("searching");
if (xmlSearching != null)
{
var xmlBasicSearch = xmlSearching.Element("search");
if (xmlBasicSearch == null || xmlBasicSearch.Attribute("available").Value != "yes")
{
capabilities.SupportedSearchParameters = null;
}
else if (xmlBasicSearch.Attribute("supportedParams") != null)
{
capabilities.SupportedSearchParameters = xmlBasicSearch.Attribute("supportedParams").Value.Split(',');
}
var xmlMovieSearch = xmlSearching.Element("movie-search");
if (xmlMovieSearch == null || xmlMovieSearch.Attribute("available").Value != "yes")
{
capabilities.SupportedMovieSearchParameters = null;
}
else if (xmlMovieSearch.Attribute("supportedParams") != null)
{
capabilities.SupportedMovieSearchParameters = xmlMovieSearch.Attribute("supportedParams").Value.Split(',');
capabilities.SupportsAggregateIdSearch = true;
}
}
var xmlCategories = xmlRoot.Element("categories");
if (xmlCategories != null)
{
foreach (var xmlCategory in xmlCategories.Elements("category"))
{
var cat = new NewznabCategory
{
Id = int.Parse(xmlCategory.Attribute("id").Value),
Name = xmlCategory.Attribute("name").Value,
Description = xmlCategory.Attribute("description") != null ? xmlCategory.Attribute("description").Value : string.Empty,
Subcategories = new List<NewznabCategory>()
};
foreach (var xmlSubcat in xmlCategory.Elements("subcat"))
{
cat.Subcategories.Add(new NewznabCategory
{
Id = int.Parse(xmlSubcat.Attribute("id").Value),
Name = xmlSubcat.Attribute("name").Value,
Description = xmlSubcat.Attribute("description") != null ? xmlSubcat.Attribute("description").Value : string.Empty
});
}
capabilities.Categories.Add(cat);
}
}
return capabilities;
}
}
}
@@ -0,0 +1,17 @@
using NzbDrone.Core.Indexers.Exceptions;
namespace NzbDrone.Core.Indexers.Newznab
{
public class NewznabException : IndexerException
{
public NewznabException(IndexerResponse response, string message, params object[] args)
: base(response, message, args)
{
}
public NewznabException(IndexerResponse response, string message)
: base(response, message)
{
}
}
}
@@ -0,0 +1,192 @@
using System;
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.Newznab
{
public class NewznabRequestGenerator : IIndexerRequestGenerator
{
private readonly INewznabCapabilitiesProvider _capabilitiesProvider;
public int MaxPages { get; set; }
public int PageSize { get; set; }
public NewznabSettings Settings { get; set; }
public NewznabRequestGenerator(INewznabCapabilitiesProvider capabilitiesProvider)
{
_capabilitiesProvider = capabilitiesProvider;
MaxPages = 30;
PageSize = 100;
}
private bool SupportsSearch
{
get
{
var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
return capabilities.SupportedSearchParameters != null &&
capabilities.SupportedSearchParameters.Contains("q");
}
}
private bool SupportsImdbSearch
{
get
{
var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
return capabilities.SupportedMovieSearchParameters != null &&
capabilities.SupportedMovieSearchParameters.Contains("imdbid");
}
}
private bool SupportsTmdbSearch
{
get
{
var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
return capabilities.SupportedMovieSearchParameters != null &&
capabilities.SupportedMovieSearchParameters.Contains("tmdbid");
}
}
private bool SupportsAggregatedIdSearch
{
get
{
var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
return capabilities.SupportsAggregateIdSearch;
}
}
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
// Some indexers might forget to enable movie search, but normal search still works fine. Thus we force a normal search.
if (capabilities.SupportedMovieSearchParameters != null)
{
pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "movie", ""));
}
else if (capabilities.SupportedSearchParameters != null)
{
pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "search", ""));
}
return pageableRequests;
}
public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
AddMovieIdPageableRequests(pageableRequests, MaxPages, Settings.Categories, searchCriteria);
return pageableRequests;
}
private void AddMovieIdPageableRequests(IndexerPageableRequestChain chain, int maxPages, IEnumerable<int> categories, SearchCriteriaBase searchCriteria)
{
var includeTmdbSearch = SupportsTmdbSearch && searchCriteria.TmdbId > 0;
var includeImdbSearch = SupportsImdbSearch && searchCriteria.ImdbId.IsNotNullOrWhiteSpace();
if (SupportsAggregatedIdSearch && (includeTmdbSearch || includeImdbSearch))
{
var ids = "";
if (includeTmdbSearch)
{
ids += "&tmdbid=" + searchCriteria.TmdbId;
}
if (includeImdbSearch)
{
ids += "&imdbid=" + searchCriteria.ImdbId.Substring(2);
}
chain.Add(GetPagedRequests(maxPages, categories, "movie", ids));
}
else
{
if (includeTmdbSearch)
{
chain.Add(GetPagedRequests(maxPages,
categories,
"movie",
string.Format("&tmdbid={0}", searchCriteria.TmdbId)));
}
else if (includeImdbSearch)
{
chain.Add(GetPagedRequests(maxPages,
categories,
"movie",
string.Format("&imdbid={0}", searchCriteria.ImdbId.Substring(2))));
}
}
if (SupportsSearch)
{
chain.AddTier();
foreach (var queryTitle in searchCriteria.QueryTitles)
{
var searchQuery = queryTitle;
if (!Settings.RemoveYear)
{
searchQuery = string.Format("{0}", searchQuery);
}
chain.Add(GetPagedRequests(MaxPages,
Settings.Categories,
"movie",
string.Format("&q={0}", NewsnabifyTitle(searchQuery))));
}
}
}
private IEnumerable<IndexerRequest> GetPagedRequests(int maxPages, IEnumerable<int> categories, string searchType, string parameters)
{
if (categories.Empty())
{
yield break;
}
var categoriesQuery = string.Join(",", categories.Distinct());
var baseUrl = string.Format("{0}{1}?t={2}&cat={3}&extended=1{4}", Settings.BaseUrl.TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchType, categoriesQuery, Settings.AdditionalParameters);
if (Settings.ApiKey.IsNotNullOrWhiteSpace())
{
baseUrl += "&apikey=" + Settings.ApiKey;
}
if (PageSize == 0)
{
yield return new IndexerRequest(string.Format("{0}{1}", baseUrl, parameters), HttpAccept.Rss);
}
else
{
for (var page = 0; page < maxPages; page++)
{
yield return new IndexerRequest(string.Format("{0}&offset={1}&limit={2}{3}", baseUrl, page * PageSize, PageSize, parameters), HttpAccept.Rss);
}
}
}
private static string NewsnabifyTitle(string title)
{
return title.Replace("+", "%20");
}
public Func<IDictionary<string, string>> GetCookies { get; set; }
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
}
}
@@ -0,0 +1,192 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Xml.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Newznab
{
public class NewznabRssParser : RssParser
{
public const string ns = "{http://www.newznab.com/DTD/2010/feeds/attributes/}";
private readonly NewznabSettings _settings;
public NewznabRssParser(NewznabSettings settings)
{
PreferredEnclosureMimeTypes = UsenetEnclosureMimeTypes;
UseEnclosureUrl = true;
_settings = settings;
}
public static void CheckError(XDocument xdoc, IndexerResponse indexerResponse)
{
var error = xdoc.Descendants("error").FirstOrDefault();
if (error == null)
{
return;
}
var code = Convert.ToInt32(error.Attribute("code").Value);
var errorMessage = error.Attribute("description").Value;
if (code >= 100 && code <= 199)
{
throw new ApiKeyException(errorMessage);
}
if (!indexerResponse.Request.Url.FullUri.Contains("apikey=") && (errorMessage == "Missing parameter" || errorMessage.Contains("apikey")))
{
throw new ApiKeyException("Indexer requires an API key");
}
if (errorMessage == "Request limit reached")
{
throw new RequestLimitReachedException("API limit reached");
}
throw new NewznabException(indexerResponse, errorMessage);
}
protected override bool PreProcess(IndexerResponse indexerResponse)
{
var xdoc = LoadXmlDocument(indexerResponse);
CheckError(xdoc, indexerResponse);
return true;
}
protected override bool PostProcess(IndexerResponse indexerResponse, List<XElement> items, List<ReleaseInfo> releases)
{
var enclosureTypes = items.SelectMany(GetEnclosures).Select(v => v.Type).Distinct().ToArray();
if (enclosureTypes.Any() && enclosureTypes.Intersect(PreferredEnclosureMimeTypes).Empty())
{
if (enclosureTypes.Intersect(TorrentEnclosureMimeTypes).Any())
{
_logger.Warn("Feed does not contain {0}, found {1}, did you intend to add a Torznab indexer?", NzbEnclosureMimeType, enclosureTypes[0]);
}
else
{
_logger.Warn("Feed does not contain {0}, found {1}.", NzbEnclosureMimeType, enclosureTypes[0]);
}
}
return true;
}
protected override ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo)
{
releaseInfo = base.ProcessItem(item, releaseInfo);
releaseInfo.ImdbId = GetImdbId(item);
return releaseInfo;
}
protected override string GetInfoUrl(XElement item)
{
return ParseUrl(item.TryGetValue("comments").TrimEnd("#comments"));
}
protected override string GetCommentUrl(XElement item)
{
return ParseUrl(item.TryGetValue("comments"));
}
protected override long GetSize(XElement item)
{
long size;
var sizeString = TryGetNewznabAttribute(item, "size");
if (!sizeString.IsNullOrWhiteSpace() && long.TryParse(sizeString, out size))
{
return size;
}
size = GetEnclosureLength(item);
return size;
}
protected override DateTime GetPublishDate(XElement item)
{
var dateString = TryGetNewznabAttribute(item, "usenetdate");
if (!dateString.IsNullOrWhiteSpace())
{
return XElementExtensions.ParseDate(dateString);
}
return base.GetPublishDate(item);
}
protected override string GetDownloadUrl(XElement item)
{
var url = base.GetDownloadUrl(item);
if (!Uri.IsWellFormedUriString(url, UriKind.Absolute))
{
url = ParseUrl((string)item.Element("enclosure").Attribute("url"));
}
return url;
}
protected virtual int GetImdbId(XElement item)
{
var imdbIdString = TryGetNewznabAttribute(item, "imdb");
int imdbId;
if (!imdbIdString.IsNullOrWhiteSpace() && int.TryParse(imdbIdString, out imdbId))
{
return imdbId;
}
return 0;
}
protected virtual string GetImdbTitle(XElement item)
{
var imdbTitle = TryGetNewznabAttribute(item, "imdbtitle");
if (!imdbTitle.IsNullOrWhiteSpace())
{
return CultureInfo.CurrentCulture.TextInfo.ToTitleCase(
Parser.Parser.ReplaceGermanUmlauts(
Parser.Parser.NormalizeTitle(imdbTitle).Replace(" ", ".")));
}
return string.Empty;
}
protected virtual int GetImdbYear(XElement item)
{
var imdbYearString = TryGetNewznabAttribute(item, "imdbyear");
int imdbYear;
if (!imdbYearString.IsNullOrWhiteSpace() && int.TryParse(imdbYearString, out imdbYear))
{
return imdbYear;
}
return 1900;
}
protected string TryGetNewznabAttribute(XElement item, string key, string defaultValue = "")
{
var attrElement = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase));
if (attrElement != null)
{
var attrValue = attrElement.Attribute("value");
if (attrValue != null)
{
return attrValue.Value;
}
}
return defaultValue;
}
}
}
@@ -0,0 +1,98 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using FluentValidation;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Newznab
{
public class NewznabSettingsValidator : AbstractValidator<NewznabSettings>
{
private static readonly string[] ApiKeyWhiteList =
{
"nzbs.org",
"nzb.su",
"dognzb.cr",
"nzbplanet.net",
"nzbid.org",
"nzbndx.com",
"nzbindex.in"
};
private static bool ShouldHaveApiKey(NewznabSettings settings)
{
if (settings.BaseUrl == null)
{
return false;
}
return ApiKeyWhiteList.Any(c => settings.BaseUrl.ToLowerInvariant().Contains(c));
}
private static readonly Regex AdditionalParametersRegex = new Regex(@"(&.+?\=.+?)+", RegexOptions.Compiled);
public NewznabSettingsValidator()
{
RuleFor(c => c).Custom((c, context) =>
{
if (c.Categories.Empty())
{
context.AddFailure("'Categories' must be provided");
}
});
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.ApiPath).ValidUrlBase("/api");
RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey);
RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex)
.When(c => !c.AdditionalParameters.IsNullOrWhiteSpace());
}
}
public class NewznabSettings : IIndexerSettings
{
private static readonly NewznabSettingsValidator Validator = new NewznabSettingsValidator();
public NewznabSettings()
{
ApiPath = "/api";
Categories = new[] { 2000, 2010, 2020, 2030, 2035, 2040, 2045, 2050, 2060 };
MultiLanguages = new List<int>();
}
[FieldDefinition(0, Label = "URL")]
public string BaseUrl { get; set; }
[FieldDefinition(1, Label = "API Path", HelpText = "Path to the api, usually /api", Advanced = true)]
public string ApiPath { get; set; }
[FieldDefinition(1, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
public IEnumerable<int> MultiLanguages { get; set; }
[FieldDefinition(2, Label = "API Key", Privacy = PrivacyLevel.ApiKey)]
public string ApiKey { get; set; }
[FieldDefinition(3, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable all categories", Advanced = true)]
public IEnumerable<int> Categories { get; set; }
[FieldDefinition(5, Label = "Additional Parameters", HelpText = "Additional Newznab parameters", Advanced = true)]
public string AdditionalParameters { get; set; }
[FieldDefinition(6,
Label = "Remove year from search string",
HelpText = "Should Prowlarr remove the year after the title when searching this indexer?",
Advanced = true,
Type = FieldType.Checkbox)]
public bool RemoveYear { get; set; }
// Field 8 is used by TorznabSettings MinimumSeeders
// If you need to add another field here, update TorznabSettings as well and this comment
public virtual NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}
@@ -0,0 +1,31 @@
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
namespace NzbDrone.Core.Indexers.Nyaa
{
public class Nyaa : HttpIndexerBase<NyaaSettings>
{
public override string Name => "Nyaa";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override int PageSize => 100;
public Nyaa(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, indexerStatusService, configService, logger)
{
}
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new NyaaRequestGenerator() { Settings = Settings, PageSize = PageSize };
}
public override IParseIndexerResponse GetParser()
{
return new TorrentRssParser() { UseGuidInfoUrl = true, ParseSizeInDescription = true, ParseSeedersInDescription = true };
}
}
}
@@ -0,0 +1,74 @@
using System;
using System.Collections.Generic;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.Nyaa
{
public class NyaaRequestGenerator : IIndexerRequestGenerator
{
public NyaaSettings Settings { get; set; }
public int MaxPages { get; set; }
public int PageSize { get; set; }
public NyaaRequestGenerator()
{
MaxPages = 30;
PageSize = 100;
}
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(MaxPages, null));
return pageableRequests;
}
private IEnumerable<IndexerRequest> GetPagedRequests(int maxPages, string term)
{
var baseUrl = string.Format("{0}/?page=rss{1}", Settings.BaseUrl.TrimEnd('/'), Settings.AdditionalParameters);
if (term != null)
{
baseUrl += "&term=" + term;
}
if (PageSize == 0)
{
yield return new IndexerRequest(baseUrl, HttpAccept.Rss);
}
else
{
yield return new IndexerRequest(baseUrl, HttpAccept.Rss);
for (var page = 1; page < maxPages; page++)
{
yield return new IndexerRequest($"{baseUrl}&offset={page + 1}", HttpAccept.Rss);
}
}
}
private string PrepareQuery(string query)
{
return query.Replace(' ', '+');
}
public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
foreach (var queryTitle in searchCriteria.QueryTitles)
{
pageableRequests.Add(GetPagedRequests(MaxPages, PrepareQuery(string.Format("{0}", queryTitle))));
}
return pageableRequests;
}
public Func<IDictionary<string, string>> GetCookies { get; set; }
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
}
}
@@ -0,0 +1,53 @@
using System.Collections.Generic;
using System.Text.RegularExpressions;
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Nyaa
{
public class NyaaSettingsValidator : AbstractValidator<NyaaSettings>
{
public NyaaSettingsValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.AdditionalParameters).Matches("(&[a-z]+=[a-z0-9_]+)*", RegexOptions.IgnoreCase);
}
}
public class NyaaSettings : ITorrentIndexerSettings
{
private static readonly NyaaSettingsValidator Validator = new NyaaSettingsValidator();
public NyaaSettings()
{
BaseUrl = "";
AdditionalParameters = "";
MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
MultiLanguages = new List<int>();
RequiredFlags = new List<int>();
}
[FieldDefinition(0, Label = "Website URL")]
public string BaseUrl { get; set; }
[FieldDefinition(1, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
public IEnumerable<int> MultiLanguages { get; set; }
[FieldDefinition(2, Label = "Additional Parameters", Advanced = true, HelpText = "Please note if you change the category you will have to add required/restricted rules about the subgroups to avoid foreign language releases.")]
public string AdditionalParameters { get; set; }
[FieldDefinition(3, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
public int MinimumSeeders { get; set; }
[FieldDefinition(4, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", HelpLink = "https://github.com/Prowlarr/Prowlarr/wiki/Indexer-Flags#1-required-flags", Advanced = true)]
public IEnumerable<int> RequiredFlags { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}
@@ -0,0 +1,51 @@
using System.Collections.Generic;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
namespace NzbDrone.Core.Indexers.PassThePopcorn
{
public class PassThePopcorn : HttpIndexerBase<PassThePopcornSettings>
{
public override string Name => "PassThePopcorn";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override bool SupportsRss => true;
public override bool SupportsSearch => true;
public override IndexerCapabilities Capabilities => new IndexerCapabilities
{
MovieSearchParams = new List<MovieSearchParam>
{
MovieSearchParam.Q, MovieSearchParam.ImdbId
}
};
public override int PageSize => 50;
public PassThePopcorn(IHttpClient httpClient,
ICacheManager cacheManager,
IIndexerStatusService indexerStatusService,
IConfigService configService,
Logger logger)
: base(httpClient, indexerStatusService, configService, logger)
{
}
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new PassThePopcornRequestGenerator()
{
Settings = Settings,
HttpClient = _httpClient,
Logger = _logger,
};
}
public override IParseIndexerResponse GetParser()
{
return new PassThePopcornParser(Settings, _logger);
}
}
}
@@ -0,0 +1,65 @@
using System;
using System.Collections.Generic;
namespace NzbDrone.Core.Indexers.PassThePopcorn
{
public class Director
{
public string Name { get; set; }
public string Id { get; set; }
}
public class Torrent
{
public int Id { get; set; }
public string Quality { get; set; }
public string Source { get; set; }
public string Container { get; set; }
public string Codec { get; set; }
public string Resolution { get; set; }
public bool Scene { get; set; }
public string Size { get; set; }
public DateTime UploadTime { get; set; }
public string RemasterTitle { get; set; }
public string Snatched { get; set; }
public string Seeders { get; set; }
public string Leechers { get; set; }
public string ReleaseName { get; set; }
public bool Checked { get; set; }
public bool GoldenPopcorn { get; set; }
public string FreeleechType { get; set; }
}
public class Movie
{
public string GroupId { get; set; }
public string Title { get; set; }
public string Year { get; set; }
public string Cover { get; set; }
public List<string> Tags { get; set; }
public List<Director> Directors { get; set; }
public string ImdbId { get; set; }
public int TotalLeechers { get; set; }
public int TotalSeeders { get; set; }
public int TotalSnatched { get; set; }
public long MaxSize { get; set; }
public string LastUploadTime { get; set; }
public List<Torrent> Torrents { get; set; }
}
public class PassThePopcornResponse
{
public string TotalResults { get; set; }
public List<Movie> Movies { get; set; }
public string Page { get; set; }
public string AuthKey { get; set; }
public string PassKey { get; set; }
}
public class PassThePopcornAuthResponse
{
public string Result { get; set; }
public string Popcron { get; set; }
public string AntiCsrfToken { get; set; }
}
}
@@ -0,0 +1,11 @@
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.PassThePopcorn
{
public class PassThePopcornInfo : TorrentInfo
{
public bool? Golden { get; set; }
public bool? Scene { get; set; }
public bool? Approved { get; set; }
}
}
@@ -0,0 +1,151 @@
using System;
using System.Collections.Generic;
using System.Net;
using Newtonsoft.Json;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.PassThePopcorn
{
public class PassThePopcornParser : IParseIndexerResponse
{
private readonly PassThePopcornSettings _settings;
private readonly Logger _logger;
public PassThePopcornParser(PassThePopcornSettings settings, Logger logger)
{
_settings = settings;
_logger = logger;
}
public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var torrentInfos = new List<ReleaseInfo>();
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
// Remove cookie cache
if (indexerResponse.HttpResponse.HasHttpRedirect && indexerResponse.HttpResponse.Headers["Location"]
.ContainsIgnoreCase("login.php"))
{
CookiesUpdater(null, null);
throw new IndexerException(indexerResponse, "We are being redirected to the PTP login page. Most likely your session expired or was killed. Try testing the indexer in the settings.");
}
throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request");
}
if (indexerResponse.HttpResponse.Headers.ContentType != HttpAccept.Json.Value)
{
if (indexerResponse.HttpResponse.Request.Url.Path.ContainsIgnoreCase("login.php"))
{
CookiesUpdater(null, null);
throw new IndexerException(indexerResponse, "We are currently on the login page. Most likely your session expired or was killed. Try testing the indexer in the settings.");
}
// Remove cookie cache
throw new IndexerException(indexerResponse, $"Unexpected response header {indexerResponse.HttpResponse.Headers.ContentType} from API request, expected {HttpAccept.Json.Value}");
}
var jsonResponse = JsonConvert.DeserializeObject<PassThePopcornResponse>(indexerResponse.Content);
if (jsonResponse.TotalResults == "0" ||
jsonResponse.TotalResults.IsNullOrWhiteSpace() ||
jsonResponse.Movies == null)
{
return torrentInfos;
}
foreach (var result in jsonResponse.Movies)
{
foreach (var torrent in result.Torrents)
{
var id = torrent.Id;
var title = torrent.ReleaseName;
IndexerFlags flags = 0;
if (torrent.GoldenPopcorn)
{
flags |= IndexerFlags.PTP_Golden; //title = $"{title} 🍿";
}
if (torrent.Checked)
{
flags |= IndexerFlags.PTP_Approved; //title = $"{title} ✔";
}
if (torrent.FreeleechType == "Freeleech")
{
flags |= IndexerFlags.G_Freeleech;
}
if (torrent.Scene)
{
flags |= IndexerFlags.G_Scene;
}
// Only add approved torrents
try
{
torrentInfos.Add(new PassThePopcornInfo()
{
Guid = string.Format("PassThePopcorn-{0}", id),
Title = title,
Size = long.Parse(torrent.Size),
DownloadUrl = GetDownloadUrl(id, jsonResponse.AuthKey, jsonResponse.PassKey),
InfoUrl = GetInfoUrl(result.GroupId, id),
Seeders = int.Parse(torrent.Seeders),
Peers = int.Parse(torrent.Leechers) + int.Parse(torrent.Seeders),
PublishDate = torrent.UploadTime.ToUniversalTime(),
Golden = torrent.GoldenPopcorn,
Scene = torrent.Scene,
Approved = torrent.Checked,
ImdbId = result.ImdbId.IsNotNullOrWhiteSpace() ? int.Parse(result.ImdbId) : 0,
IndexerFlags = flags
});
}
catch (Exception e)
{
_logger.Error(e, "Encountered exception parsing PTP torrent: {" +
$"Size: {torrent.Size}" +
$"UploadTime: {torrent.UploadTime}" +
$"Seeders: {torrent.Seeders}" +
$"Leechers: {torrent.Leechers}" +
$"ReleaseName: {torrent.ReleaseName}" +
$"ID: {torrent.Id}" +
"}. Please immediately report this info on https://github.com/Prowlarr/Prowlarr/issues/1584.");
throw;
}
}
}
return
torrentInfos;
}
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
private string GetDownloadUrl(int torrentId, string authKey, string passKey)
{
var url = new HttpUri(_settings.BaseUrl)
.CombinePath("/torrents.php")
.AddQueryParam("action", "download")
.AddQueryParam("id", torrentId)
.AddQueryParam("authkey", authKey)
.AddQueryParam("torrent_pass", passKey);
return url.FullUri;
}
private string GetInfoUrl(string groupId, int torrentId)
{
var url = new HttpUri(_settings.BaseUrl)
.CombinePath("/torrents.php")
.AddQueryParam("id", groupId)
.AddQueryParam("torrentid", torrentId);
return url.FullUri;
}
}
}
@@ -0,0 +1,73 @@
using System;
using System.Collections.Generic;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.PassThePopcorn
{
public class PassThePopcornRequestGenerator : IIndexerRequestGenerator
{
public PassThePopcornSettings Settings { get; set; }
public IDictionary<string, string> Cookies { get; set; }
public IHttpClient HttpClient { get; set; }
public Logger Logger { get; set; }
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetRequest(null));
return pageableRequests;
}
public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace())
{
pageableRequests.Add(GetRequest(searchCriteria.ImdbId));
}
else
{
foreach (var queryTitle in searchCriteria.QueryTitles)
{
pageableRequests.Add(GetRequest(string.Format("{0}", queryTitle)));
}
}
return pageableRequests;
}
public Func<IDictionary<string, string>> GetCookies { get; set; }
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
private IEnumerable<IndexerRequest> GetRequest(string searchParameters)
{
var request =
new IndexerRequest(
$"{Settings.BaseUrl.Trim().TrimEnd('/')}/torrents.php?action=advanced&json=noredirect&searchstr={searchParameters}",
HttpAccept.Json);
request.HttpRequest.Headers["ApiUser"] = Settings.APIUser;
request.HttpRequest.Headers["ApiKey"] = Settings.APIKey;
if (Settings.APIKey.IsNullOrWhiteSpace())
{
foreach (var cookie in Cookies)
{
request.HttpRequest.Cookies[cookie.Key] = cookie.Value;
}
CookiesUpdater(Cookies, DateTime.Now + TimeSpan.FromDays(30));
}
yield return request;
}
}
}
@@ -0,0 +1,55 @@
using System.Collections.Generic;
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.PassThePopcorn
{
public class PassThePopcornSettingsValidator : AbstractValidator<PassThePopcornSettings>
{
public PassThePopcornSettingsValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.APIUser).NotEmpty();
RuleFor(c => c.APIKey).NotEmpty();
}
}
public class PassThePopcornSettings : ITorrentIndexerSettings
{
private static readonly PassThePopcornSettingsValidator Validator = new PassThePopcornSettingsValidator();
public PassThePopcornSettings()
{
BaseUrl = "https://passthepopcorn.me";
MinimumSeeders = 0;
MultiLanguages = new List<int>();
RequiredFlags = new List<int>();
}
[FieldDefinition(0, Label = "URL", Advanced = true, HelpText = "Do not change this unless you know what you're doing. Since your cookie will be sent to that host.")]
public string BaseUrl { get; set; }
[FieldDefinition(1, Label = "APIUser", HelpText = "These settings are found in your PassThePopcorn security settings (Edit Profile > Security).", Privacy = PrivacyLevel.UserName)]
public string APIUser { get; set; }
[FieldDefinition(2, Label = "APIKey", Type = FieldType.Password, Privacy = PrivacyLevel.Password)]
public string APIKey { get; set; }
[FieldDefinition(3, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
public IEnumerable<int> MultiLanguages { get; set; }
[FieldDefinition(4, Type = FieldType.Textbox, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
public int MinimumSeeders { get; set; }
[FieldDefinition(6, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", HelpLink = "https://github.com/Prowlarr/Prowlarr/wiki/Indexer-Flags#1-required-flags", Advanced = true)]
public IEnumerable<int> RequiredFlags { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}
@@ -0,0 +1,113 @@
using System;
using System.Collections.Generic;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Http.CloudFlare;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Rarbg
{
public class Rarbg : HttpIndexerBase<RarbgSettings>
{
private readonly IRarbgTokenProvider _tokenProvider;
public override string Name => "Rarbg";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Public;
public override TimeSpan RateLimit => TimeSpan.FromSeconds(2);
public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, indexerStatusService, configService, logger)
{
_tokenProvider = tokenProvider;
}
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new RarbgRequestGenerator(_tokenProvider) { Settings = Settings };
}
public override IParseIndexerResponse GetParser()
{
return new RarbgParser();
}
public override object RequestAction(string action, IDictionary<string, string> query)
{
if (action == "checkCaptcha")
{
Settings.Validate().Filter("BaseUrl").ThrowOnError();
try
{
var request = new HttpRequestBuilder(Settings.BaseUrl.Trim('/'))
.Resource("/pubapi_v2.php?get_token=get_token")
.Accept(HttpAccept.Json)
.Build();
_httpClient.Get(request);
}
catch (CloudFlareCaptchaException ex)
{
return new
{
captchaRequest = new
{
host = ex.CaptchaRequest.Host,
ray = ex.CaptchaRequest.Ray,
siteKey = ex.CaptchaRequest.SiteKey,
secretToken = ex.CaptchaRequest.SecretToken,
responseUrl = ex.CaptchaRequest.ResponseUrl.FullUri,
}
};
}
return new
{
captchaToken = ""
};
}
else if (action == "getCaptchaCookie")
{
if (query["responseUrl"].IsNullOrWhiteSpace())
{
throw new BadRequestException("QueryParam responseUrl invalid.");
}
if (query["ray"].IsNullOrWhiteSpace())
{
throw new BadRequestException("QueryParam ray invalid.");
}
if (query["captchaResponse"].IsNullOrWhiteSpace())
{
throw new BadRequestException("QueryParam captchaResponse invalid.");
}
var request = new HttpRequestBuilder(query["responseUrl"])
.AddQueryParam("id", query["ray"])
.AddQueryParam("g-recaptcha-response", query["captchaResponse"])
.Build();
request.UseSimplifiedUserAgent = true;
request.AllowAutoRedirect = false;
var response = _httpClient.Get(request);
var cfClearanceCookie = response.GetCookies()["cf_clearance"];
return new
{
captchaToken = cfClearanceCookie
};
}
return new { };
}
}
}
@@ -0,0 +1,97 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Text.RegularExpressions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Rarbg
{
public class RarbgParser : IParseIndexerResponse
{
private static readonly Regex RegexGuid = new Regex(@"^magnet:\?xt=urn:btih:([a-f0-9]+)", RegexOptions.Compiled);
public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var results = new List<ReleaseInfo>();
switch (indexerResponse.HttpResponse.StatusCode)
{
default:
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
}
break;
}
var jsonResponse = new HttpResponse<RarbgResponse>(indexerResponse.HttpResponse);
if (jsonResponse.Resource.error_code.HasValue)
{
if (jsonResponse.Resource.error_code == 20 || jsonResponse.Resource.error_code == 8
|| jsonResponse.Resource.error_code == 9 || jsonResponse.Resource.error_code == 10)
{
// No results or imdbid not found
return results;
}
throw new IndexerException(indexerResponse, "Indexer API call returned error {0}: {1}", jsonResponse.Resource.error_code, jsonResponse.Resource.error);
}
if (jsonResponse.Resource.torrent_results == null)
{
return results;
}
foreach (var torrent in jsonResponse.Resource.torrent_results)
{
var torrentInfo = new TorrentInfo();
torrentInfo.Guid = GetGuid(torrent);
torrentInfo.Title = torrent.title;
torrentInfo.Size = torrent.size;
torrentInfo.DownloadUrl = torrent.download;
torrentInfo.InfoUrl = torrent.info_page + "&app_id=Prowlarr";
torrentInfo.PublishDate = torrent.pubdate.ToUniversalTime();
torrentInfo.Seeders = torrent.seeders;
torrentInfo.Peers = torrent.leechers + torrent.seeders;
if (torrent.movie_info != null)
{
if (torrent.movie_info.tvdb != null)
{
torrentInfo.TvdbId = torrent.movie_info.tvdb.Value;
}
if (torrent.movie_info.tvrage != null)
{
torrentInfo.TvRageId = torrent.movie_info.tvrage.Value;
}
}
results.Add(torrentInfo);
}
return results;
}
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
private string GetGuid(RarbgTorrent torrent)
{
var match = RegexGuid.Match(torrent.download);
if (match.Success)
{
return string.Format("rarbg-{0}", match.Groups[1].Value);
}
else
{
return string.Format("rarbg-{0}", torrent.download);
}
}
}
}
@@ -0,0 +1,122 @@
using System;
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.Rarbg
{
public class RarbgRequestGenerator : IIndexerRequestGenerator
{
private readonly IRarbgTokenProvider _tokenProvider;
public RarbgSettings Settings { get; set; }
public RarbgRequestGenerator(IRarbgTokenProvider tokenProvider)
{
_tokenProvider = tokenProvider;
}
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests("list", null, null));
return pageableRequests;
}
public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetMovieRequest(searchCriteria));
return pageableRequests;
}
private IEnumerable<IndexerRequest> GetPagedRequests(string mode, int? imdbId, string query, params object[] args)
{
var requestBuilder = new HttpRequestBuilder(Settings.BaseUrl)
.Resource("/pubapi_v2.php")
.Accept(HttpAccept.Json);
if (Settings.CaptchaToken.IsNotNullOrWhiteSpace())
{
requestBuilder.UseSimplifiedUserAgent = true;
requestBuilder.SetCookie("cf_clearance", Settings.CaptchaToken);
}
requestBuilder.AddQueryParam("mode", mode);
if (imdbId.HasValue)
{
requestBuilder.AddQueryParam("search_imdb", imdbId.Value);
}
if (query.IsNotNullOrWhiteSpace())
{
requestBuilder.AddQueryParam("search_string", string.Format(query, args));
}
if (!Settings.RankedOnly)
{
requestBuilder.AddQueryParam("ranked", "0");
}
var categoryParam = string.Join(";", Settings.Categories.Distinct());
requestBuilder.AddQueryParam("category", categoryParam);
requestBuilder.AddQueryParam("limit", "100");
requestBuilder.AddQueryParam("token", _tokenProvider.GetToken(Settings));
requestBuilder.AddQueryParam("format", "json_extended");
requestBuilder.AddQueryParam("app_id", "Prowlarr");
yield return new IndexerRequest(requestBuilder.Build());
}
private IEnumerable<IndexerRequest> GetMovieRequest(MovieSearchCriteria searchCriteria)
{
var requestBuilder = new HttpRequestBuilder(Settings.BaseUrl)
.Resource("/pubapi_v2.php")
.Accept(HttpAccept.Json);
if (Settings.CaptchaToken.IsNotNullOrWhiteSpace())
{
requestBuilder.UseSimplifiedUserAgent = true;
requestBuilder.SetCookie("cf_clearance", Settings.CaptchaToken);
}
requestBuilder.AddQueryParam("mode", "search");
if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace())
{
requestBuilder.AddQueryParam("search_imdb", searchCriteria.ImdbId);
}
else if (searchCriteria.TmdbId > 0)
{
requestBuilder.AddQueryParam("search_themoviedb", searchCriteria.TmdbId);
}
else if (searchCriteria.QueryTitles.Count > 0)
{
requestBuilder.AddQueryParam("search_string", $"{searchCriteria.QueryTitles.First()}");
}
if (!Settings.RankedOnly)
{
requestBuilder.AddQueryParam("ranked", "0");
}
var categoryParam = string.Join(";", Settings.Categories.Distinct());
requestBuilder.AddQueryParam("category", categoryParam);
requestBuilder.AddQueryParam("limit", "100");
requestBuilder.AddQueryParam("token", _tokenProvider.GetToken(Settings));
requestBuilder.AddQueryParam("format", "json_extended");
requestBuilder.AddQueryParam("app_id", BuildInfo.AppName);
yield return new IndexerRequest(requestBuilder.Build());
}
public Func<IDictionary<string, string>> GetCookies { get; set; }
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
}
}
@@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
namespace NzbDrone.Core.Indexers.Rarbg
{
public class RarbgResponse
{
public string error { get; set; }
public int? error_code { get; set; }
public List<RarbgTorrent> torrent_results { get; set; }
}
public class RarbgTorrent
{
public string title { get; set; }
public string category { get; set; }
public string download { get; set; }
public int? seeders { get; set; }
public int? leechers { get; set; }
public long size { get; set; }
public DateTime pubdate { get; set; }
public RarbgTorrentInfo movie_info { get; set; }
public int? ranked { get; set; }
public string info_page { get; set; }
}
public class RarbgTorrentInfo
{
public string imdb { get; set; }
public int? tvrage { get; set; }
public int? tvdb { get; set; }
}
}
@@ -0,0 +1,60 @@
using System.Collections.Generic;
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Rarbg
{
public class RarbgSettingsValidator : AbstractValidator<RarbgSettings>
{
public RarbgSettingsValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.Categories).NotEmpty();
}
}
public class RarbgSettings : ITorrentIndexerSettings
{
private static readonly RarbgSettingsValidator Validator = new RarbgSettingsValidator();
public RarbgSettings()
{
BaseUrl = "https://torrentapi.org";
RankedOnly = false;
MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
Categories = new[] { 14, 48, 17, 44, 45, 47, 50, 51, 52, 42, 46 };
MultiLanguages = new List<int>();
RequiredFlags = new List<int>();
}
[FieldDefinition(0, Label = "API URL", HelpText = "URL to Rarbg api, not the website.")]
public string BaseUrl { get; set; }
[FieldDefinition(1, Type = FieldType.Checkbox, Label = "Ranked Only", HelpText = "Only include ranked results.")]
public bool RankedOnly { get; set; }
[FieldDefinition(2, Type = FieldType.Captcha, Label = "CAPTCHA Token", HelpText = "CAPTCHA Clearance token used to handle CloudFlare Anti-DDOS measures on shared-ip VPNs.")]
public string CaptchaToken { get; set; }
[FieldDefinition(3, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
public IEnumerable<int> MultiLanguages { get; set; }
[FieldDefinition(4, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
public int MinimumSeeders { get; set; }
[FieldDefinition(5, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", HelpLink = "https://github.com/Prowlarr/Prowlarr/wiki/Indexer-Flags#1-required-flags", Advanced = true)]
public IEnumerable<int> RequiredFlags { get; set; }
[FieldDefinition(6, Type = FieldType.Textbox, Label = "Categories", HelpText = "Comma Separated list, you can retrieve the ID by checking the URL behind the category on the website (i.e. Movie/x264/1080 = 44)", HelpLink = "https://rarbgmirror.org/torrents.php?category=movies", Advanced = true)]
public IEnumerable<int> Categories { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}
@@ -0,0 +1,51 @@
using System;
using Newtonsoft.Json.Linq;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
namespace NzbDrone.Core.Indexers.Rarbg
{
public interface IRarbgTokenProvider
{
string GetToken(RarbgSettings settings);
}
public class RarbgTokenProvider : IRarbgTokenProvider
{
private readonly IHttpClient _httpClient;
private readonly ICached<string> _tokenCache;
private readonly Logger _logger;
public RarbgTokenProvider(IHttpClient httpClient, ICacheManager cacheManager, Logger logger)
{
_httpClient = httpClient;
_tokenCache = cacheManager.GetCache<string>(GetType());
_logger = logger;
}
public string GetToken(RarbgSettings settings)
{
return _tokenCache.Get(settings.BaseUrl,
() =>
{
var requestBuilder = new HttpRequestBuilder(settings.BaseUrl.Trim('/'))
.WithRateLimit(3.0)
.Resource("/pubapi_v2.php?get_token=get_token&app_id=Prowlarr")
.Accept(HttpAccept.Json);
if (settings.CaptchaToken.IsNotNullOrWhiteSpace())
{
requestBuilder.UseSimplifiedUserAgent = true;
requestBuilder.SetCookie("cf_clearance", settings.CaptchaToken);
}
var response = _httpClient.Get<JObject>(requestBuilder.Build());
return response.Resource["token"].ToString();
},
TimeSpan.FromMinutes(14.0));
}
}
}
@@ -0,0 +1,47 @@
using System;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
namespace NzbDrone.Core.Indexers.TorrentPotato
{
public class TorrentPotato : HttpIndexerBase<TorrentPotatoSettings>
{
public override string Name => "TorrentPotato";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override TimeSpan RateLimit => TimeSpan.FromSeconds(2);
public TorrentPotato(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, indexerStatusService, configService, logger)
{
}
private IndexerDefinition GetDefinition(string name, TorrentPotatoSettings settings)
{
return new IndexerDefinition
{
EnableRss = false,
EnableAutomaticSearch = false,
EnableInteractiveSearch = false,
Name = name,
Implementation = GetType().Name,
Settings = settings,
Protocol = DownloadProtocol.Torrent,
SupportsRss = SupportsRss,
SupportsSearch = SupportsSearch
};
}
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new TorrentPotatoRequestGenerator() { Settings = Settings };
}
public override IParseIndexerResponse GetParser()
{
return new TorrentPotatoParser();
}
}
}
@@ -0,0 +1,68 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Text.RegularExpressions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.TorrentPotato
{
public class TorrentPotatoParser : IParseIndexerResponse
{
private static readonly Regex RegexGuid = new Regex(@"^magnet:\?xt=urn:btih:([a-f0-9]+)", RegexOptions.Compiled);
public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var results = new List<ReleaseInfo>();
switch (indexerResponse.HttpResponse.StatusCode)
{
default:
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
}
break;
}
var jsonResponse = new HttpResponse<TorrentPotatoResponse>(indexerResponse.HttpResponse);
foreach (var torrent in jsonResponse.Resource.results)
{
var torrentInfo = new TorrentInfo();
torrentInfo.Guid = GetGuid(torrent);
torrentInfo.Title = torrent.release_name;
torrentInfo.Size = (long)torrent.size * 1000 * 1000;
torrentInfo.DownloadUrl = torrent.download_url;
torrentInfo.InfoUrl = torrent.details_url;
torrentInfo.PublishDate = torrent.publish_date.ToUniversalTime();
torrentInfo.Seeders = torrent.seeders;
torrentInfo.Peers = torrent.leechers + torrent.seeders;
torrentInfo.Freeleech = torrent.freeleech;
results.Add(torrentInfo);
}
return results;
}
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
private string GetGuid(Result torrent)
{
var match = RegexGuid.Match(torrent.download_url);
if (match.Success)
{
return string.Format("potato-{0}", match.Groups[1].Value);
}
else
{
return string.Format("potato-{0}", torrent.download_url);
}
}
}
}
@@ -0,0 +1,86 @@
using System;
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.TorrentPotato
{
public class TorrentPotatoRequestGenerator : IIndexerRequestGenerator
{
public TorrentPotatoSettings Settings { get; set; }
public TorrentPotatoRequestGenerator()
{
}
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests("list", null, null));
return pageableRequests;
}
private IEnumerable<IndexerRequest> GetPagedRequests(string mode, int? tvdbId, string query, params object[] args)
{
var requestBuilder = new HttpRequestBuilder(Settings.BaseUrl)
.Accept(HttpAccept.Json);
requestBuilder.AddQueryParam("passkey", Settings.Passkey);
if (!string.IsNullOrWhiteSpace(Settings.User))
{
requestBuilder.AddQueryParam("user", Settings.User);
}
else
{
requestBuilder.AddQueryParam("user", "");
}
requestBuilder.AddQueryParam("search", "-");
yield return new IndexerRequest(requestBuilder.Build());
}
private IEnumerable<IndexerRequest> GetMovieRequest(MovieSearchCriteria searchCriteria)
{
var requestBuilder = new HttpRequestBuilder(Settings.BaseUrl)
.Accept(HttpAccept.Json);
requestBuilder.AddQueryParam("passkey", Settings.Passkey);
if (!string.IsNullOrWhiteSpace(Settings.User))
{
requestBuilder.AddQueryParam("user", Settings.User);
}
else
{
requestBuilder.AddQueryParam("user", "");
}
if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace())
{
requestBuilder.AddQueryParam("imdbid", searchCriteria.ImdbId);
}
else if (searchCriteria.QueryTitles.Count > 0)
{
//TODO: Hack for now
requestBuilder.AddQueryParam("search", $"{searchCriteria.QueryTitles.First()}");
}
yield return new IndexerRequest(requestBuilder.Build());
}
public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetMovieRequest(searchCriteria));
return pageableRequests;
}
public Func<IDictionary<string, string>> GetCookies { get; set; }
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
}
}
@@ -0,0 +1,24 @@
using System;
namespace NzbDrone.Core.Indexers.TorrentPotato
{
public class TorrentPotatoResponse
{
public Result[] results { get; set; }
public int total_results { get; set; }
}
public class Result
{
public string release_name { get; set; }
public string torrent_id { get; set; }
public string details_url { get; set; }
public string download_url { get; set; }
public bool freeleech { get; set; }
public string type { get; set; }
public int size { get; set; }
public int leechers { get; set; }
public int seeders { get; set; }
public DateTime publish_date { get; set; }
}
}
@@ -0,0 +1,53 @@
using System.Collections.Generic;
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.TorrentPotato
{
public class TorrentPotatoSettingsValidator : AbstractValidator<TorrentPotatoSettings>
{
public TorrentPotatoSettingsValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
}
}
public class TorrentPotatoSettings : ITorrentIndexerSettings
{
private static readonly TorrentPotatoSettingsValidator Validator = new TorrentPotatoSettingsValidator();
public TorrentPotatoSettings()
{
BaseUrl = "http://127.0.0.1";
MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
MultiLanguages = new List<int>();
RequiredFlags = new List<int>();
}
[FieldDefinition(0, Label = "API URL", HelpText = "URL to TorrentPotato api.")]
public string BaseUrl { get; set; }
[FieldDefinition(1, Label = "Username", HelpText = "The username you use at your indexer.", Privacy = PrivacyLevel.UserName)]
public string User { get; set; }
[FieldDefinition(2, Label = "Passkey", HelpText = "The password you use at your Indexer.", Privacy = PrivacyLevel.Password)]
public string Passkey { get; set; }
[FieldDefinition(3, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
public IEnumerable<int> MultiLanguages { get; set; }
[FieldDefinition(4, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
public int MinimumSeeders { get; set; }
[FieldDefinition(6, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", Advanced = true)]
public IEnumerable<int> RequiredFlags { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}
@@ -0,0 +1,34 @@
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
namespace NzbDrone.Core.Indexers.TorrentRss
{
public class TorrentRssIndexer : HttpIndexerBase<TorrentRssIndexerSettings>
{
public override string Name => "Torrent RSS Feed";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override bool SupportsSearch => false;
public override int PageSize => 0;
private readonly ITorrentRssParserFactory _torrentRssParserFactory;
public TorrentRssIndexer(ITorrentRssParserFactory torrentRssParserFactory, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, indexerStatusService, configService, logger)
{
_torrentRssParserFactory = torrentRssParserFactory;
}
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new TorrentRssIndexerRequestGenerator { Settings = Settings };
}
public override IParseIndexerResponse GetParser()
{
return _torrentRssParserFactory.GetParser(Settings);
}
}
}
@@ -0,0 +1,12 @@
namespace NzbDrone.Core.Indexers.TorrentRss
{
public class TorrentRssIndexerParserSettings
{
public bool UseEZTVFormat { get; set; }
public bool ParseSeedersInDescription { get; set; }
public bool UseEnclosureUrl { get; set; }
public bool UseEnclosureLength { get; set; }
public bool ParseSizeInDescription { get; set; }
public string SizeElementName { get; set; }
}
}
@@ -0,0 +1,45 @@
using System;
using System.Collections.Generic;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.TorrentRss
{
public class TorrentRssIndexerRequestGenerator : IIndexerRequestGenerator
{
public TorrentRssIndexerSettings Settings { get; set; }
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetRssRequests(null));
return pageableRequests;
}
public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
{
return new IndexerPageableRequestChain();
}
private IEnumerable<IndexerRequest> GetRssRequests(string searchParameters)
{
var request = new IndexerRequest(Settings.BaseUrl.Trim().TrimEnd('/'), HttpAccept.Rss);
if (Settings.Cookie.IsNotNullOrWhiteSpace())
{
foreach (var cookie in HttpHeader.ParseCookies(Settings.Cookie))
{
request.HttpRequest.Cookies[cookie.Key] = cookie.Value;
}
}
yield return request;
}
public Func<IDictionary<string, string>> GetCookies { get; set; }
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
}
}
@@ -0,0 +1,54 @@
using System.Collections.Generic;
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Languages;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.TorrentRss
{
public class TorrentRssIndexerSettingsValidator : AbstractValidator<TorrentRssIndexerSettings>
{
public TorrentRssIndexerSettingsValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
}
}
public class TorrentRssIndexerSettings : ITorrentIndexerSettings
{
private static readonly TorrentRssIndexerSettingsValidator Validator = new TorrentRssIndexerSettingsValidator();
public TorrentRssIndexerSettings()
{
BaseUrl = string.Empty;
AllowZeroSize = false;
MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
MultiLanguages = new List<int>();
RequiredFlags = new List<int>();
}
[FieldDefinition(0, Label = "Full RSS Feed URL")]
public string BaseUrl { get; set; }
[FieldDefinition(1, Label = "Cookie", HelpText = "If you site requires a login cookie to access the rss, you'll have to retrieve it via a browser.")]
public string Cookie { get; set; }
[FieldDefinition(2, Type = FieldType.Checkbox, Label = "Allow Zero Size", HelpText = "Enabling this will allow you to use feeds that don't specify release size, but be careful, size related checks will not be performed.")]
public bool AllowZeroSize { get; set; }
[FieldDefinition(3, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
public IEnumerable<int> MultiLanguages { get; set; }
[FieldDefinition(4, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
public int MinimumSeeders { get; set; }
[FieldDefinition(6, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", HelpLink = "https://github.com/Prowlarr/Prowlarr/wiki/Indexer-Flags#1-required-flags", Advanced = true)]
public IEnumerable<int> RequiredFlags { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}
@@ -0,0 +1,65 @@
using System;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Indexers.Exceptions;
namespace NzbDrone.Core.Indexers.TorrentRss
{
public interface ITorrentRssParserFactory
{
TorrentRssParser GetParser(TorrentRssIndexerSettings settings);
}
public class TorrentRssParserFactory : ITorrentRssParserFactory
{
protected readonly Logger _logger;
private readonly ICached<TorrentRssIndexerParserSettings> _settingsCache;
private readonly ITorrentRssSettingsDetector _torrentRssSettingsDetector;
public TorrentRssParserFactory(ICacheManager cacheManager, ITorrentRssSettingsDetector torrentRssSettingsDetector, Logger logger)
{
_settingsCache = cacheManager.GetCache<TorrentRssIndexerParserSettings>(GetType());
_torrentRssSettingsDetector = torrentRssSettingsDetector;
_logger = logger;
}
public TorrentRssParser GetParser(TorrentRssIndexerSettings indexerSettings)
{
var key = indexerSettings.ToJson();
var parserSettings = _settingsCache.Get(key, () => DetectParserSettings(indexerSettings), TimeSpan.FromDays(7));
if (parserSettings.UseEZTVFormat)
{
return new EzrssTorrentRssParser();
}
else
{
return new TorrentRssParser
{
UseGuidInfoUrl = false,
ParseSeedersInDescription = parserSettings.ParseSeedersInDescription,
UseEnclosureUrl = parserSettings.UseEnclosureUrl,
UseEnclosureLength = parserSettings.UseEnclosureLength,
ParseSizeInDescription = parserSettings.ParseSizeInDescription,
SizeElementName = parserSettings.SizeElementName
};
}
}
private TorrentRssIndexerParserSettings DetectParserSettings(TorrentRssIndexerSettings indexerSettings)
{
var settings = _torrentRssSettingsDetector.Detect(indexerSettings);
if (settings == null)
{
throw new UnsupportedFeedException("Could not parse feed from {0}", indexerSettings.BaseUrl);
}
return settings;
}
}
}
@@ -0,0 +1,307 @@
using System;
using System.IO;
using System.Linq;
using System.Xml;
using System.Xml.Linq;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.TorrentRss
{
public interface ITorrentRssSettingsDetector
{
TorrentRssIndexerParserSettings Detect(TorrentRssIndexerSettings settings);
}
public class TorrentRssSettingsDetector : ITorrentRssSettingsDetector
{
private const long ValidSizeThreshold = 2 * 1024 * 1024;
protected readonly Logger _logger;
private readonly IHttpClient _httpClient;
public TorrentRssSettingsDetector(IHttpClient httpClient, Logger logger)
{
_httpClient = httpClient;
_logger = logger;
}
/// <summary>
/// Detect settings for Parser, based on URL
/// </summary>
/// <param name="settings">Indexer Settings to use for Parser</param>
/// <returns>Parsed Settings or <c>null</c></returns>
public TorrentRssIndexerParserSettings Detect(TorrentRssIndexerSettings indexerSettings)
{
_logger.Debug("Evaluating TorrentRss feed '{0}'", indexerSettings.BaseUrl);
var requestGenerator = new TorrentRssIndexerRequestGenerator { Settings = indexerSettings };
var request = requestGenerator.GetRecentRequests().GetAllTiers().First().First();
HttpResponse httpResponse = null;
try
{
httpResponse = _httpClient.Execute(request.HttpRequest);
}
catch (Exception ex)
{
_logger.Warn(ex, string.Format("Unable to connect to indexer {0}: {1}", request.Url, ex.Message));
return null;
}
var indexerResponse = new IndexerResponse(request, httpResponse);
return GetParserSettings(indexerResponse, indexerSettings);
}
private TorrentRssIndexerParserSettings GetParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)
{
var settings = GetEzrssParserSettings(response, indexerSettings);
if (settings != null)
{
return settings;
}
settings = GetGenericTorrentRssParserSettings(response, indexerSettings);
if (settings != null)
{
return settings;
}
return null;
}
private TorrentRssIndexerParserSettings GetEzrssParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)
{
if (!IsEZTVFeed(response))
{
return null;
}
_logger.Trace("Feed has Ezrss schema");
var parser = new EzrssTorrentRssParser();
var releases = ParseResponse(parser, response);
try
{
ValidateReleases(releases, indexerSettings);
ValidateReleaseSize(releases, indexerSettings);
_logger.Debug("Feed was parseable by Ezrss Parser");
return new TorrentRssIndexerParserSettings
{
UseEZTVFormat = true
};
}
catch (Exception ex)
{
_logger.Trace(ex, "Feed wasn't parsable by Ezrss Parser");
return null;
}
}
private TorrentRssIndexerParserSettings GetGenericTorrentRssParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)
{
var parser = new TorrentRssParser
{
UseEnclosureUrl = true,
UseEnclosureLength = true,
ParseSeedersInDescription = true
};
var item = parser.GetItems(response).FirstOrDefault();
if (item == null)
{
throw new UnsupportedFeedException("Empty feed, cannot check if feed is parsable.");
}
var settings = new TorrentRssIndexerParserSettings()
{
UseEnclosureUrl = true,
UseEnclosureLength = true,
ParseSeedersInDescription = true
};
if (item.Element("enclosure") == null)
{
parser.UseEnclosureUrl = settings.UseEnclosureUrl = false;
}
var releases = ParseResponse(parser, response);
ValidateReleases(releases, indexerSettings);
if (!releases.Any(v => v.Seeders.HasValue))
{
_logger.Trace("Feed doesn't have Seeders in Description, disabling option.");
parser.ParseSeedersInDescription = settings.ParseSeedersInDescription = false;
}
if (!releases.Any(r => r.Size < ValidSizeThreshold))
{
_logger.Trace("Feed has valid size in enclosure.");
return settings;
}
parser.UseEnclosureLength = settings.UseEnclosureLength = false;
foreach (var sizeElementName in new[] { "size", "Size" })
{
parser.SizeElementName = settings.SizeElementName = sizeElementName;
releases = ParseResponse(parser, response);
ValidateReleases(releases, indexerSettings);
if (!releases.Any(r => r.Size < ValidSizeThreshold))
{
_logger.Trace("Feed has valid size in Size element.");
return settings;
}
}
parser.SizeElementName = settings.SizeElementName = null;
parser.ParseSizeInDescription = settings.ParseSizeInDescription = true;
releases = ParseResponse(parser, response);
ValidateReleases(releases, indexerSettings);
if (releases.Count(r => r.Size >= ValidSizeThreshold) > releases.Length / 2)
{
if (releases.Any(r => r.Size < ValidSizeThreshold))
{
_logger.Debug("Feed {0} contains very small releases.", response.Request.Url);
}
_logger.Trace("Feed has valid size in description.");
return settings;
}
parser.ParseSizeInDescription = settings.ParseSizeInDescription = false;
_logger.Debug("Feed doesn't have release size.");
releases = ParseResponse(parser, response);
ValidateReleases(releases, indexerSettings);
ValidateReleaseSize(releases, indexerSettings);
return settings;
}
private bool IsEZTVFeed(IndexerResponse response)
{
var content = XmlCleaner.ReplaceEntities(response.Content);
content = XmlCleaner.ReplaceUnicode(content);
using (var xmlTextReader = XmlReader.Create(new StringReader(content), new XmlReaderSettings { DtdProcessing = DtdProcessing.Parse, ValidationType = ValidationType.None, IgnoreComments = true, XmlResolver = null }))
{
var document = XDocument.Load(xmlTextReader);
// Check Namespace
if (document.Root == null)
{
throw new InvalidDataException("Could not parse IndexerResponse into XML.");
}
var ns = document.Root.GetNamespaceOfPrefix("torrent");
if (ns == "http://xmlns.ezrss.it/0.1/")
{
_logger.Trace("Identified feed as EZTV compatible by EZTV Namespace");
return true;
}
// Check DTD in DocType
if (document.DocumentType != null && document.DocumentType.SystemId == "http://xmlns.ezrss.it/0.1/dtd/")
{
_logger.Trace("Identified feed as EZTV compatible by EZTV DTD");
return true;
}
// Check namespaces
if (document.Descendants().Any(v => v.GetDefaultNamespace().NamespaceName == "http://xmlns.ezrss.it/0.1/"))
{
_logger.Trace("Identified feed as EZTV compatible by EZTV Namespace");
return true;
}
return false;
}
}
private TorrentInfo[] ParseResponse(IParseIndexerResponse parser, IndexerResponse response)
{
try
{
var releases = parser.ParseResponse(response).Cast<TorrentInfo>().ToArray();
return releases;
}
catch (Exception ex)
{
_logger.Debug(ex, "Unable to parse indexer feed: " + ex.Message);
throw new UnsupportedFeedException("Unable to parse indexer: " + ex.Message);
}
}
private void ValidateReleases(TorrentInfo[] releases, TorrentRssIndexerSettings indexerSettings)
{
if (releases == null || releases.Empty())
{
throw new UnsupportedFeedException("Empty feed, cannot check if feed is parsable.");
}
var torrentInfo = releases.First();
_logger.Trace("TorrentInfo: \n{0}", torrentInfo.ToString("L"));
if (releases.Any(r => r.Title.IsNullOrWhiteSpace()))
{
throw new UnsupportedFeedException("Feed contains releases without title.");
}
if (releases.Any(r => !IsValidDownloadUrl(r.DownloadUrl)))
{
throw new UnsupportedFeedException("Failed to find a valid download url in the feed.");
}
var total = releases.Where(v => v.Guid != null).Select(v => v.Guid).ToArray();
var distinct = total.Distinct().ToArray();
if (distinct.Length != total.Length)
{
throw new UnsupportedFeedException("Feed contains releases with same guid, rejecting malformed rss feed.");
}
}
private void ValidateReleaseSize(TorrentInfo[] releases, TorrentRssIndexerSettings indexerSettings)
{
if (!indexerSettings.AllowZeroSize && releases.Any(r => r.Size == 0))
{
throw new UnsupportedFeedException("Feed doesn't contain the release content size.");
}
if (releases.Any(r => r.Size != 0 && r.Size < ValidSizeThreshold))
{
throw new UnsupportedFeedException("Size of one more releases lower than {0}, feed must contain release content size.", ValidSizeThreshold.SizeSuffix());
}
}
private static bool IsValidDownloadUrl(string url)
{
if (url.IsNullOrWhiteSpace())
{
return false;
}
if (url.StartsWith("magnet:") ||
url.StartsWith("http:") ||
url.StartsWith("https:"))
{
return true;
}
return false;
}
}
}
@@ -0,0 +1,152 @@
using System;
using System.Collections.Generic;
using System.Linq;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Indexers.Newznab;
using NzbDrone.Core.ThingiProvider;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Torznab
{
public class Torznab : HttpIndexerBase<TorznabSettings>
{
private readonly INewznabCapabilitiesProvider _capabilitiesProvider;
public override string Name => "Torznab";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override int PageSize => _capabilitiesProvider.GetCapabilities(Settings).DefaultPageSize;
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new NewznabRequestGenerator(_capabilitiesProvider)
{
PageSize = PageSize,
Settings = Settings
};
}
public override IParseIndexerResponse GetParser()
{
return new TorznabRssParser();
}
public override IEnumerable<ProviderDefinition> DefaultDefinitions
{
get
{
yield return GetDefinition("Jackett", GetSettings("http://localhost:9117/api/v2.0/indexers/YOURINDEXER/results/torznab/"));
yield return GetDefinition("HD4Free.xyz", GetSettings("http://hd4free.xyz"));
}
}
public Torznab(INewznabCapabilitiesProvider capabilitiesProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, indexerStatusService, configService, logger)
{
_capabilitiesProvider = capabilitiesProvider;
}
private IndexerDefinition GetDefinition(string name, TorznabSettings settings)
{
return new IndexerDefinition
{
EnableRss = false,
EnableAutomaticSearch = false,
EnableInteractiveSearch = false,
Name = name,
Implementation = GetType().Name,
Settings = settings,
Protocol = DownloadProtocol.Usenet,
SupportsRss = SupportsRss,
SupportsSearch = SupportsSearch
};
}
private TorznabSettings GetSettings(string url, string apiPath = null, int[] categories = null)
{
var settings = new TorznabSettings { BaseUrl = url };
if (categories != null)
{
settings.Categories = categories;
}
if (apiPath.IsNotNullOrWhiteSpace())
{
settings.ApiPath = apiPath;
}
return settings;
}
protected override void Test(List<ValidationFailure> failures)
{
base.Test(failures);
if (failures.HasErrors())
{
return;
}
failures.AddIfNotNull(TestCapabilities());
}
protected static List<int> CategoryIds(List<NewznabCategory> categories)
{
var l = categories.Select(c => c.Id).ToList();
foreach (var category in categories)
{
if (category.Subcategories != null)
{
l.AddRange(CategoryIds(category.Subcategories));
}
}
return l;
}
protected virtual ValidationFailure TestCapabilities()
{
try
{
var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
var notSupported = Settings.Categories.Except(CategoryIds(capabilities.Categories));
if (notSupported.Any())
{
_logger.Warn($"{Definition.Name} does not support the following categories: {string.Join(", ", notSupported)}. You should probably remove them.");
if (notSupported.Count() == Settings.Categories.Count())
{
return new ValidationFailure(string.Empty, $"This indexer does not support any of the selected categories! (You may need to turn on advanced settings to see them)");
}
}
if (capabilities.SupportedSearchParameters != null && capabilities.SupportedSearchParameters.Contains("q"))
{
return null;
}
if (capabilities.SupportedMovieSearchParameters != null &&
new[] { "q", "imdbid" }.Any(v => capabilities.SupportedMovieSearchParameters.Contains(v)) &&
new[] { "imdbtitle", "imdbyear" }.All(v => capabilities.SupportedMovieSearchParameters.Contains(v)))
{
return null;
}
return new ValidationFailure(string.Empty, "This indexer does not support searching for movies :(. Tell your indexer staff to enable this or force add the indexer by disabling search, adding the indexer and then enabling it again.");
}
catch (Exception ex)
{
_logger.Warn(ex, "Unable to connect to indexer: " + ex.Message);
return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log for more details");
}
}
}
}
@@ -0,0 +1,17 @@
using NzbDrone.Common.Exceptions;
namespace NzbDrone.Core.Indexers.Torznab
{
public class TorznabException : NzbDroneException
{
public TorznabException(string message, params object[] args)
: base(message, args)
{
}
public TorznabException(string message)
: base(message)
{
}
}
}
@@ -0,0 +1,228 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Xml.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Torznab
{
public class TorznabRssParser : TorrentRssParser
{
public const string ns = "{http://torznab.com/schemas/2015/feed}";
public TorznabRssParser()
{
UseEnclosureUrl = true;
}
protected override bool PreProcess(IndexerResponse indexerResponse)
{
var xdoc = LoadXmlDocument(indexerResponse);
var error = xdoc.Descendants("error").FirstOrDefault();
if (error == null)
{
return true;
}
var code = Convert.ToInt32(error.Attribute("code").Value);
var errorMessage = error.Attribute("description").Value;
if (code >= 100 && code <= 199)
{
throw new ApiKeyException("Invalid API key");
}
if (!indexerResponse.Request.Url.FullUri.Contains("apikey=") && errorMessage == "Missing parameter")
{
throw new ApiKeyException("Indexer requires an API key");
}
if (errorMessage == "Request limit reached")
{
throw new RequestLimitReachedException("API limit reached");
}
throw new TorznabException("Torznab error detected: {0}", errorMessage);
}
protected override ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo)
{
var torrentInfo = base.ProcessItem(item, releaseInfo) as TorrentInfo;
if (torrentInfo != null)
{
if (GetImdbId(item) != null)
{
torrentInfo.ImdbId = int.Parse(GetImdbId(item).Substring(2));
}
torrentInfo.IndexerFlags = GetFlags(item);
}
return torrentInfo;
}
protected override bool PostProcess(IndexerResponse indexerResponse, List<XElement> items, List<ReleaseInfo> releases)
{
var enclosureTypes = items.SelectMany(GetEnclosures).Select(v => v.Type).Distinct().ToArray();
if (enclosureTypes.Any() && enclosureTypes.Intersect(PreferredEnclosureMimeTypes).Empty())
{
if (enclosureTypes.Intersect(UsenetEnclosureMimeTypes).Any())
{
_logger.Warn("Feed does not contain {0}, found {1}, did you intend to add a Newznab indexer?", TorrentEnclosureMimeType, enclosureTypes[0]);
}
else
{
_logger.Warn("Feed does not contain {0}, found {1}.", TorrentEnclosureMimeType, enclosureTypes[0]);
}
}
return true;
}
protected override string GetInfoUrl(XElement item)
{
return ParseUrl(item.TryGetValue("comments").TrimEnd("#comments"));
}
protected override string GetCommentUrl(XElement item)
{
return ParseUrl(item.TryGetValue("comments"));
}
protected override long GetSize(XElement item)
{
long size;
var sizeString = TryGetTorznabAttribute(item, "size");
if (!sizeString.IsNullOrWhiteSpace() && long.TryParse(sizeString, out size))
{
return size;
}
size = GetEnclosureLength(item);
return size;
}
protected override DateTime GetPublishDate(XElement item)
{
return base.GetPublishDate(item);
}
protected override string GetDownloadUrl(XElement item)
{
var url = base.GetDownloadUrl(item);
if (!Uri.IsWellFormedUriString(url, UriKind.Absolute))
{
url = ParseUrl((string)item.Element("enclosure").Attribute("url"));
}
return url;
}
protected virtual string GetImdbId(XElement item)
{
var imdbIdString = TryGetTorznabAttribute(item, "imdbid");
return !imdbIdString.IsNullOrWhiteSpace() ? imdbIdString.Substring(2) : null;
}
protected override string GetInfoHash(XElement item)
{
return TryGetTorznabAttribute(item, "infohash");
}
protected override string GetMagnetUrl(XElement item)
{
return TryGetTorznabAttribute(item, "magneturl");
}
protected override int? GetSeeders(XElement item)
{
var seeders = TryGetTorznabAttribute(item, "seeders");
if (seeders.IsNotNullOrWhiteSpace())
{
return int.Parse(seeders);
}
return base.GetSeeders(item);
}
protected override int? GetPeers(XElement item)
{
var peers = TryGetTorznabAttribute(item, "peers");
if (peers.IsNotNullOrWhiteSpace())
{
return int.Parse(peers);
}
var seeders = TryGetTorznabAttribute(item, "seeders");
var leechers = TryGetTorznabAttribute(item, "leechers");
if (seeders.IsNotNullOrWhiteSpace() && leechers.IsNotNullOrWhiteSpace())
{
return int.Parse(seeders) + int.Parse(leechers);
}
return base.GetPeers(item);
}
protected IndexerFlags GetFlags(XElement item)
{
IndexerFlags flags = 0;
var downloadFactor = TryGetFloatTorznabAttribute(item, "downloadvolumefactor", 1);
var uploadFactor = TryGetFloatTorznabAttribute(item, "uploadvolumefactor", 1);
if (uploadFactor == 2)
{
flags |= IndexerFlags.G_DoubleUpload;
}
if (downloadFactor == 0.5)
{
flags |= IndexerFlags.G_Halfleech;
}
if (downloadFactor == 0.0)
{
flags |= IndexerFlags.G_Freeleech;
}
return flags;
}
protected string TryGetTorznabAttribute(XElement item, string key, string defaultValue = "")
{
var attr = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.CurrentCultureIgnoreCase));
if (attr != null)
{
return attr.Attribute("value").Value;
}
return defaultValue;
}
protected float TryGetFloatTorznabAttribute(XElement item, string key, float defaultValue = 0)
{
var attr = TryGetTorznabAttribute(item, key, defaultValue.ToString());
float result = 0;
if (float.TryParse(attr, out result))
{
return result;
}
return defaultValue;
}
}
}
@@ -0,0 +1,71 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using FluentValidation;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Indexers.Newznab;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Torznab
{
public class TorznabSettingsValidator : AbstractValidator<TorznabSettings>
{
private static readonly string[] ApiKeyWhiteList =
{
"hd4free.xyz",
};
private static bool ShouldHaveApiKey(TorznabSettings settings)
{
if (settings.BaseUrl == null)
{
return false;
}
return ApiKeyWhiteList.Any(c => settings.BaseUrl.ToLowerInvariant().Contains(c));
}
private static readonly Regex AdditionalParametersRegex = new Regex(@"(&.+?\=.+?)+", RegexOptions.Compiled);
public TorznabSettingsValidator()
{
RuleFor(c => c).Custom((c, context) =>
{
if (c.Categories.Empty())
{
context.AddFailure("'Categories' must be provided");
}
});
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.ApiPath).ValidUrlBase("/api");
RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey);
RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex)
.When(c => !c.AdditionalParameters.IsNullOrWhiteSpace());
}
}
public class TorznabSettings : NewznabSettings, ITorrentIndexerSettings
{
private static readonly TorznabSettingsValidator Validator = new TorznabSettingsValidator();
public TorznabSettings()
{
MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
RequiredFlags = new List<int>();
}
[FieldDefinition(8, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
public int MinimumSeeders { get; set; }
[FieldDefinition(9, Type = FieldType.TagSelect, SelectOptions = typeof(IndexerFlags), Label = "Required Flags", HelpText = "What indexer flags are required?", HelpLink = "https://github.com/Prowlarr/Prowlarr/wiki/Indexer-Flags#1-required-flags", Advanced = true)]
public IEnumerable<int> RequiredFlags { get; set; }
public override NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}