New: Readarr 0.1

This commit is contained in:
ta264
2020-05-06 21:14:11 +01:00
parent 476f2d6047
commit 08496c82af
911 changed files with 14837 additions and 24442 deletions
@@ -5,7 +5,6 @@ using NzbDrone.Common.Cache;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser;
using NzbDrone.Core.ThingiProvider;
namespace NzbDrone.Core.Indexers.Gazelle
{
@@ -46,16 +45,6 @@ namespace NzbDrone.Core.Indexers.Gazelle
return new GazelleParser(Settings);
}
public override IEnumerable<ProviderDefinition> DefaultDefinitions
{
get
{
yield return GetDefinition("Orpheus Network", GetSettings("https://orpheus.network"));
yield return GetDefinition("REDacted", GetSettings("https://redacted.ch"));
yield return GetDefinition("Not What CD", GetSettings("https://notwhat.cd"));
}
}
private IndexerDefinition GetDefinition(string name, GazelleSettings settings)
{
return new IndexerDefinition
@@ -1,79 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser;
namespace NzbDrone.Core.Indexers.Headphones
{
public class Headphones : HttpIndexerBase<HeadphonesSettings>
{
private readonly IHeadphonesCapabilitiesProvider _capabilitiesProvider;
public override string Name => "Headphones VIP";
public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
public override int PageSize => _capabilitiesProvider.GetCapabilities(Settings).DefaultPageSize;
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new HeadphonesRequestGenerator(_capabilitiesProvider)
{
PageSize = PageSize,
Settings = Settings
};
}
public override IParseIndexerResponse GetParser()
{
return new HeadphonesRssParser
{
Settings = Settings
};
}
public Headphones(IHeadphonesCapabilitiesProvider capabilitiesProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
: base(httpClient, indexerStatusService, configService, parsingService, logger)
{
_capabilitiesProvider = capabilitiesProvider;
}
protected override void Test(List<ValidationFailure> failures)
{
base.Test(failures);
if (failures.Any())
{
return;
}
failures.AddIfNotNull(TestCapabilities());
}
protected virtual ValidationFailure TestCapabilities()
{
try
{
var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
if (capabilities.SupportedSearchParameters != null && capabilities.SupportedSearchParameters.Contains("q"))
{
return null;
}
return new ValidationFailure(string.Empty, "Indexer does not support required search parameters");
}
catch (Exception ex)
{
_logger.Warn(ex, "Unable to connect to indexer: " + ex.Message);
return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log for more details");
}
}
}
}
@@ -1,21 +0,0 @@
using System.Collections.Generic;
using NzbDrone.Core.Indexers.Newznab;
namespace NzbDrone.Core.Indexers.Headphones
{
public class HeadphonesCapabilities
{
public int DefaultPageSize { get; set; }
public int MaxPageSize { get; set; }
public string[] SupportedSearchParameters { get; set; }
public List<NewznabCategory> Categories { get; set; }
public HeadphonesCapabilities()
{
DefaultPageSize = 100;
MaxPageSize = 100;
SupportedSearchParameters = new[] { "q" };
Categories = new List<NewznabCategory>();
}
}
}
@@ -1,154 +0,0 @@
using System;
using System.Collections.Generic;
using System.Xml;
using System.Xml.Linq;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Indexers.Newznab;
namespace NzbDrone.Core.Indexers.Headphones
{
public interface IHeadphonesCapabilitiesProvider
{
HeadphonesCapabilities GetCapabilities(HeadphonesSettings settings);
}
public class HeadphonesCapabilitiesProvider : IHeadphonesCapabilitiesProvider
{
private readonly ICached<HeadphonesCapabilities> _capabilitiesCache;
private readonly IHttpClient _httpClient;
private readonly Logger _logger;
public HeadphonesCapabilitiesProvider(ICacheManager cacheManager, IHttpClient httpClient, Logger logger)
{
_capabilitiesCache = cacheManager.GetCache<HeadphonesCapabilities>(GetType());
_httpClient = httpClient;
_logger = logger;
}
public HeadphonesCapabilities GetCapabilities(HeadphonesSettings indexerSettings)
{
var key = indexerSettings.ToJson();
var capabilities = _capabilitiesCache.Get(key, () => FetchCapabilities(indexerSettings), TimeSpan.FromDays(7));
return capabilities;
}
private HeadphonesCapabilities FetchCapabilities(HeadphonesSettings indexerSettings)
{
var capabilities = new HeadphonesCapabilities();
var url = string.Format("{0}{1}?t=caps", indexerSettings.BaseUrl.TrimEnd('/'), indexerSettings.ApiPath.TrimEnd('/'));
if (indexerSettings.ApiKey.IsNotNullOrWhiteSpace())
{
url += "&apikey=" + indexerSettings.ApiKey;
}
var request = new HttpRequest(url, HttpAccept.Rss);
request.AddBasicAuthentication(indexerSettings.Username, indexerSettings.Password);
HttpResponse response;
try
{
response = _httpClient.Get(request);
}
catch (Exception ex)
{
_logger.Debug(ex, "Failed to get headphones api capabilities from {0}", indexerSettings.BaseUrl);
throw;
}
try
{
capabilities = ParseCapabilities(response);
}
catch (XmlException ex)
{
_logger.Debug(ex, "Failed to parse headphones api capabilities for {0}", indexerSettings.BaseUrl);
ex.WithData(response);
throw;
}
catch (Exception ex)
{
_logger.Error(ex, "Failed to determine headphones api capabilities for {0}, using the defaults instead till Readarr restarts", indexerSettings.BaseUrl);
}
return capabilities;
}
private HeadphonesCapabilities ParseCapabilities(HttpResponse response)
{
var capabilities = new HeadphonesCapabilities();
var xDoc = XDocument.Parse(response.Content);
if (xDoc == null)
{
throw new XmlException("Invalid XML");
}
var xmlRoot = xDoc.Element("caps");
if (xmlRoot == null)
{
throw new XmlException("Unexpected XML");
}
var xmlLimits = xmlRoot.Element("limits");
if (xmlLimits != null)
{
capabilities.DefaultPageSize = int.Parse(xmlLimits.Attribute("default").Value);
capabilities.MaxPageSize = int.Parse(xmlLimits.Attribute("max").Value);
}
var xmlSearching = xmlRoot.Element("searching");
if (xmlSearching != null)
{
var xmlBasicSearch = xmlSearching.Element("search");
if (xmlBasicSearch == null || xmlBasicSearch.Attribute("available").Value != "yes")
{
capabilities.SupportedSearchParameters = null;
}
else if (xmlBasicSearch.Attribute("supportedParams") != null)
{
capabilities.SupportedSearchParameters = xmlBasicSearch.Attribute("supportedParams").Value.Split(',');
}
}
var xmlCategories = xmlRoot.Element("categories");
if (xmlCategories != null)
{
foreach (var xmlCategory in xmlCategories.Elements("category"))
{
var cat = new NewznabCategory
{
Id = int.Parse(xmlCategory.Attribute("id").Value),
Name = xmlCategory.Attribute("name").Value,
Description = xmlCategory.Attribute("description") != null ? xmlCategory.Attribute("description").Value : string.Empty,
Subcategories = new List<NewznabCategory>()
};
foreach (var xmlSubcat in xmlCategory.Elements("subcat"))
{
cat.Subcategories.Add(new NewznabCategory
{
Id = int.Parse(xmlSubcat.Attribute("id").Value),
Name = xmlSubcat.Attribute("name").Value,
Description = xmlSubcat.Attribute("description") != null ? xmlSubcat.Attribute("description").Value : string.Empty
});
}
capabilities.Categories.Add(cat);
}
}
return capabilities;
}
}
}
@@ -1,102 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.Headphones
{
public class HeadphonesRequestGenerator : IIndexerRequestGenerator
{
private readonly IHeadphonesCapabilitiesProvider _capabilitiesProvider;
public int MaxPages { get; set; }
public int PageSize { get; set; }
public HeadphonesSettings Settings { get; set; }
public HeadphonesRequestGenerator(IHeadphonesCapabilitiesProvider capabilitiesProvider)
{
_capabilitiesProvider = capabilitiesProvider;
MaxPages = 30;
PageSize = 100;
}
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "search", ""));
return pageableRequests;
}
public virtual IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.AddTier();
pageableRequests.Add(GetPagedRequests(MaxPages,
Settings.Categories,
"search",
NewsnabifyTitle($"&q={searchCriteria.ArtistQuery}+{searchCriteria.AlbumQuery}")));
return pageableRequests;
}
public virtual IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.AddTier();
pageableRequests.Add(GetPagedRequests(MaxPages,
Settings.Categories,
"search",
NewsnabifyTitle($"&q={searchCriteria.ArtistQuery}")));
return pageableRequests;
}
private IEnumerable<IndexerRequest> GetPagedRequests(int maxPages, IEnumerable<int> categories, string searchType, string parameters)
{
if (categories.Empty())
{
yield break;
}
var categoriesQuery = string.Join(",", categories.Distinct());
var baseUrl =
$"{Settings.BaseUrl.TrimEnd('/')}{Settings.ApiPath.TrimEnd('/')}?t={searchType}&cat={categoriesQuery}&extended=1";
if (Settings.ApiKey.IsNotNullOrWhiteSpace())
{
baseUrl += "&apikey=" + Settings.ApiKey;
}
if (PageSize == 0)
{
var request = new IndexerRequest($"{baseUrl}{parameters}", HttpAccept.Rss);
request.HttpRequest.AddBasicAuthentication(Settings.Username, Settings.Password);
yield return request;
}
else
{
for (var page = 0; page < maxPages; page++)
{
var request = new IndexerRequest($"{baseUrl}&offset={page * PageSize}&limit={PageSize}{parameters}", HttpAccept.Rss);
request.HttpRequest.AddBasicAuthentication(Settings.Username, Settings.Password);
yield return request;
}
}
}
private static string NewsnabifyTitle(string title)
{
return title.Replace("+", "%20");
}
}
}
@@ -1,22 +0,0 @@
using System;
using System.Text;
using NzbDrone.Core.Indexers.Newznab;
namespace NzbDrone.Core.Indexers.Headphones
{
public class HeadphonesRssParser : NewznabRssParser
{
public HeadphonesSettings Settings { get; set; }
public HeadphonesRssParser()
{
PreferredEnclosureMimeTypes = UsenetEnclosureMimeTypes;
UseEnclosureUrl = true;
}
protected override string GetBasicAuth()
{
return Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes($"{Settings.Username}:{Settings.Password}"));
}
}
}
@@ -1,61 +0,0 @@
using System.Collections.Generic;
using FluentValidation;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Headphones
{
public class HeadphonesSettingsValidator : AbstractValidator<HeadphonesSettings>
{
public HeadphonesSettingsValidator()
{
RuleFor(c => c).Custom((c, context) =>
{
if (c.Categories.Empty())
{
context.AddFailure("'Categories' must be provided");
}
});
RuleFor(c => c.Username).NotEmpty();
RuleFor(c => c.Password).NotEmpty();
}
}
public class HeadphonesSettings : IIndexerSettings
{
private static readonly HeadphonesSettingsValidator Validator = new HeadphonesSettingsValidator();
public HeadphonesSettings()
{
ApiPath = "/api";
BaseUrl = "https://indexer.codeshy.com";
ApiKey = "964d601959918a578a670984bdee9357";
Categories = new[] { 3000, 3010, 3020, 3030, 3040 };
}
public string BaseUrl { get; set; }
public string ApiPath { get; set; }
public string ApiKey { get; set; }
[FieldDefinition(0, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable standard/daily shows", Advanced = true)]
public IEnumerable<int> Categories { get; set; }
[FieldDefinition(1, Label = "Username")]
public string Username { get; set; }
[FieldDefinition(2, Label = "Password", Type = FieldType.Password)]
public string Password { get; set; }
[FieldDefinition(3, Type = FieldType.Number, Label = "Early Download Limit", Unit = "days", HelpText = "Time before release date Readarr will download from this indexer, empty is no limit", Advanced = true)]
public int? EarlyReleaseLimit { get; set; }
public virtual NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}
@@ -58,7 +58,7 @@ namespace NzbDrone.Core.Indexers.Newznab
public NewznabSettings()
{
ApiPath = "/api";
Categories = new[] { 3000, 3010, 3020, 3030, 3040 };
Categories = new[] { 7020, 8010 };
}
[FieldDefinition(0, Label = "URL")]
@@ -1,30 +0,0 @@
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser;
namespace NzbDrone.Core.Indexers.Waffles
{
public class Waffles : HttpIndexerBase<WafflesSettings>
{
public override string Name => "Waffles";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override int PageSize => 15;
public Waffles(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
: base(httpClient, indexerStatusService, configService, parsingService, logger)
{
}
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new WafflesRequestGenerator() { Settings = Settings };
}
public override IParseIndexerResponse GetParser()
{
return new WafflesRssParser() { ParseSizeInDescription = true, ParseSeedersInDescription = true };
}
}
}
@@ -1,63 +0,0 @@
using System.Collections.Generic;
using System.Text;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.Waffles
{
public class WafflesRequestGenerator : IIndexerRequestGenerator
{
public WafflesSettings Settings { get; set; }
public int MaxPages { get; set; }
public WafflesRequestGenerator()
{
MaxPages = 5;
}
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(MaxPages, null));
return pageableRequests;
}
public IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(MaxPages, string.Format("&q=artist:{0} album:{1}", searchCriteria.ArtistQuery, searchCriteria.AlbumQuery)));
return pageableRequests;
}
public IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(MaxPages, string.Format("&q=artist:{0}", searchCriteria.ArtistQuery)));
return pageableRequests;
}
private IEnumerable<IndexerRequest> GetPagedRequests(int maxPages, string query)
{
var url = new StringBuilder();
url.AppendFormat("{0}/browse.php?rss=1&c0=1&uid={1}&passkey={2}", Settings.BaseUrl.Trim().TrimEnd('/'), Settings.UserId, Settings.RssPasskey);
if (query.IsNotNullOrWhiteSpace())
{
url.AppendFormat(query);
}
for (var page = 0; page < maxPages; page++)
{
yield return new IndexerRequest(string.Format("{0}&p={1}", url, page), HttpAccept.Rss);
}
}
}
}
@@ -1,93 +0,0 @@
using System;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Waffles
{
public class WafflesRssParser : TorrentRssParser
{
public const string ns = "{http://purl.org/rss/1.0/}";
public const string dc = "{http://purl.org/dc/elements/1.1/}";
protected override bool PreProcess(IndexerResponse indexerResponse)
{
var xdoc = LoadXmlDocument(indexerResponse);
var error = xdoc.Descendants("error").FirstOrDefault();
if (error == null)
{
return true;
}
var code = Convert.ToInt32(error.Attribute("code").Value);
var errorMessage = error.Attribute("description").Value;
if (code >= 100 && code <= 199)
{
throw new ApiKeyException("Invalid Pass key");
}
if (!indexerResponse.Request.Url.FullUri.Contains("passkey=") && errorMessage == "Missing parameter")
{
throw new ApiKeyException("Indexer requires an Pass key");
}
if (errorMessage == "Request limit reached")
{
throw new RequestLimitReachedException("API limit reached");
}
throw new IndexerException(indexerResponse, errorMessage);
}
protected override ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo)
{
var torrentInfo = base.ProcessItem(item, releaseInfo) as TorrentInfo;
return torrentInfo;
}
protected override string GetInfoUrl(XElement item)
{
return ParseUrl(item.TryGetValue("comments").TrimEnd("#comments"));
}
protected override string GetCommentUrl(XElement item)
{
return ParseUrl(item.TryGetValue("comments"));
}
private static readonly Regex ParseSizeRegex = new Regex(@"(?:Size: )(?<value>\d+)<",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
protected override long GetSize(XElement item)
{
var match = ParseSizeRegex.Matches(item.Element("description").Value);
if (match.Count != 0)
{
var value = decimal.Parse(Regex.Replace(match[0].Groups["value"].Value, "\\,", ""), CultureInfo.InvariantCulture);
return (long)value;
}
return 0;
}
protected override DateTime GetPublishDate(XElement item)
{
var dateString = item.TryGetValue(dc + "date");
if (dateString.IsNullOrWhiteSpace())
{
throw new UnsupportedFeedException("Rss feed must have a pubDate element with a valid publish date.");
}
return XElementExtensions.ParseDate(dateString);
}
}
}
@@ -1,50 +0,0 @@
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Waffles
{
public class WafflesSettingsValidator : AbstractValidator<WafflesSettings>
{
public WafflesSettingsValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.UserId).NotEmpty();
RuleFor(c => c.RssPasskey).NotEmpty();
}
}
public class WafflesSettings : ITorrentIndexerSettings
{
private static readonly WafflesSettingsValidator Validator = new WafflesSettingsValidator();
public WafflesSettings()
{
BaseUrl = "https://www.waffles.ch";
MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS;
}
[FieldDefinition(0, Label = "Website URL")]
public string BaseUrl { get; set; }
[FieldDefinition(1, Label = "UserId")]
public string UserId { get; set; }
[FieldDefinition(2, Label = "RSS Passkey")]
public string RssPasskey { get; set; }
[FieldDefinition(3, Type = FieldType.Textbox, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)]
public int MinimumSeeders { get; set; }
[FieldDefinition(4)]
public SeedCriteriaSettings SeedCriteria { get; set; } = new SeedCriteriaSettings();
[FieldDefinition(5, Type = FieldType.Number, Label = "Early Download Limit", Unit = "days", HelpText = "Time before release date Readarr will download from this indexer, empty is no limit", Advanced = true)]
public int? EarlyReleaseLimit { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}