Fixed: Refactored the Indexer architecture to support non-rss indexers.

2026-04-24 22:35:39 -04:00 · 2014-09-07 13:44:24 +02:00
parent 22c9bc402f
commit 5e62c2335f
57 changed files with 2196 additions and 1470 deletions
@@ -0,0 +1,56 @@
+using System;
+using System.Linq;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Animezb;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Test.Framework;
+using NzbDrone.Core.ThingiProvider;
+
+namespace NzbDrone.Core.Test.IndexerTests.AnimezbTests
+{
+    [TestFixture]
+    public class AnimezbFixture : CoreTest<Animezb>
+    {
+        [SetUp]
+        public void Setup()
+        {
+            Subject.Definition = new IndexerDefinition()
+                {
+                    Name = "Animezb",
+                    Settings = new NullConfig()
+                };
+
+        }
+
+        [Test]
+        public void should_parse_recent_feed_from_Animezb()
+        {
+            Assert.Inconclusive("Waiting for animezb to get back up.");
+
+            var recentFeed = ReadAllText(@"Files/RSS/Animezb.xml");
+
+            Mocker.GetMock<IHttpClient>()
+                .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.GET)))
+                .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), recentFeed));
+            
+            var releases = Subject.FetchRecent();
+
+            releases.Should().HaveCount(3);
+
+            var releaseInfo = releases.First();
+
+            //releaseInfo.Title.Should().Be("[Vivid] Hanayamata - 10 [A33D6606]");
+            releaseInfo.DownloadProtocol.Should().Be(DownloadProtocol.Usenet);
+            //releaseInfo.DownloadUrl.Should().Be("http://fanzub.com/nzb/296464/Vivid%20Hanayamata%20-%2010.nzb");
+            releaseInfo.InfoUrl.Should().BeNullOrEmpty();
+            releaseInfo.CommentUrl.Should().BeNullOrEmpty();
+            releaseInfo.Indexer.Should().Be(Subject.Definition.Name);
+            //releaseInfo.PublishDate.Should().Be(DateTime.Parse("2014/09/13 12:56:53"));
+            //releaseInfo.Size.Should().Be(556246858);
+        }
+    }
+}
@@ -5,7 +5,7 @@ using NzbDrone.Core.Test.Framework;

 namespace NzbDrone.Core.Test.IndexerTests
 {
-    public class BasicRssParserFixture : CoreTest<RssParserBase>
+    public class BasicRssParserFixture : CoreTest<RssParser>
    {
        [TestCase("5.64 GB", 6055903887)]
        [TestCase("5.54 GiB", 5948529705)]
@@ -16,7 +16,7 @@ namespace NzbDrone.Core.Test.IndexerTests
        [TestCase("845 MB", 886046720)]
        public void parse_size(string sizeString, long expectedSize)
        {
-            var result = RssParserBase.ParseSize(sizeString, true);
+            var result = RssParser.ParseSize(sizeString, true);

            result.Should().Be(expectedSize);
        }
@@ -0,0 +1,53 @@
+using System;
+using System.Linq;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Fanzub;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Test.Framework;
+using NzbDrone.Core.ThingiProvider;
+
+namespace NzbDrone.Core.Test.IndexerTests.FanzubTests
+{
+    [TestFixture]
+    public class FanzubFixture : CoreTest<Fanzub>
+    {
+        [SetUp]
+        public void Setup()
+        {
+            Subject.Definition = new IndexerDefinition()
+                {
+                    Name = "Fanzub",
+                    Settings = new NullConfig()
+                };
+        }
+
+        [Test]
+        public void should_parse_recent_feed_from_fanzub()
+        {
+            var recentFeed = ReadAllText(@"Files/RSS/fanzub.xml");
+
+            Mocker.GetMock<IHttpClient>()
+                .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.GET)))
+                .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), recentFeed));
+            
+            var releases = Subject.FetchRecent();
+
+            releases.Should().HaveCount(3);
+
+            var releaseInfo = releases.First();
+
+            releaseInfo.Title.Should().Be("[Vivid] Hanayamata - 10 [A33D6606]");
+            releaseInfo.DownloadProtocol.Should().Be(DownloadProtocol.Usenet);
+            releaseInfo.DownloadUrl.Should().Be("http://fanzub.com/nzb/296464/Vivid%20Hanayamata%20-%2010.nzb");
+            releaseInfo.InfoUrl.Should().BeNullOrEmpty();
+            releaseInfo.CommentUrl.Should().BeNullOrEmpty();
+            releaseInfo.Indexer.Should().Be(Subject.Definition.Name);
+            releaseInfo.PublishDate.Should().Be(DateTime.Parse("2014/09/13 12:56:53"));
+            releaseInfo.Size.Should().Be(556246858);
+        }
+    }
+}
@@ -20,9 +20,9 @@ namespace NzbDrone.Core.Test.IndexerTests
        {
            _indexers = new List<IIndexer>();

-            _indexers.Add(Mocker.GetMock<Newznab>().Object);
-            _indexers.Add(new Omgwtfnzbs());
-            _indexers.Add(new Wombles());
+            _indexers.Add(Mocker.Resolve<Newznab>());
+            _indexers.Add(Mocker.Resolve<Omgwtfnzbs>());
+            _indexers.Add(Mocker.Resolve<Wombles>());

            Mocker.SetConstant<IEnumerable<IIndexer>>(_indexers);
        }
@@ -13,7 +13,7 @@ using System.Linq;
 namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests
 {
    [IntegrationTest]
-    public class IndexerIntegrationTests : CoreTest<FetchFeedService>
+    public class IndexerIntegrationTests : CoreTest<Wombles>
    {
        [SetUp]
        public void SetUp()
@@ -24,15 +24,13 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests
        [Test]
        public void wombles_rss()
        {
-            var indexer = new Wombles();
-
-            indexer.Definition = new IndexerDefinition
+            Subject.Definition = new IndexerDefinition
            {
                Name = "Wombles",
                Settings = NullConfig.Instance
            };

-            var result = Subject.FetchRss(indexer);
+            var result = Subject.FetchRecent();

            ValidateResult(result, skipSize: true, skipInfo: true);
        }
@@ -0,0 +1,57 @@
+using System;
+using System.Linq;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Newznab;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Test.Framework;
+using NzbDrone.Core.ThingiProvider;
+
+namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
+{
+    [TestFixture]
+    public class NewznabFixture : CoreTest<Newznab>
+    {
+        [SetUp]
+        public void Setup()
+        {
+            Subject.Definition = new IndexerDefinition()
+                {
+                    Name = "Newznab",
+                    Settings = new NewznabSettings()
+                        {
+                            Url = "http://indexer.local/",
+                            Categories = new Int32[] { 1 }
+                        }
+                };
+        }
+
+        [Test]
+        public void should_parse_recent_feed_from_newznab_nzb_su()
+        {
+            var recentFeed = ReadAllText(@"Files/RSS/newznab_nzb_su.xml");
+
+            Mocker.GetMock<IHttpClient>()
+                .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.GET)))
+                .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), recentFeed));
+            
+            var releases = Subject.FetchRecent();
+
+            releases.Should().HaveCount(100);
+
+            var releaseInfo = releases.First();
+
+            releaseInfo.Title.Should().Be("White.Collar.S03E05.720p.HDTV.X264-DIMENSION");
+            releaseInfo.DownloadProtocol.Should().Be(DownloadProtocol.Usenet);
+            releaseInfo.DownloadUrl.Should().Be("http://nzb.su/getnzb/24967ef4c2e26296c65d3bbfa97aa8fe.nzb&i=37292&r=xxx");
+            releaseInfo.InfoUrl.Should().Be("http://nzb.su/details/24967ef4c2e26296c65d3bbfa97aa8fe");
+            releaseInfo.CommentUrl.Should().Be("http://nzb.su/details/24967ef4c2e26296c65d3bbfa97aa8fe#comments");
+            releaseInfo.Indexer.Should().Be(Subject.Definition.Name);
+            releaseInfo.PublishDate.Should().Be(DateTime.Parse("2012/02/27 16:09:39"));
+            releaseInfo.Size.Should().Be(1183105773);
+        }
+    }
+}
@@ -0,0 +1,123 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using FluentAssertions;
+using NUnit.Framework;
+using NzbDrone.Core.Indexers.Newznab;
+using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
+{
+    public class NewznabRequestGeneratorFixture : CoreTest<NewznabRequestGenerator>
+    {
+        AnimeEpisodeSearchCriteria _animeSearchCriteria;
+
+        [SetUp]
+        public void SetUp()
+        {
+            Subject.Settings = new NewznabSettings()
+            {
+                 Url = "http://127.0.0.1:1234/",
+                 Categories = new [] { 1, 2 },
+                 AnimeCategories = new [] { 3, 4 },
+                 ApiKey = "abcd",
+            };
+
+            _animeSearchCriteria = new AnimeEpisodeSearchCriteria()
+            {
+                SceneTitles = new List<String>() { "Monkey+Island" },
+                AbsoluteEpisodeNumber = 100
+            };
+        }
+        
+        [Test]
+        public void should_return_one_page_for_feed()
+        {
+            var results = Subject.GetRecentRequests();
+
+            results.Should().HaveCount(1);
+
+            var pages = results.First().Take(10).ToList();
+
+            pages.Should().HaveCount(1);
+        }
+
+        [Test]
+        public void should_use_all_categories_for_feed()
+        {
+            var results = Subject.GetRecentRequests();
+
+            results.Should().HaveCount(1);
+
+            var page = results.First().First();
+
+            page.Url.Query.Should().Contain("&cat=1,2,3,4&");
+        }
+
+        [Test]
+        public void should_not_have_duplicate_categories()
+        {
+            Subject.Settings.Categories = new[] { 1, 2, 3 };
+
+            var results = Subject.GetRecentRequests();
+
+            results.Should().HaveCount(1);
+
+            var page = results.First().First();
+
+            page.Url.Query.Should().Contain("&cat=1,2,3,4&");
+        }
+
+        [Test]
+        public void should_use_only_anime_categories_for_anime_search()
+        {
+            var results = Subject.GetSearchRequests(_animeSearchCriteria);
+
+            results.Should().HaveCount(1);
+
+            var page = results.First().First();
+
+            page.Url.Query.Should().Contain("&cat=3,4&");
+        }
+        
+        [Test]
+        public void should_use_mode_search_for_anime()
+        {
+            var results = Subject.GetSearchRequests(_animeSearchCriteria);
+
+            results.Should().HaveCount(1);
+
+            var page = results.First().First();
+
+            page.Url.Query.Should().Contain("?t=search&");
+        }
+
+        [Test]
+        public void should_return_subsequent_pages()
+        {
+            var results = Subject.GetSearchRequests(_animeSearchCriteria);
+
+            results.Should().HaveCount(1);
+
+            var pages = results.First().Take(3).ToList();
+
+            pages[0].Url.Query.Should().Contain("&offset=0&");
+            pages[1].Url.Query.Should().Contain("&offset=100&");
+            pages[2].Url.Query.Should().Contain("&offset=200&");
+        }
+
+        [Test]
+        public void should_not_get_unlimited_pages()
+        {
+            var results = Subject.GetSearchRequests(_animeSearchCriteria);
+
+            results.Should().HaveCount(1);
+
+            var pages = results.First().Take(500).ToList();
+
+            pages.Count.Should().BeLessThan(500);
+        }
+    }
+}
@@ -0,0 +1,57 @@
+using System;
+using System.Linq;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Omgwtfnzbs;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Test.Framework;
+using NzbDrone.Core.ThingiProvider;
+
+namespace NzbDrone.Core.Test.IndexerTests.OmgwtfnzbsTests
+{
+    [TestFixture]
+    public class OmgwtfnzbsFixture : CoreTest<Omgwtfnzbs>
+    {
+        [SetUp]
+        public void Setup()
+        {
+            Subject.Definition = new IndexerDefinition()
+                {
+                    Name = "Omgwtfnzbs",
+                    Settings = new OmgwtfnzbsSettings()
+                        {
+                            ApiKey = "xxx",
+                            Username = "me@my.domain"
+                        }
+                };
+        }
+
+        [Test]
+        public void should_parse_recent_feed_from_omgwtfnzbs()
+        {
+            var recentFeed = ReadAllText(@"Files/RSS/omgwtfnzbs.xml");
+
+            Mocker.GetMock<IHttpClient>()
+                .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.GET)))
+                .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), recentFeed));
+            
+            var releases = Subject.FetchRecent();
+
+            releases.Should().HaveCount(100);
+
+            var releaseInfo = releases.First();
+
+            releaseInfo.Title.Should().Be("Stephen.Fry.Gadget.Man.S01E05.HDTV.x264-C4TV");
+            releaseInfo.DownloadProtocol.Should().Be(DownloadProtocol.Usenet);
+            releaseInfo.DownloadUrl.Should().Be("http://api.omgwtfnzbs.org/sn.php?id=OAl4g&user=nzbdrone&api=nzbdrone");
+            releaseInfo.InfoUrl.Should().Be("http://omgwtfnzbs.org/details.php?id=OAl4g");
+            releaseInfo.CommentUrl.Should().BeNullOrEmpty();
+            releaseInfo.Indexer.Should().Be(Subject.Definition.Name);
+            releaseInfo.PublishDate.Should().Be(DateTime.Parse("2012/12/17 23:30:13"));
+            releaseInfo.Size.Should().Be(236822906);
+        }
+    }
+}
@@ -3,10 +3,13 @@ using System.Collections.Generic;
 using FizzWare.NBuilder;
 using FluentValidation.Results;
 using Moq;
+using NLog;
 using NUnit.Framework;
 using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Indexers;
 using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Core.Parser;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Tv;
@@ -15,7 +18,7 @@ using NzbDrone.Test.Common;
 namespace NzbDrone.Core.Test.IndexerTests
 {
    [TestFixture]
-    public class SeasonSearchFixture : TestBase<FetchFeedService>
+    public class SeasonSearchFixture : TestBase<TestIndexer>
    {
        private Series _series;

@@ -25,67 +28,68 @@ namespace NzbDrone.Core.Test.IndexerTests
            _series = Builder<Series>.CreateNew().Build();

            Mocker.GetMock<IHttpClient>()
-                .Setup(o => o.Get(It.IsAny<HttpRequest>()))
+                .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.GET)))
                .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "<xml></xml>"));
        }

-        private IndexerBase<TestIndexerSettings> WithIndexer(bool paging, int resultCount)
+        private void WithIndexer(bool paging, int resultCount)
        {
+            var definition = new IndexerDefinition();
+            definition.Name = "Test";
+            Subject.Definition = definition;
+
+            Subject._supportedPageSize = paging ? 100 : 0;
+
+            var requestGenerator = Mocker.GetMock<IIndexerRequestGenerator>();
+            Subject._requestGenerator = requestGenerator.Object;
+            
+            var requests = Builder<IndexerRequest>.CreateListOfSize(paging ? 100 : 1)
+                .All()
+                .WithConstructor(() => new IndexerRequest("http://my.feed.local/", HttpAccept.Rss))
+                .With(v => v.HttpRequest.Method = HttpMethod.GET)
+                .Build();
+
+            requestGenerator.Setup(s => s.GetSearchRequests(It.IsAny<SeasonSearchCriteria>()))
+                .Returns(new List<IEnumerable<IndexerRequest>> { requests });
+
+            var parser = Mocker.GetMock<IParseIndexerResponse>();
+            Subject._parser = parser.Object;
+
            var results = Builder<ReleaseInfo>.CreateListOfSize(resultCount)
                .Build();

-            var indexer = Mocker.GetMock<IndexerBase<TestIndexerSettings>>();
-
-            indexer.Setup(s => s.Parser.Process(It.IsAny<String>(), It.IsAny<String>()))
+            parser.Setup(s => s.ParseResponse(It.IsAny<IndexerResponse>()))
                   .Returns(results);
-
-            indexer.Setup(s => s.GetSeasonSearchUrls(It.IsAny<List<String>>(), It.IsAny<Int32>(), It.IsAny<Int32>(), It.IsAny<Int32>()))
-                .Returns(new List<string> { "http://www.nzbdrone.com" });
-
-            indexer.SetupGet(s => s.SupportedPageSize).Returns(paging ? 100 : 0);
-
-            var definition = new IndexerDefinition();
-            definition.Name = "Test";
-
-            indexer.SetupGet(s => s.Definition)
-                .Returns(definition);
-
-            return indexer.Object;
        }

        [Test]
        public void should_not_use_offset_if_result_count_is_less_than_90()
        {
-            var indexer = WithIndexer(true, 25);
-            Subject.Fetch(indexer, new SeasonSearchCriteria { Series = _series, SceneTitles = new List<string>{_series.Title} });
+            WithIndexer(true, 25);

-            Mocker.GetMock<IHttpClient>().Verify(v => v.Get(It.IsAny<HttpRequest>()), Times.Once());
+            Subject.Fetch(new SeasonSearchCriteria { Series = _series, SceneTitles = new List<string>{_series.Title} });
+
+            Mocker.GetMock<IHttpClient>().Verify(v => v.Execute(It.IsAny<HttpRequest>()), Times.Once());
        }

        [Test]
        public void should_not_use_offset_for_sites_that_do_not_support_it()
        {
-            var indexer = WithIndexer(false, 125);
-            Subject.Fetch(indexer, new SeasonSearchCriteria { Series = _series, SceneTitles = new List<string> { _series.Title } });
+            WithIndexer(false, 125);

-            Mocker.GetMock<IHttpClient>().Verify(v => v.Get(It.IsAny<HttpRequest>()), Times.Once());
+            Subject.Fetch(new SeasonSearchCriteria { Series = _series, SceneTitles = new List<string> { _series.Title } });
+
+            Mocker.GetMock<IHttpClient>().Verify(v => v.Execute(It.IsAny<HttpRequest>()), Times.Once());
        }

        [Test]
        public void should_not_use_offset_if_its_already_tried_10_times()
        {
-            var indexer = WithIndexer(true, 100);
-            Subject.Fetch(indexer, new SeasonSearchCriteria { Series = _series, SceneTitles = new List<string> { _series.Title } });
+            WithIndexer(true, 100);

-            Mocker.GetMock<IHttpClient>().Verify(v => v.Get(It.IsAny<HttpRequest>()), Times.Exactly(10));
-        }
-    }
+            Subject.Fetch(new SeasonSearchCriteria { Series = _series, SceneTitles = new List<string> { _series.Title } });

-    public class TestIndexerSettings : IProviderConfig
-    {
-        public ValidationResult Validate()
-        {
-            throw new NotImplementedException();
+            Mocker.GetMock<IHttpClient>().Verify(v => v.Execute(It.IsAny<HttpRequest>()), Times.Exactly(10));
        }
    }
 }
@@ -0,0 +1,38 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using NLog;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Parser;
+
+namespace NzbDrone.Core.Test.IndexerTests
+{
+    public class TestIndexer : HttpIndexerBase<TestIndexerSettings>
+    {
+        public override DownloadProtocol Protocol { get { return DownloadProtocol.Usenet; } }
+
+        public Int32 _supportedPageSize;
+        public override Int32 PageSize { get { return _supportedPageSize; } }
+
+        public TestIndexer(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
+            : base(httpClient, configService, parsingService, logger)
+        {
+
+        }
+
+        public IIndexerRequestGenerator _requestGenerator;
+        public override IIndexerRequestGenerator GetRequestGenerator()
+        {
+            return _requestGenerator;
+        }
+
+        public IParseIndexerResponse _parser;
+        public override IParseIndexerResponse GetParser()
+        {
+            return _parser;
+        }
+    }
+}
@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using FluentValidation.Results;
+using NzbDrone.Core.ThingiProvider;
+
+namespace NzbDrone.Core.Test.IndexerTests
+{
+    public class TestIndexerSettings : IProviderConfig
+    {
+        public ValidationResult Validate()
+        {
+            throw new NotImplementedException();
+        }
+    }
+}