New: Don't import episodes that don't match grab history

(cherry picked from commit 978618f041c478121f8e014910ad092f8e648596)

azure-pipelines.yml

@@ -9,7 +9,7 @@ variables:
   testsFolder: './_tests'
   yarnCacheFolder: $(Pipeline.Workspace)/.yarn
   nugetCacheFolder: $(Pipeline.Workspace)/.nuget/packages
-  majorVersion: '0.1.3'
+  majorVersion: '0.1.5'
   minorVersion: $[counter('minorVersion', 1)]
   readarrVersion: '$(majorVersion).$(minorVersion)'
   buildName: '$(Build.SourceBranchName).$(readarrVersion)'

frontend/src/Helpers/Props/filterBuilderTypes.js

@@ -47,6 +47,10 @@ export const possibleFilterTypes = {
     { key: filterTypes.CONTAINS, value: 'contains' },
     { key: filterTypes.NOT_CONTAINS, value: 'does not contain' },
     { key: filterTypes.EQUAL, value: 'equal' },
-    { key: filterTypes.NOT_EQUAL, value: 'not equal' }
+    { key: filterTypes.NOT_EQUAL, value: 'not equal' },
+    { key: filterTypes.STARTS_WITH, value: 'starts with' },
+    { key: filterTypes.NOT_STARTS_WITH, value: 'does not start with' },
+    { key: filterTypes.ENDS_WITH, value: 'ends with' },
+    { key: filterTypes.NOT_ENDS_WITH, value: 'does not end with' }
   ]
 };

frontend/src/Helpers/Props/filterTypePredicates.js

@@ -39,6 +39,22 @@ const filterTypePredicates = {
 
   [filterTypes.NOT_EQUAL]: function(itemValue, filterValue) {
     return itemValue !== filterValue;
+  },
+
+  [filterTypes.STARTS_WITH]: function(itemValue, filterValue) {
+    return itemValue.toLowerCase().startsWith(filterValue.toLowerCase());
+  },
+
+  [filterTypes.NOT_STARTS_WITH]: function(itemValue, filterValue) {
+    return !itemValue.toLowerCase().startsWith(filterValue.toLowerCase());
+  },
+
+  [filterTypes.ENDS_WITH]: function(itemValue, filterValue) {
+    return itemValue.toLowerCase().endsWith(filterValue.toLowerCase());
+  },
+
+  [filterTypes.NOT_ENDS_WITH]: function(itemValue, filterValue) {
+    return !itemValue.toLowerCase().endsWith(filterValue.toLowerCase());
   }
 };
 

frontend/src/Helpers/Props/filterTypes.js

@@ -10,6 +10,10 @@ export const LESS_THAN = 'lessThan';
 export const LESS_THAN_OR_EQUAL = 'lessThanOrEqual';
 export const NOT_CONTAINS = 'notContains';
 export const NOT_EQUAL = 'notEqual';
+export const STARTS_WITH = 'startsWith';
+export const NOT_STARTS_WITH = 'notStartsWith';
+export const ENDS_WITH = 'endsWith';
+export const NOT_ENDS_WITH = 'notEndsWith';
 
 export const all = [
   CONTAINS,
@@ -23,5 +27,9 @@ export const all = [
   IN_LAST,
   NOT_IN_LAST,
   IN_NEXT,
-  NOT_IN_NEXT
+  NOT_IN_NEXT,
+  STARTS_WITH,
+  NOT_STARTS_WITH,
+  ENDS_WITH,
+  NOT_ENDS_WITH
 ];

src/NzbDrone.Core.Test/Download/DownloadClientTests/TransmissionTests/TransmissionFixture.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using FluentAssertions;
@@ -275,7 +276,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.TransmissionTests
 
         [TestCase(-1)] // Infinite/Unknown
         [TestCase(-2)] // Magnet Downloading
-        public void should_ignore_negative_eta(int eta)
+        public void should_ignore_negative_eta(long eta)
         {
             _completed.Eta = eta;
 
@@ -284,6 +285,26 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.TransmissionTests
             item.RemainingTime.Should().NotHaveValue();
         }
 
+        [TestCase(2147483648)] // 2038-01-19T03:14:08Z > int.MaxValue as unix timestamp can be either an int or a long
+        public void should_support_long_values_for_eta_in_seconds(long eta)
+        {
+            _downloading.Eta = eta;
+
+            PrepareClientToReturnDownloadingItem();
+            var item = Subject.GetItems().Single();
+            item.RemainingTime.Should().Be(TimeSpan.FromSeconds(eta));
+        }
+
+        [TestCase(2147483648000)] // works with milliseconds format too
+        public void should_support_long_values_for_eta_in_milliseconds(long eta)
+        {
+            _downloading.Eta = eta;
+
+            PrepareClientToReturnDownloadingItem();
+            var item = Subject.GetItems().Single();
+            item.RemainingTime.Should().Be(TimeSpan.FromMilliseconds(eta));
+        }
+
         [Test]
         public void should_not_be_removable_and_should_not_allow_move_files_if_max_ratio_reached_and_not_stopped()
         {

src/NzbDrone.Core.Test/Download/DownloadClientTests/VuzeTests/VuzeFixture.cs

@@ -271,7 +271,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.VuzeTests
 
         [TestCase(-1)] // Infinite/Unknown
         [TestCase(-2)] // Magnet Downloading
-        public void should_ignore_negative_eta(int eta)
+        public void should_ignore_negative_eta(long eta)
         {
             _completed.Eta = eta;
 

src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Specifications/MatchesGrabSpecificationFixture.cs

@@ -0,0 +1,119 @@
+using System.Collections.Generic;
+using FizzWare.NBuilder;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.Download;
+using NzbDrone.Core.History;
+using NzbDrone.Core.MediaFiles.EpisodeImport.Specifications;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Test.Framework;
+using NzbDrone.Core.Tv;
+using NzbDrone.Test.Common;
+
+namespace NzbDrone.Core.Test.MediaFiles.EpisodeImport.Specifications
+{
+    [TestFixture]
+    public class MatchesGrabSpecificationFixture : CoreTest<MatchesGrabSpecification>
+    {
+        private Episode _episode1;
+        private Episode _episode2;
+        private Episode _episode3;
+        private LocalEpisode _localEpisode;
+        private DownloadClientItem _downloadClientItem;
+
+        [SetUp]
+        public void Setup()
+        {
+            _episode1 = Builder<Episode>.CreateNew()
+                .With(e => e.Id = 1)
+                .Build();
+
+            _episode2 = Builder<Episode>.CreateNew()
+                .With(e => e.Id = 2)
+                .Build();
+
+            _episode3 = Builder<Episode>.CreateNew()
+                .With(e => e.Id = 3)
+                .Build();
+
+            _localEpisode = Builder<LocalEpisode>.CreateNew()
+                                                 .With(l => l.Path = @"C:\Test\Unsorted\Series.Title.S01E01.720p.HDTV-Sonarr\S01E05.mkv".AsOsAgnostic())
+                                                 .With(l => l.Episodes = new List<Episode> { _episode1 })
+                                                 .Build();
+
+            _downloadClientItem = Builder<DownloadClientItem>.CreateNew().Build();
+        }
+
+        private void GivenHistoryForEpisodes(params Episode[] episodes)
+        {
+            var history = new List<EpisodeHistory>();
+
+            foreach (var episode in episodes)
+            {
+                history.Add(Builder<EpisodeHistory>.CreateNew()
+                    .With(h => h.EventType = EpisodeHistoryEventType.Grabbed)
+                    .With(h => h.EpisodeId = episode.Id)
+                    .Build());
+            }
+
+            Mocker.GetMock<IHistoryService>()
+                  .Setup(s => s.FindByDownloadId(It.IsAny<string>()))
+                  .Returns(history);
+        }
+
+        [Test]
+        public void should_be_accepted_for_existing_file()
+        {
+            _localEpisode.ExistingFile = true;
+
+            Subject.IsSatisfiedBy(_localEpisode, null).Accepted.Should().BeTrue();
+        }
+
+        [Test]
+        public void should_be_accepted_if_no_download_client_item()
+        {
+            Subject.IsSatisfiedBy(_localEpisode, null).Accepted.Should().BeTrue();
+        }
+
+        [Test]
+        public void should_be_accepted_if_no_grab_history()
+        {
+            GivenHistoryForEpisodes();
+
+            Subject.IsSatisfiedBy(_localEpisode, _downloadClientItem).Accepted.Should().BeTrue();
+        }
+
+        [Test]
+        public void should_be_accepted_if_file_episode_matches_single_grab_history()
+        {
+            GivenHistoryForEpisodes(_episode1);
+
+            Subject.IsSatisfiedBy(_localEpisode, _downloadClientItem).Accepted.Should().BeTrue();
+        }
+
+        [Test]
+        public void should_be_accepted_if_file_episode_is_in_multi_episode_grab_history()
+        {
+            GivenHistoryForEpisodes(_episode1, _episode2);
+
+            Subject.IsSatisfiedBy(_localEpisode, _downloadClientItem).Accepted.Should().BeTrue();
+        }
+
+        [Test]
+        public void should_be_rejected_if_file_episode_does_not_match_single_grab_history()
+        {
+            GivenHistoryForEpisodes(_episode2);
+
+            Subject.IsSatisfiedBy(_localEpisode, _downloadClientItem).Accepted.Should().BeFalse();
+        }
+
+        [Test]
+        public void should_be_rejected_if_file_episode_is_not_in_multi_episode_grab_history()
+        {
+            GivenHistoryForEpisodes(_episode2, _episode3);
+
+            Subject.IsSatisfiedBy(_localEpisode, _downloadClientItem).Accepted.Should().BeFalse();
+        }
+    }
+}

src/NzbDrone.Core/Download/Clients/Transmission/TransmissionBase.cs

@@ -80,7 +80,14 @@ namespace NzbDrone.Core.Download.Clients.Transmission
 
                 if (torrent.Eta >= 0)
                 {
-                    item.RemainingTime = TimeSpan.FromSeconds(torrent.Eta);
+                    try
+                    {
+                        item.RemainingTime = TimeSpan.FromSeconds(torrent.Eta);
+                    }
+                    catch (OverflowException)
+                    {
+                        item.RemainingTime = TimeSpan.FromMilliseconds(torrent.Eta);
+                    }
                 }
 
                 if (!torrent.ErrorString.IsNullOrWhiteSpace())

src/NzbDrone.Core/Download/Clients/Transmission/TransmissionTorrent.cs

@@ -9,7 +9,7 @@ namespace NzbDrone.Core.Download.Clients.Transmission
         public long TotalSize { get; set; }
         public long LeftUntilDone { get; set; }
         public bool IsFinished { get; set; }
-        public int Eta { get; set; }
+        public long Eta { get; set; }
         public TransmissionTorrentStatus Status { get; set; }
         public int SecondsDownloading { get; set; }
         public int SecondsSeeding { get; set; }

src/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/MatchesGrabSpecification.cs

@@ -0,0 +1,70 @@
+using System.Collections.Generic;
+using System.Linq;
+using NLog;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.DecisionEngine;
+using NzbDrone.Core.Download;
+using NzbDrone.Core.History;
+using NzbDrone.Core.Parser;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.Tv;
+
+namespace NzbDrone.Core.MediaFiles.EpisodeImport.Specifications
+{
+    public class MatchesGrabSpecification : IImportDecisionEngineSpecification
+    {
+        private readonly Logger _logger;
+        private readonly IParsingService _parsingService;
+        private readonly IHistoryService _historyService;
+
+        public MatchesGrabSpecification(IParsingService parsingService, IHistoryService historyService, Logger logger)
+        {
+            _logger = logger;
+            _parsingService = parsingService;
+            _historyService = historyService;
+        }
+
+        public Decision IsSatisfiedBy(LocalEpisode localEpisode, DownloadClientItem downloadClientItem)
+        {
+            if (localEpisode.ExistingFile)
+            {
+                return Decision.Accept();
+            }
+
+            if (downloadClientItem == null)
+            {
+                return Decision.Accept();
+            }
+
+            var grabbedHistory = _historyService.FindByDownloadId(downloadClientItem.DownloadId)
+                .Where(h => h.EventType == EpisodeHistoryEventType.Grabbed)
+                .ToList();
+
+            if (grabbedHistory.Empty())
+            {
+                return Decision.Accept();
+            }
+
+            var unexpected = localEpisode.Episodes.Where(e => grabbedHistory.All(o => o.EpisodeId != e.Id)).ToList();
+
+            if (unexpected.Any())
+            {
+                _logger.Debug("Unexpected episode(s) in file: {0}", FormatEpisode(unexpected));
+
+                if (unexpected.Count == 1)
+                {
+                    return Decision.Reject("Episode {0} was not found in the grabbed release: {1}", FormatEpisode(unexpected), grabbedHistory.First().SourceTitle);
+                }
+
+                return Decision.Reject("Episodes {0} were not found in the grabbed release: {1}", FormatEpisode(unexpected), grabbedHistory.First().SourceTitle);
+            }
+
+            return Decision.Accept();
+        }
+
+        private string FormatEpisode(List<Episode> episodes)
+        {
+            return string.Join(", ", episodes.Select(e => $"{e.SeasonNumber}x{e.EpisodeNumber:00}"));
+        }
+    }
+}

src/NzbDrone.Core/MetadataSource/Goodreads/Resources/BestBookResource.cs

@@ -51,7 +51,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
             }
 
             ImageUrl = element.ElementAsString("image_url");
-            ImageUrl = element.ElementAsString("large_image_url");
+            LargeImageUrl = element.ElementAsString("large_image_url");
         }
     }
 }

src/NzbDrone.Core/Notifications/Webhook/WebhookBase.cs

@@ -28,13 +28,7 @@ namespace NzbDrone.Core.Notifications.Webhook
                 EventType = WebhookEventType.Grab,
                 InstanceName = _configFileProvider.InstanceName,
                 Author = new WebhookAuthor(message.Author),
-                Books = remoteBook.Books.ConvertAll(x => new WebhookBook(x)
-                {
-                    // TODO: Stop passing these parameters inside an book v3
-                    Quality = quality.Quality.Name,
-                    QualityVersion = quality.Revision.Version,
-                    ReleaseGroup = remoteBook.ParsedBookInfo.ReleaseGroup
-                }),
+                Books = remoteBook.Books.ConvertAll(x => new WebhookBook(x)),
                 Release = new WebhookRelease(quality, remoteBook),
                 DownloadClient = message.DownloadClientName,
                 DownloadClientType = message.DownloadClientType,
@@ -93,7 +87,7 @@ namespace NzbDrone.Core.Notifications.Webhook
         {
             return new WebhookBookDeletePayload
             {
-                EventType = WebhookEventType.Delete,
+                EventType = WebhookEventType.BookDelete,
                 InstanceName = _configFileProvider.InstanceName,
                 Author = new WebhookAuthor(deleteMessage.Book.Author),
                 Book = new WebhookBook(deleteMessage.Book),
@@ -105,7 +99,7 @@ namespace NzbDrone.Core.Notifications.Webhook
         {
             return new WebhookBookFileDeletePayload
             {
-                EventType = WebhookEventType.Delete,
+                EventType = WebhookEventType.BookFileDelete,
                 InstanceName = _configFileProvider.InstanceName,
                 Author = new WebhookAuthor(deleteMessage.Book.Author),
                 Book = new WebhookBook(deleteMessage.Book),
@@ -117,7 +111,7 @@ namespace NzbDrone.Core.Notifications.Webhook
         {
             return new WebhookAuthorDeletePayload
             {
-                EventType = WebhookEventType.Delete,
+                EventType = WebhookEventType.AuthorDelete,
                 InstanceName = _configFileProvider.InstanceName,
                 Author = new WebhookAuthor(deleteMessage.Author),
                 DeletedFiles = deleteMessage.DeletedFiles

src/NzbDrone.Core/Notifications/Webhook/WebhookBook.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using NzbDrone.Core.Books;
 
 namespace NzbDrone.Core.Notifications.Webhook
@@ -12,17 +14,16 @@ namespace NzbDrone.Core.Notifications.Webhook
         public WebhookBook(Book book)
         {
             Id = book.Id;
+            GoodreadsId = book.ForeignBookId;
             Title = book.Title;
             ReleaseDate = book.ReleaseDate;
+            Edition = new WebhookBookEdition(book.Editions.Value.Single(e => e.Monitored));
         }
 
         public int Id { get; set; }
+        public string GoodreadsId { get; set; }
         public string Title { get; set; }
+        public WebhookBookEdition Edition { get; set; }
         public DateTime? ReleaseDate { get; set; }
-
-        public string Quality { get; set; }
-        public int QualityVersion { get; set; }
-        public string ReleaseGroup { get; set; }
-        public string SceneName { get; set; }
     }
 }

src/NzbDrone.Core/Notifications/Webhook/WebhookBookEdition.cs

@@ -0,0 +1,20 @@
+using NzbDrone.Core.Books;
+
+namespace NzbDrone.Core.Notifications.Webhook
+{
+    public class WebhookBookEdition
+    {
+        public WebhookBookEdition(Edition edition)
+        {
+            GoodreadsId = edition.ForeignEditionId;
+            Title = edition.Title;
+            Asin = edition.Asin;
+            Isbn13 = edition.Isbn13;
+        }
+
+        public string Title { get; set; }
+        public string GoodreadsId { get; set; }
+        public string Asin { get; set; }
+        public string Isbn13 { get; set; }
+    }
+}

src/NzbDrone.Core/Notifications/Webhook/WebhookEventType.cs

@@ -12,7 +12,9 @@ namespace NzbDrone.Core.Notifications.Webhook
         Grab,
         Download,
         Rename,
-        Delete,
+        AuthorDelete,
+        BookDelete,
+        BookFileDelete,
         Health,
         Retag,
         ApplicationUpdate

src/NzbDrone.Host/Bootstrap.cs

@@ -14,6 +14,7 @@ using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Hosting.WindowsServices;
 using NLog;
+using Npgsql;
 using NzbDrone.Common.Composition.Extensions;
 using NzbDrone.Common.Disk;
 using NzbDrone.Common.EnvironmentInfo;
@@ -55,6 +56,7 @@ namespace NzbDrone.Host
                 Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
 
                 var appMode = GetApplicationMode(startupContext);
+                var config = GetConfiguration(startupContext);
 
                 switch (appMode)
                 {
@@ -83,12 +85,22 @@ namespace NzbDrone.Host
                     // Utility mode
                     default:
                     {
-                        new Container(rules => rules.WithNzbDroneRules())
-                            .AutoAddServices(ASSEMBLIES)
-                            .AddNzbDroneLogger()
-                            .AddStartupContext(startupContext)
-                            .Resolve<UtilityModeRouter>()
-                            .Route(appMode);
+                        new HostBuilder()
+                            .UseServiceProviderFactory(new DryIocServiceProviderFactory(new Container(rules => rules.WithNzbDroneRules())))
+                            .ConfigureContainer<IContainer>(c =>
+                            {
+                                c.AutoAddServices(Bootstrap.ASSEMBLIES)
+                                    .AddNzbDroneLogger()
+                                    .AddDatabase()
+                                    .AddStartupContext(startupContext)
+                                    .Resolve<UtilityModeRouter>()
+                                    .Route(appMode);
+                            })
+                            .ConfigureServices(services =>
+                            {
+                                services.Configure<PostgresOptions>(config.GetSection("Readarr:Postgres"));
+                            }).Build();
+
                         break;
                     }
                 }
@@ -111,6 +123,7 @@ namespace NzbDrone.Host
             GC.Collect();
             GC.WaitForPendingFinalizers();
             SQLiteConnection.ClearAllPools();
+            NpgsqlConnection.ClearAllPools();
         }
 
         public static IHostBuilder CreateConsoleHostBuilder(string[] args, StartupContext context)