tedkulp/Sonarr
1
0
Fork 0
mirror of https://github.com/Sonarr/Sonarr.git synced 2026-03-05 13:20:20 -05:00
Code Issues Projects Releases Wiki Activity

Fixed: Loading queue or processing releases slow with many Pending Releases

Browse Source
This commit is contained in:
Mark McDowall
2025-10-25 16:34:17 -07:00
parent 550cf8d399
commit 064cbdbfb1
8 changed files with 131 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs
View File

@@ -8,6 +8,7 @@ using NzbDrone.Common.Extensions;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.DecisionEngine;
using NzbDrone.Core.Download.Pending;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Profiles.Qualities;
@@ -54,8 +55,10 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
_release = Builder<ReleaseInfo>.CreateNew().Build();
_parsedEpisodeInfo = Builder<ParsedEpisodeInfo>.CreateNew().Build();
_parsedEpisodeInfo.Quality = new QualityModel(Quality.HDTV720p);
_parsedEpisodeInfo = Builder<ParsedEpisodeInfo>.CreateNew()
.With(p => p.Quality = new QualityModel(Quality.HDTV720p))
.With(p => p.AirDate = null)
.Build();
_remoteEpisode = new RemoteEpisode();
_remoteEpisode.Episodes = new List<Episode> { _episode };
@@ -87,6 +90,10 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
.Setup(s => s.GetEpisodes(It.IsAny<ParsedEpisodeInfo>(), _series, true, null))
.Returns(new List<Episode> { _episode });
Mocker.GetMock<IParsingService>()
.Setup(s => s.Map(It.IsAny<ParsedEpisodeInfo>(), _series))
.Returns(_remoteEpisode);
Mocker.GetMock<IPrioritizeDownloadDecision>()
.Setup(s => s.PrioritizeDecisions(It.IsAny<List<DownloadDecision>>()))
.Returns((List<DownloadDecision> d) => d);
@@ -110,9 +117,15 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
_heldReleases.AddRange(heldReleases);
}
private void InitializeReleases()
{
Subject.Handle(new ApplicationStartedEvent());
}
[Test]
public void should_add()
{
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
VerifyInsert();
@@ -123,6 +136,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
{
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate);
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
VerifyNoInsert();
@@ -134,6 +148,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.DownloadClientUnavailable);
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.Fallback);
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
VerifyNoInsert();
@@ -145,6 +160,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.DownloadClientUnavailable);
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.Fallback);
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Fallback);
Mocker.GetMock<IPendingReleaseRepository>()
@@ -156,6 +172,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
{
GivenHeldRelease(_release.Title + "-RP", _release.Indexer, _release.PublishDate);
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
VerifyInsert();
@@ -166,6 +183,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
{
GivenHeldRelease(_release.Title, "AnotherIndexer", _release.PublishDate);
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
VerifyInsert();
@@ -176,6 +194,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
{
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate.AddHours(1));
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
VerifyInsert();

9
src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs
View File

@@ -7,6 +7,7 @@ using NzbDrone.Common.Extensions;
using NzbDrone.Core.DecisionEngine;
using NzbDrone.Core.Download;
using NzbDrone.Core.Download.Pending;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Profiles.Qualities;
@@ -112,11 +113,17 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
_heldReleases.AddRange(heldReleases);
}
private void InitializeReleases()
{
Subject.Handle(new ApplicationStartedEvent());
}
[Test]
public void should_delete_if_the_grabbed_quality_is_the_same()
{
GivenHeldRelease(_parsedEpisodeInfo.Quality);
InitializeReleases();
Subject.Handle(new EpisodeGrabbedEvent(_remoteEpisode));
VerifyDelete();
@@ -127,6 +134,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
{
GivenHeldRelease(new QualityModel(Quality.SDTV));
InitializeReleases();
Subject.Handle(new EpisodeGrabbedEvent(_remoteEpisode));
VerifyDelete();
@@ -137,6 +145,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
{
GivenHeldRelease(new QualityModel(Quality.Bluray720p));
InitializeReleases();
Subject.Handle(new EpisodeGrabbedEvent(_remoteEpisode));
VerifyNoDelete();

12
src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingFixture.cs
View File

@@ -5,6 +5,7 @@ using Moq;
using NUnit.Framework;
using NzbDrone.Common.Crypto;
using NzbDrone.Core.Download.Pending;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Test.Framework;
@@ -62,6 +63,11 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
});
}
private void InitializeReleases()
{
Subject.Handle(new ApplicationStartedEvent());
}
[Test]
public void should_remove_same_release()
{
@@ -69,6 +75,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
var queueId = HashConverter.GetHashInt31($"pending-{1}");
InitializeReleases();
Subject.RemovePendingQueueItems(queueId);
AssertRemoved(1);
@@ -84,6 +91,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
var queueId = HashConverter.GetHashInt31($"pending-{3}");
InitializeReleases();
Subject.RemovePendingQueueItems(queueId);
AssertRemoved(3, 4);
@@ -99,6 +107,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
var queueId = HashConverter.GetHashInt31($"pending-{1}");
InitializeReleases();
Subject.RemovePendingQueueItems(queueId);
AssertRemoved(1, 2);
@@ -114,6 +123,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
var queueId = HashConverter.GetHashInt31($"pending-{1}");
InitializeReleases();
Subject.RemovePendingQueueItems(queueId);
AssertRemoved(1, 2);
@@ -127,6 +137,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
var queueId = HashConverter.GetHashInt31($"pending-{1}");
InitializeReleases();
Subject.RemovePendingQueueItems(queueId);
AssertRemoved(1);
@@ -140,6 +151,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
var queueId = HashConverter.GetHashInt31($"pending-{2}");
InitializeReleases();
Subject.RemovePendingQueueItems(queueId);
AssertRemoved(2);

12
src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingObsoleteFixture.cs
View File

@@ -5,6 +5,7 @@ using Moq;
using NUnit.Framework;
using NzbDrone.Common.Crypto;
using NzbDrone.Core.Download.Pending;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Test.Framework;
@@ -62,6 +63,11 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
});
}
private void InitializeReleases()
{
Subject.Handle(new ApplicationStartedEvent());
}
[Test]
public void should_remove_same_release()
{
@@ -69,6 +75,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 1, _episode.Id));
InitializeReleases();
Subject.RemovePendingQueueItemsObsolete(queueId);
AssertRemoved(1);
@@ -84,6 +91,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 3, _episode.Id));
InitializeReleases();
Subject.RemovePendingQueueItemsObsolete(queueId);
AssertRemoved(3, 4);
@@ -99,6 +107,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 1, _episode.Id));
InitializeReleases();
Subject.RemovePendingQueueItemsObsolete(queueId);
AssertRemoved(1, 2);
@@ -114,6 +123,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 1, _episode.Id));
InitializeReleases();
Subject.RemovePendingQueueItemsObsolete(queueId);
AssertRemoved(1, 2);
@@ -127,6 +137,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 1, _episode.Id));
InitializeReleases();
Subject.RemovePendingQueueItemsObsolete(queueId);
AssertRemoved(1);
@@ -140,6 +151,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 2, _episode.Id));
InitializeReleases();
Subject.RemovePendingQueueItemsObsolete(queueId);
AssertRemoved(2);

10
src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveRejectedFixture.cs
View File

@@ -9,6 +9,7 @@ using NzbDrone.Core.DecisionEngine;
using NzbDrone.Core.Download;
using NzbDrone.Core.Download.Pending;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Profiles.Qualities;
@@ -109,11 +110,17 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
.Returns(heldReleases);
}
private void InitializeReleases()
{
Subject.Handle(new ApplicationStartedEvent());
}
[Test]
public void should_remove_if_it_is_the_same_release_from_the_same_indexer()
{
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate);
InitializeReleases();
Subject.Handle(new RssSyncCompleteEvent(new ProcessedDecisions(new List<DownloadDecision>(),
new List<DownloadDecision>(),
new List<DownloadDecision> { _temporarilyRejected })));
@@ -126,6 +133,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
{
GivenHeldRelease(_release.Title + "-RP", _release.Indexer, _release.PublishDate);
InitializeReleases();
Subject.Handle(new RssSyncCompleteEvent(new ProcessedDecisions(new List<DownloadDecision>(),
new List<DownloadDecision>(),
new List<DownloadDecision> { _temporarilyRejected })));
@@ -138,6 +146,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
{
GivenHeldRelease(_release.Title, "AnotherIndexer", _release.PublishDate);
InitializeReleases();
Subject.Handle(new RssSyncCompleteEvent(new ProcessedDecisions(new List<DownloadDecision>(),
new List<DownloadDecision>(),
new List<DownloadDecision> { _temporarilyRejected })));
@@ -150,6 +159,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
{
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate.AddHours(1));
InitializeReleases();
Subject.Handle(new RssSyncCompleteEvent(new ProcessedDecisions(new List<DownloadDecision>(),
new List<DownloadDecision>(),
new List<DownloadDecision> { _temporarilyRejected })));

66
src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs
View File

@@ -5,15 +5,18 @@ using NLog;
using NzbDrone.Common.Crypto;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Configuration.Events;
using NzbDrone.Core.CustomFormats;
using NzbDrone.Core.DecisionEngine;
using NzbDrone.Core.Download.Aggregation;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Jobs;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Profiles.Delay;
using NzbDrone.Core.Profiles.Qualities;
using NzbDrone.Core.Qualities;
using NzbDrone.Core.Queue;
using NzbDrone.Core.Tv;
@@ -37,9 +40,14 @@ namespace NzbDrone.Core.Download.Pending
}
public class PendingReleaseService : IPendingReleaseService,
IHandle<SeriesEditedEvent>,
IHandle<SeriesUpdatedEvent>,
IHandle<SeriesDeletedEvent>,
IHandle<EpisodeGrabbedEvent>,
IHandle<RssSyncCompleteEvent>
IHandle<RssSyncCompleteEvent>,
IHandle<QualityProfileUpdatedEvent>,
IHandle<ConfigSavedEvent>,
IHandle<ApplicationStartedEvent>
{
private readonly IIndexerStatusService _indexerStatusService;
private readonly IPendingReleaseRepository _repository;
@@ -55,6 +63,8 @@ namespace NzbDrone.Core.Download.Pending
private readonly IEventAggregator _eventAggregator;
private readonly Logger _logger;
private static List<PendingRelease> _pendingReleases = new();
public PendingReleaseService(IIndexerStatusService indexerStatusService,
IPendingReleaseRepository repository,
ISeriesService seriesService,
@@ -91,11 +101,14 @@ namespace NzbDrone.Core.Download.Pending
public void AddMany(List<Tuple<DownloadDecision, PendingReleaseReason>> decisions)
{
var pending = _pendingReleases;
foreach (var seriesDecisions in decisions.GroupBy(v => v.Item1.RemoteEpisode.Series.Id))
{
var series = seriesDecisions.First().Item1.RemoteEpisode.Series;
var alreadyPending = _repository.AllBySeriesId(series.Id);
var alreadyPending = _pendingReleases.Where(p => p.SeriesId == series.Id).SelectList(s => s.JsonClone());
// TODO: Do we need IncludeRemoteEpisodes?
alreadyPending = IncludeRemoteEpisodes(alreadyPending, seriesDecisions.ToDictionaryIgnoreDuplicates(v => v.Item1.RemoteEpisode.Release.Title, v => v.Item1.RemoteEpisode));
var alreadyPendingByEpisode = CreateEpisodeLookup(alreadyPending);
@@ -152,6 +165,8 @@ namespace NzbDrone.Core.Download.Pending
Insert(decision, reason);
}
}
UpdatePendingReleases();
}
public List<ReleaseInfo> GetPending()
@@ -175,16 +190,14 @@ namespace NzbDrone.Core.Download.Pending
public List<RemoteEpisode> GetPendingRemoteEpisodes(int seriesId)
{
return IncludeRemoteEpisodes(_repository.AllBySeriesId(seriesId)).Select(v => v.RemoteEpisode).ToList();
return _pendingReleases.Where(p => p.SeriesId == seriesId).Select(p => p.RemoteEpisode).ToList();
}
public List<Queue.Queue> GetPendingQueue()
{
var queued = new List<Queue.Queue>();
var nextRssSync = new Lazy<DateTime>(() => _taskManager.GetNextExecution(typeof(RssSyncCommand)));
var pendingReleases = IncludeRemoteEpisodes(_repository.WithoutFallback());
var pendingReleases = _pendingReleases.Where(p => p.Reason != PendingReleaseReason.Fallback).ToList();
foreach (var pendingRelease in pendingReleases)
{
@@ -218,10 +231,8 @@ namespace NzbDrone.Core.Download.Pending
public List<Queue.Queue> GetPendingQueueObsolete()
{
var queued = new List<Queue.Queue>();
var nextRssSync = new Lazy<DateTime>(() => _taskManager.GetNextExecution(typeof(RssSyncCommand)));
var pendingReleases = IncludeRemoteEpisodes(_repository.WithoutFallback());
var pendingReleases = _pendingReleases.Where(p => p.Reason != PendingReleaseReason.Fallback).ToList();
foreach (var pendingRelease in pendingReleases)
{
@@ -317,12 +328,12 @@ namespace NzbDrone.Core.Download.Pending
private List<PendingRelease> GetPendingReleases()
{
return IncludeRemoteEpisodes(_repository.All().ToList());
return _pendingReleases;
}
private List<PendingRelease> GetPendingReleases(int seriesId)
{
return IncludeRemoteEpisodes(_repository.AllBySeriesId(seriesId).ToList());
return _pendingReleases.Where(p => p.SeriesId == seriesId).ToList();
}
private List<PendingRelease> IncludeRemoteEpisodes(List<PendingRelease> releases, Dictionary<string, RemoteEpisode> knownRemoteEpisodes = null)
@@ -632,19 +643,52 @@ namespace NzbDrone.Core.Download.Pending
return 1;
}
private void UpdatePendingReleases()
{
_pendingReleases = IncludeRemoteEpisodes(_repository.All().ToList());
}
public void Handle(SeriesEditedEvent message)
{
UpdatePendingReleases();
}
public void Handle(SeriesUpdatedEvent message)
{
UpdatePendingReleases();
}
public void Handle(SeriesDeletedEvent message)
{
_repository.DeleteBySeriesIds(message.Series.Select(m => m.Id).ToList());
UpdatePendingReleases();
}
public void Handle(EpisodeGrabbedEvent message)
{
RemoveGrabbed(message.Episode);
UpdatePendingReleases();
}
public void Handle(RssSyncCompleteEvent message)
{
RemoveRejected(message.ProcessedDecisions.Rejected);
UpdatePendingReleases();
}
public void Handle(QualityProfileUpdatedEvent message)
{
UpdatePendingReleases();
}
public void Handle(ApplicationStartedEvent message)
{
UpdatePendingReleases();
}
public void Handle(ConfigSavedEvent message)
{
UpdatePendingReleases();
}
private static Func<PendingRelease, bool> MatchingReleasePredicate(ReleaseInfo release)

4
src/NzbDrone.Core/Profiles/Qualities/QualityProfileService.cs
View File

@@ -33,18 +33,21 @@ namespace NzbDrone.Core.Profiles.Qualities
private readonly IImportListFactory _importListFactory;
private readonly ICustomFormatService _formatService;
private readonly ISeriesService _seriesService;
private readonly IEventAggregator _eventAggregator;
private readonly Logger _logger;
public QualityProfileService(IQualityProfileRepository qualityProfileRepository,
IImportListFactory importListFactory,
ICustomFormatService formatService,
ISeriesService seriesService,
IEventAggregator eventAggregator,
Logger logger)
{
_qualityProfileRepository = qualityProfileRepository;
_importListFactory = importListFactory;
_formatService = formatService;
_seriesService = seriesService;
_eventAggregator = eventAggregator;
_logger = logger;
}
@@ -56,6 +59,7 @@ namespace NzbDrone.Core.Profiles.Qualities
public void Update(QualityProfile profile)
{
_qualityProfileRepository.Update(profile);
_eventAggregator.PublishEvent(new QualityProfileUpdatedEvent(profile.Id));
}
public void Delete(int id)

8
src/NzbDrone.Core/Profiles/Qualities/QualityProfileUpdatedEvent.cs Normal file
View File

@@ -0,0 +1,8 @@
using NzbDrone.Common.Messaging;
namespace NzbDrone.Core.Profiles.Qualities;
public class QualityProfileUpdatedEvent(int id) : IEvent
{
public int Id { get; private set; } = id;
}