Prevent ProgressMessageTarget from ever reading the command from the database.

Fixed: Further optimized the MediaInfo discovery to reduce the read data to a couple of dozen kbytes, but less detailed.

src/NzbDrone.Api/Commands/CommandModule.cs

@@ -62,7 +62,7 @@ namespace NzbDrone.Api.Commands
         {
             if (message.Command.Body.SendUpdatesToClient)
             {
-                BroadcastResourceChange(ModelAction.Updated, message.Command.Id);
+                BroadcastResourceChange(ModelAction.Updated, message.Command.InjectTo<CommandResource>());
             }
         }
     }

src/NzbDrone.Common.Test/Http/HttpClientFixture.cs

@@ -9,6 +9,7 @@ using NzbDrone.Common.Http;
 using NzbDrone.Test.Common;
 using NzbDrone.Test.Common.Categories;
 using NLog;
+using NzbDrone.Common.TPL;
 
 namespace NzbDrone.Common.Test.Http
 {
@@ -20,6 +21,7 @@ namespace NzbDrone.Common.Test.Http
         public void SetUp()
         {
             Mocker.SetConstant<ICacheManager>(Mocker.Resolve<CacheManager>());
+            Mocker.SetConstant<IRateLimitService>(Mocker.Resolve<RateLimitService>());
         }
 
         [Test]
@@ -140,7 +142,7 @@ namespace NzbDrone.Common.Test.Http
             var oldRequest = new HttpRequest("http://eu.httpbin.org/get");
             oldRequest.AddCookie("my", "cookie");
 
-            var oldClient = new HttpClient(Mocker.Resolve<ICacheManager>(), Mocker.Resolve<Logger>());
+            var oldClient = new HttpClient(Mocker.Resolve<ICacheManager>(), Mocker.Resolve<IRateLimitService>(), Mocker.Resolve<Logger>());
 
             oldClient.Should().NotBeSameAs(Subject);
 

src/NzbDrone.Common.Test/NzbDrone.Common.Test.csproj

@@ -90,6 +90,7 @@
     <Compile Include="ServiceFactoryFixture.cs" />
     <Compile Include="ServiceProviderTests.cs" />
     <Compile Include="TPLTests\DebouncerFixture.cs" />
+    <Compile Include="TPLTests\RateLimitServiceFixture.cs" />
     <Compile Include="WebClientTests.cs" />
   </ItemGroup>
   <ItemGroup>

src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs

@@ -0,0 +1,94 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using NUnit.Framework;
+using NzbDrone.Common.Cache;
+using NzbDrone.Common.TPL;
+using NzbDrone.Test.Common;
+using FluentAssertions;
+
+namespace NzbDrone.Common.Test.TPLTests
+{
+    [TestFixture]
+    public class RateLimitServiceFixture : TestBase<RateLimitService>
+    {
+        private DateTime _epoch;
+
+        [SetUp]
+        public void SetUp()
+        {
+            // Make sure it's there so we don't affect measurements.
+            Subject.GetType();
+
+            _epoch = DateTime.UtcNow;
+        }
+
+        private ConcurrentDictionary<string, DateTime> GetRateLimitStore()
+        {
+            var cache = Mocker.Resolve<ICacheManager>()
+                .GetCache<ConcurrentDictionary<string, DateTime>>(typeof(RateLimitService), "rateLimitStore");
+
+            return cache.Get("rateLimitStore", () => new ConcurrentDictionary<string, DateTime>());
+        }
+
+        private void GivenExisting(string key, DateTime dateTime)
+        {
+            GetRateLimitStore().AddOrUpdate(key, dateTime, (s, i) => dateTime);
+        }
+
+        [Test]
+        public void should_not_delay_if_unset()
+        {
+            var watch = Stopwatch.StartNew();
+            Subject.WaitAndPulse("me", TimeSpan.FromMilliseconds(100));
+            watch.Stop();
+
+            watch.ElapsedMilliseconds.Should().BeLessThan(100);
+        }
+
+        [Test]
+        public void should_not_delay_unrelated_key()
+        {
+            GivenExisting("other", _epoch + TimeSpan.FromMilliseconds(200));
+
+            var watch = Stopwatch.StartNew();
+            Subject.WaitAndPulse("me", TimeSpan.FromMilliseconds(100));
+            watch.Stop();
+
+            watch.ElapsedMilliseconds.Should().BeLessThan(50);
+        }
+
+        [Test]
+        public void should_wait_for_existing()
+        {
+            GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
+
+            var watch = Stopwatch.StartNew();
+            Subject.WaitAndPulse("me", TimeSpan.FromMilliseconds(400));
+            watch.Stop();
+
+            watch.ElapsedMilliseconds.Should().BeInRange(195, 250);
+        }
+
+        [Test]
+        public void should_extend_delay()
+        {
+            GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
+
+            Subject.WaitAndPulse("me", TimeSpan.FromMilliseconds(100));
+
+            (GetRateLimitStore()["me"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(300));
+        }
+
+        [Test]
+        public void should_add_delay()
+        {
+            Subject.WaitAndPulse("me", TimeSpan.FromMilliseconds(100));
+
+            (GetRateLimitStore()["me"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(100));
+        }
+    }
+}

src/NzbDrone.Common/Http/HttpClient.cs

@@ -6,6 +6,7 @@ using NLog;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Common.Extensions;
+using NzbDrone.Common.TPL;
 
 namespace NzbDrone.Common.Http
 {
@@ -23,12 +24,13 @@ namespace NzbDrone.Common.Http
     public class HttpClient : IHttpClient
     {
         private readonly Logger _logger;
-
+        private readonly IRateLimitService _rateLimitService;
         private readonly ICached<CookieContainer> _cookieContainerCache;
 
-        public HttpClient(ICacheManager cacheManager, Logger logger)
+        public HttpClient(ICacheManager cacheManager, IRateLimitService rateLimitService, Logger logger)
         {
             _logger = logger;
+            _rateLimitService = rateLimitService;
             ServicePointManager.DefaultConnectionLimit = 12;
 
             _cookieContainerCache = cacheManager.GetCache<CookieContainer>(typeof(HttpClient));
@@ -36,6 +38,11 @@ namespace NzbDrone.Common.Http
 
         public HttpResponse Execute(HttpRequest request)
         {
+            if (request.RateLimit != TimeSpan.Zero)
+            {
+                _rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimit);
+            }
+
             _logger.Trace(request);
 
             var webRequest = (HttpWebRequest)WebRequest.Create(request.Url);

src/NzbDrone.Common/Http/HttpRequest.cs

@@ -47,6 +47,7 @@ namespace NzbDrone.Common.Http
         public bool AllowAutoRedirect { get; set; }
         public Dictionary<string, string> Cookies { get; private set; }
         public bool StoreResponseCookie { get; set; }
+        public TimeSpan RateLimit { get; set; }
 
         public override string ToString()
         {

src/NzbDrone.Common/NzbDrone.Common.csproj

@@ -178,7 +178,6 @@
     <Compile Include="Processes\ProcessProvider.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Properties\SharedAssemblyInfo.cs" />
-    <Compile Include="RateGate.cs" />
     <Compile Include="Reflection\ReflectionExtensions.cs" />
     <Compile Include="Extensions\ResourceExtensions.cs" />
     <Compile Include="Security\X509CertificateValidationPolicy.cs" />
@@ -190,6 +189,7 @@
     <Compile Include="TinyIoC.cs" />
     <Compile Include="TPL\Debouncer.cs" />
     <Compile Include="TPL\LimitedConcurrencyLevelTaskScheduler.cs" />
+    <Compile Include="TPL\RateLimitService.cs" />
     <Compile Include="TPL\TaskExtensions.cs" />
     <Compile Include="Extensions\TryParseExtensions.cs" />
   </ItemGroup>

src/NzbDrone.Common/RateGate.cs

@@ -1,197 +0,0 @@
-/*
- * Code from: http://www.jackleitch.net/2010/10/better-rate-limiting-with-dot-net/
- */
-
-using System;
-using System.Collections.Concurrent;
-using System.Threading;
-
-namespace NzbDrone.Common
-{
-    /// <summary>
-    /// Used to control the rate of some occurrence per unit of time.
-    /// </summary>
-    /// <remarks>
-    ///     <para>
-    ///     To control the rate of an action using a <see cref="RateGate"/>, 
-    ///     code should simply call <see cref="WaitToProceed()"/> prior to 
-    ///     performing the action. <see cref="WaitToProceed()"/> will block
-    ///     the current thread until the action is allowed based on the rate 
-    ///     limit.
-    ///     </para>
-    ///     <para>
-    ///     This class is thread safe. A single <see cref="RateGate"/> instance 
-    ///     may be used to control the rate of an occurrence across multiple 
-    ///     threads.
-    ///     </para>
-    /// </remarks>
-    public class RateGate : IDisposable
-    {
-        // Semaphore used to count and limit the number of occurrences per
-        // unit time.
-        private readonly SemaphoreSlim _semaphore;
-
-        // Times (in millisecond ticks) at which the semaphore should be exited.
-        private readonly ConcurrentQueue<int> _exitTimes;
-
-        // Timer used to trigger exiting the semaphore.
-        private readonly Timer _exitTimer;
-
-        // Whether this instance is disposed.
-        private bool _isDisposed;
-
-        /// <summary>
-        /// Number of occurrences allowed per unit of time.
-        /// </summary>
-        public int Occurrences { get; private set; }
-
-        /// <summary>
-        /// The length of the time unit, in milliseconds.
-        /// </summary>
-        public int TimeUnitMilliseconds { get; private set; }
-
-        /// <summary>
-        /// Initializes a <see cref="RateGate"/> with a rate of <paramref name="occurrences"/> 
-        /// per <paramref name="timeUnit"/>.
-        /// </summary>
-        /// <param name="occurrences">Number of occurrences allowed per unit of time.</param>
-        /// <param name="timeUnit">Length of the time unit.</param>
-        /// <exception cref="ArgumentOutOfRangeException">
-        /// If <paramref name="occurrences"/> or <paramref name="timeUnit"/> is negative.
-        /// </exception>
-        public RateGate(int occurrences, TimeSpan timeUnit)
-        {
-            // Check the arguments.
-            if (occurrences <= 0)
-                throw new ArgumentOutOfRangeException("occurrences", "Number of occurrences must be a positive integer");
-            if (timeUnit != timeUnit.Duration())
-                throw new ArgumentOutOfRangeException("timeUnit", "Time unit must be a positive span of time");
-            if (timeUnit >= TimeSpan.FromMilliseconds(UInt32.MaxValue))
-                throw new ArgumentOutOfRangeException("timeUnit", "Time unit must be less than 2^32 milliseconds");
-
-            Occurrences = occurrences;
-            TimeUnitMilliseconds = (int)timeUnit.TotalMilliseconds;
-
-            // Create the semaphore, with the number of occurrences as the maximum count.
-            _semaphore = new SemaphoreSlim(Occurrences, Occurrences);
-
-            // Create a queue to hold the semaphore exit times.
-            _exitTimes = new ConcurrentQueue<int>();
-
-            // Create a timer to exit the semaphore. Use the time unit as the original
-            // interval length because that's the earliest we will need to exit the semaphore.
-            _exitTimer = new Timer(ExitTimerCallback, null, TimeUnitMilliseconds, -1);
-        }
-
-        // Callback for the exit timer that exits the semaphore based on exit times 
-        // in the queue and then sets the timer for the nextexit time.
-        private void ExitTimerCallback(object state)
-        {
-            // While there are exit times that are passed due still in the queue,
-            // exit the semaphore and dequeue the exit time.
-            int exitTime;
-            while (_exitTimes.TryPeek(out exitTime)
-                    && unchecked(exitTime - Environment.TickCount) <= 0)
-            {
-                _semaphore.Release();
-                _exitTimes.TryDequeue(out exitTime);
-            }
-
-            // Try to get the next exit time from the queue and compute
-            // the time until the next check should take place. If the 
-            // queue is empty, then no exit times will occur until at least
-            // one time unit has passed.
-            int timeUntilNextCheck;
-            if (_exitTimes.TryPeek(out exitTime))
-                timeUntilNextCheck = unchecked(exitTime - Environment.TickCount);
-            else
-                timeUntilNextCheck = TimeUnitMilliseconds;
-
-            // Set the timer.
-            _exitTimer.Change(timeUntilNextCheck, -1);
-        }
-
-        /// <summary>
-        /// Blocks the current thread until allowed to proceed or until the
-        /// specified timeout elapses.
-        /// </summary>
-        /// <param name="millisecondsTimeout">Number of milliseconds to wait, or -1 to wait indefinitely.</param>
-        /// <returns>true if the thread is allowed to proceed, or false if timed out</returns>
-        public bool WaitToProceed(int millisecondsTimeout)
-        {
-            // Check the arguments.
-            if (millisecondsTimeout < -1)
-                throw new ArgumentOutOfRangeException("millisecondsTimeout");
-
-            CheckDisposed();
-
-            // Block until we can enter the semaphore or until the timeout expires.
-            var entered = _semaphore.Wait(millisecondsTimeout);
-
-            // If we entered the semaphore, compute the corresponding exit time 
-            // and add it to the queue.
-            if (entered)
-            {
-                var timeToExit = unchecked(Environment.TickCount + TimeUnitMilliseconds);
-                _exitTimes.Enqueue(timeToExit);
-            }
-
-            return entered;
-        }
-
-        /// <summary>
-        /// Blocks the current thread until allowed to proceed or until the
-        /// specified timeout elapses.
-        /// </summary>
-        /// <param name="timeout"></param>
-        /// <returns>true if the thread is allowed to proceed, or false if timed out</returns>
-        public bool WaitToProceed(TimeSpan timeout)
-        {
-            return WaitToProceed((int)timeout.TotalMilliseconds);
-        }
-
-        /// <summary>
-        /// Blocks the current thread indefinitely until allowed to proceed.
-        /// </summary>
-        public void WaitToProceed()
-        {
-            WaitToProceed(Timeout.Infinite);
-        }
-
-        // Throws an ObjectDisposedException if this object is disposed.
-        private void CheckDisposed()
-        {
-            if (_isDisposed)
-                throw new ObjectDisposedException("RateGate is already disposed");
-        }
-
-        /// <summary>
-        /// Releases unmanaged resources held by an instance of this class.
-        /// </summary>
-        public void Dispose()
-        {
-            Dispose(true);
-            GC.SuppressFinalize(this);
-        }
-
-        /// <summary>
-        /// Releases unmanaged resources held by an instance of this class.
-        /// </summary>
-        /// <param name="isDisposing">Whether this object is being disposed.</param>
-        protected virtual void Dispose(bool isDisposing)
-        {
-            if (!_isDisposed)
-            {
-                if (isDisposing)
-                {
-                    // The semaphore and timer both implement IDisposable and 
-                    // therefore must be disposed.
-                    _semaphore.Dispose();
-                    _exitTimer.Dispose();
-
-                    _isDisposed = true;
-                }
-            }
-        }
-    }
-}

src/NzbDrone.Common/TPL/RateLimitService.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using NLog;
+using NzbDrone.Common.Cache;
+
+namespace NzbDrone.Common.TPL
+{
+    public interface IRateLimitService
+    {
+        void WaitAndPulse(string key, TimeSpan interval);
+    }
+
+    public class RateLimitService : IRateLimitService
+    {
+        private readonly ConcurrentDictionary<string, DateTime> _rateLimitStore;
+        private readonly Logger _logger;
+
+        public RateLimitService(ICacheManager cacheManager, Logger logger)
+        {
+            _rateLimitStore = cacheManager.GetCache<ConcurrentDictionary<string, DateTime>>(GetType(), "rateLimitStore").Get("rateLimitStore", () => new ConcurrentDictionary<string, DateTime>());
+            _logger = logger;
+        }
+
+        public void WaitAndPulse(string key, TimeSpan interval)
+        {
+            var waitUntil = _rateLimitStore.AddOrUpdate(key,
+                (s) => DateTime.UtcNow + interval,
+                (s,i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval);
+
+            waitUntil -= interval;
+
+            var delay = waitUntil - DateTime.UtcNow;
+
+            if (delay.TotalSeconds > 0.0)
+            {
+                _logger.Trace("Rate Limit triggered, delaying '{0}' for {1:0.000} sec", key, delay.TotalSeconds);
+                System.Threading.Thread.Sleep(delay);
+            }
+        }
+    }
+}

src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs

@@ -38,6 +38,7 @@ namespace NzbDrone.Core.Test.Download
 
             var releaseInfo = Builder<ReleaseInfo>.CreateNew()
                 .With(v => v.DownloadProtocol = Indexers.DownloadProtocol.Usenet)
+                .With(v => v.DownloadUrl = "http://test.site/download1.ext")
                 .Build();
 
             _parseResult = Builder<RemoteEpisode>.CreateNew()

src/NzbDrone.Core.Test/Framework/CoreTest.cs

@@ -3,6 +3,7 @@ using NUnit.Framework;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.Cloud;
 using NzbDrone.Common.Http;
+using NzbDrone.Common.TPL;
 using NzbDrone.Test.Common;
 
 namespace NzbDrone.Core.Test.Framework
@@ -17,7 +18,7 @@ namespace NzbDrone.Core.Test.Framework
         protected void UseRealHttp()
         {
             Mocker.SetConstant<IHttpProvider>(new HttpProvider(TestLogger));
-            Mocker.SetConstant<IHttpClient>(new HttpClient(Mocker.Resolve<CacheManager>(), TestLogger));
+            Mocker.SetConstant<IHttpClient>(new HttpClient(Mocker.Resolve<CacheManager>(), Mocker.Resolve<RateLimitService>(), TestLogger));
             Mocker.SetConstant<IDroneServicesRequestBuilder>(new DroneServicesHttpRequestBuilder());
         }
     }

src/NzbDrone.Core.Test/Messaging/Commands/CommandQueueManagerFixture.cs

@@ -0,0 +1,55 @@
+using System.Collections.Generic;
+using System.Linq;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.Download;
+using NzbDrone.Core.Messaging.Commands;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.Messaging.Commands
+{
+    [TestFixture]
+    public class CommandQueueManagerFixture : CoreTest<CommandQueueManager>
+    {
+        [SetUp]
+        public void Setup()
+        {
+            var id = 0;
+            var commands = new List<CommandModel>();
+
+            Mocker.GetMock<ICommandRepository>()
+                  .Setup(s => s.Insert(It.IsAny<CommandModel>()))
+                  .Returns<CommandModel>(c =>
+                  {
+                      c.Id = id + 1;
+                      commands.Add(c);
+                      id++;
+
+                      return c;
+                  });
+
+            Mocker.GetMock<ICommandRepository>()
+                  .Setup(s => s.Get(It.IsAny<int>()))
+                  .Returns<int>(c =>
+                  {
+                      return commands.SingleOrDefault(e => e.Id == c);
+                  });
+        }
+
+        [Test]
+        public void should_not_remove_commands_for_five_minutes_after_they_end()
+        {
+            var command = Subject.Push<CheckForFinishedDownloadCommand>(new CheckForFinishedDownloadCommand());
+
+            Subject.Start(command);
+            Subject.Complete(command, "All done");
+            Subject.CleanCommands();
+
+            Subject.Get(command.Id).Should().NotBeNull();
+
+            Mocker.GetMock<ICommandRepository>()
+                  .Verify(v => v.Get(It.IsAny<int>()), Times.Never());
+        }
+    }
+}

src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj

@@ -239,6 +239,7 @@
     <Compile Include="MediaFiles\EpisodeImport\Specifications\UpgradeSpecificationFixture.cs" />
     <Compile Include="MediaFiles\ImportApprovedEpisodesFixture.cs" />
     <Compile Include="MediaFiles\MediaFileRepositoryFixture.cs" />
+    <Compile Include="Messaging\Commands\CommandQueueManagerFixture.cs" />
     <Compile Include="MetadataSource\SkyHook\SkyHookProxySearchFixture.cs" />
     <Compile Include="MetadataSource\SearchSeriesComparerFixture.cs" />
     <Compile Include="MetadataSource\SkyHook\SkyHookProxyFixture.cs" />

src/NzbDrone.Core/Download/DownloadService.cs

@@ -1,7 +1,9 @@
 using System;
 using NLog;
 using NzbDrone.Common.EnsureThat;
+using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Instrumentation.Extensions;
+using NzbDrone.Common.TPL;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser.Model;
 
@@ -16,13 +18,17 @@ namespace NzbDrone.Core.Download
     public class DownloadService : IDownloadService
     {
         private readonly IProvideDownloadClient _downloadClientProvider;
+        private readonly IRateLimitService _rateLimitService;
         private readonly IEventAggregator _eventAggregator;
         private readonly Logger _logger;
 
         public DownloadService(IProvideDownloadClient downloadClientProvider,
-            IEventAggregator eventAggregator, Logger logger)
+            IRateLimitService rateLimitService,
+            IEventAggregator eventAggregator,
+            Logger logger)
         {
             _downloadClientProvider = downloadClientProvider;
+            _rateLimitService = rateLimitService;
             _eventAggregator = eventAggregator;
             _logger = logger;
         }
@@ -41,6 +47,13 @@ namespace NzbDrone.Core.Download
                 return;
             }
 
+            // Limit grabs to 2 per second.
+            if (remoteEpisode.Release.DownloadUrl.IsNotNullOrWhiteSpace() && !remoteEpisode.Release.DownloadUrl.StartsWith("magnet:"))
+            {
+                var uri = new Uri(remoteEpisode.Release.DownloadUrl);
+                _rateLimitService.WaitAndPulse(uri.Host, TimeSpan.FromSeconds(2));
+            }
+
             var downloadClientId = downloadClient.Download(remoteEpisode);
             var episodeGrabbedEvent = new EpisodeGrabbedEvent(remoteEpisode);
             episodeGrabbedEvent.DownloadClient = downloadClient.GetType().Name;

src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs

@@ -39,55 +39,50 @@ namespace NzbDrone.Core.Download
             var grabbed = new List<DownloadDecision>();
             var pending = new List<DownloadDecision>();
 
-            //Limits to 1 grab every 1 second to reduce rapid API hits
-            using (var rateGate = new RateGate(1, TimeSpan.FromSeconds(1)))
+            foreach (var report in prioritizedDecisions)
             {
-                foreach (var report in prioritizedDecisions)
+                var remoteEpisode = report.RemoteEpisode;
+
+                var episodeIds = remoteEpisode.Episodes.Select(e => e.Id).ToList();
+
+                //Skip if already grabbed
+                if (grabbed.SelectMany(r => r.RemoteEpisode.Episodes)
+                                .Select(e => e.Id)
+                                .ToList()
+                                .Intersect(episodeIds)
+                                .Any())
                 {
-                    var remoteEpisode = report.RemoteEpisode;
-
-                    var episodeIds = remoteEpisode.Episodes.Select(e => e.Id).ToList();
-
-                    //Skip if already grabbed
-                    if (grabbed.SelectMany(r => r.RemoteEpisode.Episodes)
-                                   .Select(e => e.Id)
-                                   .ToList()
-                                   .Intersect(episodeIds)
-                                   .Any())
-                    {
-                        continue;
-                    }
-
-                    if (report.TemporarilyRejected)
-                    {
-                        _pendingReleaseService.Add(report);
-                        pending.Add(report);
-                        continue;
-                    }
-
-                    if (pending.SelectMany(r => r.RemoteEpisode.Episodes)
-                           .Select(e => e.Id)
-                           .ToList()
-                           .Intersect(episodeIds)
-                           .Any())
-                    {
-                        continue;
-                    }
-
-                    try
-                    {
-                        rateGate.WaitToProceed();
-                        _downloadService.DownloadReport(remoteEpisode);
-                        grabbed.Add(report);
-                    }
-                    catch (Exception e)
-                    {
-                        //TODO: support for store & forward
-                        //We'll need to differentiate between a download client error and an indexer error
-                        _logger.WarnException("Couldn't add report to download queue. " + remoteEpisode, e);
-                    }
+                    continue;
                 }
-            }           
+
+                if (report.TemporarilyRejected)
+                {
+                    _pendingReleaseService.Add(report);
+                    pending.Add(report);
+                    continue;
+                }
+
+                if (pending.SelectMany(r => r.RemoteEpisode.Episodes)
+                        .Select(e => e.Id)
+                        .ToList()
+                        .Intersect(episodeIds)
+                        .Any())
+                {
+                    continue;
+                }
+
+                try
+                {
+                    _downloadService.DownloadReport(remoteEpisode);
+                    grabbed.Add(report);
+                }
+                catch (Exception e)
+                {
+                    //TODO: support for store & forward
+                    //We'll need to differentiate between a download client error and an indexer error
+                    _logger.WarnException("Couldn't add report to download queue. " + remoteEpisode, e);
+                }
+            }
 
             return new ProcessedDecisions(grabbed, pending, decisions.Where(d => d.Rejected).ToList());
         }

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -6,6 +6,7 @@ using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
+using NzbDrone.Common.TPL;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.IndexerSearch.Definitions;
@@ -27,6 +28,7 @@ namespace NzbDrone.Core.Indexers
         public bool SupportsPaging { get { return PageSize > 0; } }
 
         public virtual Int32 PageSize { get { return 0; } }
+        public virtual TimeSpan RateLimit { get { return TimeSpan.FromSeconds(2); } }
 
         public abstract IIndexerRequestGenerator GetRequestGenerator();
         public abstract IParseIndexerResponse GetParser();
@@ -192,6 +194,11 @@ namespace NzbDrone.Core.Indexers
         {
             _logger.Debug("Downloading Feed " + request.Url);
 
+            if (request.HttpRequest.RateLimit < RateLimit)
+            {
+                request.HttpRequest.RateLimit = RateLimit;
+            }
+
             var response = new IndexerResponse(request, _httpClient.Execute(request.HttpRequest));
 
             return parser.ParseResponse(response).ToList();

src/NzbDrone.Core/MediaFiles/MediaInfo/MediaInfoLib.cs

@@ -2,10 +2,20 @@
 using System.IO;
 using System.Runtime.InteropServices;
 using System.Text;
-using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Common.Instrumentation;
+using NLog;
 
 namespace NzbDrone.Core.MediaFiles.MediaInfo
 {
+    [Flags]
+    public enum BufferStatus
+    {
+        Accepted = 1,
+        Filled = 2,
+        Updated = 4,
+        Finalized = 8
+    }
+
     public enum StreamKind
     {
         General,
@@ -14,7 +24,7 @@ namespace NzbDrone.Core.MediaFiles.MediaInfo
         Text,
         Other,
         Image,
-        Menu,
+        Menu
     }
 
     public enum InfoKind
@@ -48,6 +58,7 @@ namespace NzbDrone.Core.MediaFiles.MediaInfo
 
     public class MediaInfo : IDisposable
     {
+        private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(MediaInfo));
         private IntPtr _handle;
 
         public bool MustUseAnsi { get; set; }
@@ -172,31 +183,40 @@ namespace NzbDrone.Core.MediaFiles.MediaInfo
 
         public int Open(Stream stream)
         {
-            var buffer = new byte[64 * 1024];
-
             var isValid = (int)MediaInfo_Open_Buffer_Init(_handle, stream.Length, 0);
             if (isValid == 1)
             {
+                var buffer = new byte[16 * 1024];
+                long seekStart = 0;
+                long totalRead = 0;
                 int bufferRead;
 
                 do
                 {
                     bufferRead = stream.Read(buffer, 0, buffer.Length);
+                    totalRead += bufferRead;
 
-                    if (MediaInfo_Open_Buffer_Continue(_handle, buffer, (IntPtr)bufferRead) == (IntPtr)0)
+                    var status = (BufferStatus)MediaInfo_Open_Buffer_Continue(_handle, buffer, (IntPtr)bufferRead);
+                    
+                    if (status.HasFlag(BufferStatus.Finalized) || status <= 0 || bufferRead == 0)
                     {
+                        Logger.Trace("Read file offset {0}-{1} ({2} bytes)", seekStart, stream.Position, stream.Position - seekStart);
                         break;
                     }
 
                     var seekPos = MediaInfo_Open_Buffer_Continue_GoTo_Get(_handle);
                     if (seekPos != -1)
                     {
+                        Logger.Trace("Read file offset {0}-{1} ({2} bytes)", seekStart, stream.Position, stream.Position - seekStart);
                         seekPos = stream.Seek(seekPos, SeekOrigin.Begin);
+                        seekStart = seekPos;
                         MediaInfo_Open_Buffer_Init(_handle, stream.Length, seekPos);
                     }
                 } while (bufferRead > 0);
 
                 MediaInfo_Open_Buffer_Finalize(_handle);
+
+                Logger.Trace("Read a total of {0} bytes ({1:0.0}%)", totalRead, totalRead * 100.0 / stream.Length);
             }
 
             return isValid;

src/NzbDrone.Core/MediaFiles/MediaInfo/VideoFileInfoReader.cs

@@ -39,7 +39,7 @@ namespace NzbDrone.Core.MediaFiles.MediaInfo
                 mediaInfo = new MediaInfo();
                 _logger.Debug("Getting media info from {0}", filename);
 
-                mediaInfo.Option("ParseSpeed", "0.2");
+                mediaInfo.Option("ParseSpeed", "0.0");
 
                 int open;
 
@@ -48,6 +48,28 @@ namespace NzbDrone.Core.MediaFiles.MediaInfo
                     open = mediaInfo.Open(stream);
                 }
 
+                if (open != 0)
+                {
+                    int audioRuntime;
+                    int videoRuntime;
+                    int generalRuntime;
+
+                    //Runtime
+                    Int32.TryParse(mediaInfo.Get(StreamKind.Video, 0, "PlayTime"), out videoRuntime);
+                    Int32.TryParse(mediaInfo.Get(StreamKind.Audio, 0, "PlayTime"), out audioRuntime);
+                    Int32.TryParse(mediaInfo.Get(StreamKind.General, 0, "PlayTime"), out generalRuntime);
+
+                    if (audioRuntime == 0 && videoRuntime == 0 && generalRuntime == 0)
+                    {
+                        mediaInfo.Option("ParseSpeed", "1.0");
+
+                        using (var stream = _diskProvider.OpenReadStream(filename))
+                        {
+                            open = mediaInfo.Open(stream);
+                        }
+                    }
+                }
+
                 if (open != 0)
                 {
                     int width;

src/NzbDrone.Core/Messaging/Commands/CommandExecutor.cs

@@ -69,9 +69,9 @@ namespace NzbDrone.Core.Messaging.Commands
                 _commandQueueManager.Start(commandModel);
                 BroadcastCommandUpdate(commandModel);
 
-                if (!MappedDiagnosticsContext.Contains("CommandId"))
+                if (ProgressMessageContext.CommandModel == null)
                 {
-                    MappedDiagnosticsContext.Set("CommandId", commandModel.Id.ToString());
+                    ProgressMessageContext.CommandModel = commandModel;
                 }
 
                 handler.Execute(command);
@@ -95,9 +95,9 @@ namespace NzbDrone.Core.Messaging.Commands
 
                 _eventAggregator.PublishEvent(new CommandExecutedEvent(commandModel));
 
-                if (MappedDiagnosticsContext.Get("CommandId") == commandModel.Id.ToString())
+                if (ProgressMessageContext.CommandModel == commandModel)
                 {
-                    MappedDiagnosticsContext.Remove("CommandId");
+                    ProgressMessageContext.CommandModel = null;
                 }
             }
 

src/NzbDrone.Core/Messaging/Commands/CommandQueueManager.cs

@@ -78,10 +78,10 @@ namespace NzbDrone.Core.Messaging.Commands
             };
 
             _logger.Trace("Inserting new command: {0}", commandModel.Name);
-            
+
             _repo.Insert(commandModel);
-            _commandQueue.Add(commandModel);
             _commandCache.Set(commandModel.Id.ToString(), commandModel);
+            _commandQueue.Add(commandModel);
 
             return commandModel;
         }
@@ -152,7 +152,7 @@ namespace NzbDrone.Core.Messaging.Commands
             _logger.Trace("Cleaning up old commands");
             _repo.Trim();
 
-            var old = _commandCache.Values.Where(c => c.EndedAt < DateTime.UtcNow.AddMinutes(5));
+            var old = _commandCache.Values.Where(c => c.EndedAt < DateTime.UtcNow.AddMinutes(-5));
 
             foreach (var command in old)
             {

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -682,6 +682,7 @@
     <Compile Include="Profiles\Delay\DelayProfileService.cs" />
     <Compile Include="Profiles\Delay\DelayProfileTagInUseValidator.cs" />
     <Compile Include="Profiles\ProfileRepository.cs" />
+    <Compile Include="ProgressMessaging\ProgressMessageContext.cs" />
     <Compile Include="Qualities\Revision.cs" />
     <Compile Include="RemotePathMappings\RemotePathMapping.cs" />
     <Compile Include="RemotePathMappings\RemotePathMappingRepository.cs" />

src/NzbDrone.Core/ProgressMessaging/ProgressMessageContext.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using NzbDrone.Core.Messaging.Commands;
+
+namespace NzbDrone.Core.ProgressMessaging
+{
+    public static class ProgressMessageContext
+    {
+        [ThreadStatic]
+        private static CommandModel _commandModel;
+
+        [ThreadStatic]
+        private static bool _reentrancyLock;
+        
+        public static CommandModel CommandModel
+        {
+            get { return _commandModel; }
+            set { _commandModel = value; }
+        }
+
+        public static bool LockReentrancy()
+        {
+            if (_reentrancyLock)
+                return false;
+
+            _reentrancyLock = true;
+            return true;
+        }
+
+        public static void UnlockReentrancy()
+        {
+            _reentrancyLock = false;
+        }
+    }
+}

src/NzbDrone.Core/ProgressMessaging/ProgressMessageTarget.cs

@@ -15,8 +15,6 @@ namespace NzbDrone.Core.ProgressMessaging
         private readonly IManageCommandQueue _commandQueueManager;
         private static LoggingRule _rule;
 
-        private const string REENTRY_LOCK = "ProgressMessagingLock";
-
         public ProgressMessageTarget(IEventAggregator eventAggregator, IManageCommandQueue commandQueueManager)
         {
             _eventAggregator = eventAggregator;
@@ -25,29 +23,20 @@ namespace NzbDrone.Core.ProgressMessaging
 
         protected override void Write(LogEventInfo logEvent)
         {
-            if (!ReentryPreventionCheck()) return;
+            var command = ProgressMessageContext.CommandModel;
 
-            var command = GetCurrentCommand();
-
-            if (IsClientMessage(logEvent, command))
+            if (!IsClientMessage(logEvent, command)) return;
+            
+            if (!ProgressMessageContext.LockReentrancy()) return;
+            try
             {
                 _commandQueueManager.SetMessage(command, logEvent.FormattedMessage);
                 _eventAggregator.PublishEvent(new CommandUpdatedEvent(command));
             }
-
-            MappedDiagnosticsContext.Remove(REENTRY_LOCK);
-        }
-
-        private CommandModel GetCurrentCommand()
-        {
-            var commandId = MappedDiagnosticsContext.Get("CommandId");
-
-            if (String.IsNullOrWhiteSpace(commandId))
+            finally
             {
-                return null;
+                ProgressMessageContext.UnlockReentrancy();
             }
-
-            return _commandQueueManager.Get(Convert.ToInt32(commandId));
         }
 
         private bool IsClientMessage(LogEventInfo logEvent, CommandModel command)
@@ -60,20 +49,6 @@ namespace NzbDrone.Core.ProgressMessaging
             return logEvent.Properties.ContainsKey("Status");
         }
 
-        private bool ReentryPreventionCheck()
-        {
-            var reentryLock = MappedDiagnosticsContext.Get(REENTRY_LOCK);
-            var commandId = MappedDiagnosticsContext.Get("CommandId");
-
-            if (reentryLock.IsNullOrWhiteSpace() || reentryLock != commandId)
-            {
-                MappedDiagnosticsContext.Set(REENTRY_LOCK, MappedDiagnosticsContext.Get("CommandId"));
-                return true;
-            }
-
-            return false;
-        }
-
         public void Handle(ApplicationStartedEvent message)
         {
             _rule = new LoggingRule("*", LogLevel.Trace, this);