New: Cache goodreads responses

src/NzbDrone.Common.Test/ServiceFactoryFixture.cs

@@ -18,6 +18,7 @@ namespace NzbDrone.Common.Test
         {
             var container = MainAppContainerBuilder.BuildContainer(new StartupContext());
             container.Register<IMainDatabase>(new MainDatabase(null));
+            container.Register<ICacheDatabase>(new CacheDatabase(null));
             container.Resolve<IAppFolderFactory>().Register();
 
             Mocker.SetConstant(container);

src/NzbDrone.Common/Extensions/PathExtensions.cs

@@ -14,6 +14,7 @@ namespace NzbDrone.Common.Extensions
         private const string DB = "readarr.db";
         private const string DB_RESTORE = "readarr.restore";
         private const string LOG_DB = "logs.db";
+        private const string CACHE_DB = "cache.db";
         private const string NLOG_CONFIG_FILE = "nlog.config";
         private const string UPDATE_CLIENT_EXE_NAME = "Readarr.Update";
 
@@ -322,6 +323,11 @@ namespace NzbDrone.Common.Extensions
             return Path.Combine(GetAppDataPath(appFolderInfo), LOG_DB);
         }
 
+        public static string GetCacheDatabase(this IAppFolderInfo appFolderInfo)
+        {
+            return Path.Combine(GetAppDataPath(appFolderInfo), CACHE_DB);
+        }
+
         public static string GetNlogConfigPath(this IAppFolderInfo appFolderInfo)
         {
             return Path.Combine(appFolderInfo.StartUpFolder, NLOG_CONFIG_FILE);

src/NzbDrone.Core.Test/Framework/CoreTest.cs

@@ -1,4 +1,5 @@
 using System;
+using Moq;
 using NUnit.Framework;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.Cloud;
@@ -28,6 +29,11 @@ namespace NzbDrone.Core.Test.Framework
             Mocker.SetConstant<IHttpClient>(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<CacheManager>(), Mocker.Resolve<RateLimitService>(), Mocker.Resolve<IHttpDispatcher>(), Mocker.Resolve<UserAgentBuilder>(), TestLogger));
             Mocker.SetConstant<IReadarrCloudRequestBuilder>(new ReadarrCloudRequestBuilder());
             Mocker.SetConstant<IMetadataRequestBuilder>(Mocker.Resolve<MetadataRequestBuilder>());
+
+            var httpClient = Mocker.Resolve<IHttpClient>();
+            Mocker.GetMock<ICachedHttpResponseService>()
+                .Setup(x => x.Get(It.IsAny<HttpRequest>(), It.IsAny<TimeSpan>()))
+                .Returns((HttpRequest request, TimeSpan ttl) => httpClient.Get(request));
         }
     }
 

src/NzbDrone.Core/Datastore/CacheDatabase.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Data;
+
+namespace NzbDrone.Core.Datastore
+{
+    public interface ICacheDatabase : IDatabase
+    {
+    }
+
+    public class CacheDatabase : ICacheDatabase
+    {
+        private readonly IDatabase _database;
+
+        public CacheDatabase(IDatabase database)
+        {
+            _database = database;
+        }
+
+        public IDbConnection OpenConnection()
+        {
+            return _database.OpenConnection();
+        }
+
+        public Version Version => _database.Version;
+
+        public int Migration => _database.Migration;
+
+        public void Vacuum()
+        {
+            _database.Vacuum();
+        }
+    }
+}

src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs

@@ -9,6 +9,7 @@ namespace NzbDrone.Core.Datastore
     {
         string MainDbConnectionString { get; }
         string LogDbConnectionString { get; }
+        string CacheDbConnectionString { get; }
         string GetDatabasePath(string connectionString);
     }
 
@@ -18,10 +19,12 @@ namespace NzbDrone.Core.Datastore
         {
             MainDbConnectionString = GetConnectionString(appFolderInfo.GetDatabase());
             LogDbConnectionString = GetConnectionString(appFolderInfo.GetLogDatabase());
+            CacheDbConnectionString = GetConnectionString(appFolderInfo.GetCacheDatabase());
         }
 
         public string MainDbConnectionString { get; private set; }
         public string LogDbConnectionString { get; private set; }
+        public string CacheDbConnectionString { get; private set; }
 
         public string GetDatabasePath(string connectionString)
         {

src/NzbDrone.Core/Datastore/DbFactory.cs

@@ -48,6 +48,10 @@ namespace NzbDrone.Core.Datastore
             var logDb = new LogDatabase(container.Resolve<IDbFactory>().Create(MigrationType.Log));
 
             container.Register<ILogDatabase>(logDb);
+
+            var cacheDb = new CacheDatabase(container.Resolve<IDbFactory>().Create(MigrationType.Cache));
+
+            container.Register<ICacheDatabase>(cacheDb);
         }
 
         public DbFactory(IMigrationController migrationController,
@@ -88,6 +92,14 @@ namespace NzbDrone.Core.Datastore
                         break;
                     }
 
+                case MigrationType.Cache:
+                    {
+                        connectionString = _connectionStringFactory.CacheDbConnectionString;
+                        CreateLog(connectionString, migrationContext);
+
+                        break;
+                    }
+
                 default:
                     {
                         throw new ArgumentException("Invalid MigrationType");

src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs

@@ -364,5 +364,15 @@ namespace NzbDrone.Core.Datastore.Migration
                   .WithColumn("ExceptionType").AsString().Nullable()
                   .WithColumn("Level").AsString();
         }
+
+        protected override void CacheDbUpgrade()
+        {
+            Create.TableForModel("HttpResponse")
+                .WithColumn("Url").AsString().Indexed()
+                .WithColumn("LastRefresh").AsDateTime()
+                .WithColumn("Expiry").AsDateTime().Indexed()
+                .WithColumn("Value").AsString()
+                .WithColumn("StatusCode").AsInt32();
+        }
     }
 }

src/NzbDrone.Core/Datastore/Migration/Framework/MigrationType.cs

@@ -3,6 +3,7 @@
     public enum MigrationType
     {
         Main,
-        Log
+        Log,
+        Cache
     }
 }

src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs

@@ -22,6 +22,10 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
         {
         }
 
+        protected virtual void CacheDbUpgrade()
+        {
+        }
+
         public int Version
         {
             get
@@ -48,6 +52,11 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
                     _logger.Info("Starting migration to " + Version);
                     LogDbUpgrade();
                     return;
+                case MigrationType.Cache:
+                    _logger.Info("Starting migration to " + Version);
+                    CacheDbUpgrade();
+                    return;
+
                 default:
                     LogDbUpgrade();
                     MainDbUpgrade();

src/NzbDrone.Core/Datastore/TableMapping.cs

@@ -14,6 +14,7 @@ using NzbDrone.Core.Download.Pending;
 using NzbDrone.Core.Extras.Metadata;
 using NzbDrone.Core.Extras.Metadata.Files;
 using NzbDrone.Core.Extras.Others;
+using NzbDrone.Core.Http;
 using NzbDrone.Core.ImportLists;
 using NzbDrone.Core.ImportLists.Exclusions;
 using NzbDrone.Core.Indexers;
@@ -186,6 +187,8 @@ namespace NzbDrone.Core.Datastore
 
             Mapper.Entity<CustomFilter>("CustomFilters").RegisterModel();
             Mapper.Entity<ImportListExclusion>("ImportListExclusions").RegisterModel();
+
+            Mapper.Entity<CachedHttpResponse>("HttpResponse").RegisterModel();
         }
 
         private static void RegisterMappers()

src/NzbDrone.Core/Housekeeping/Housekeepers/TrimHttpCache.cs

@@ -0,0 +1,23 @@
+using Dapper;
+using NzbDrone.Core.Datastore;
+
+namespace NzbDrone.Core.Housekeeping.Housekeepers
+{
+    public class TrimHttpCache : IHousekeepingTask
+    {
+        private readonly ICacheDatabase _database;
+
+        public TrimHttpCache(ICacheDatabase database)
+        {
+            _database = database;
+        }
+
+        public void Clean()
+        {
+            using (var mapper = _database.OpenConnection())
+            {
+                mapper.Execute(@"DELETE FROM HttpResponse WHERE Expiry < date('now')");
+            }
+        }
+    }
+}

src/NzbDrone.Core/Http/CachedHttpResponse.cs

@@ -0,0 +1,14 @@
+using System;
+using NzbDrone.Core.Datastore;
+
+namespace NzbDrone.Core.Http
+{
+    public class CachedHttpResponse : ModelBase
+    {
+        public string Url { get; set; }
+        public DateTime LastRefresh { get; set; }
+        public DateTime Expiry { get; set; }
+        public string Value { get; set; }
+        public int StatusCode { get; set; }
+    }
+}

src/NzbDrone.Core/Http/CachedHttpResponseRepository.cs

@@ -0,0 +1,27 @@
+using System.Linq;
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Messaging.Events;
+
+namespace NzbDrone.Core.Http
+{
+    public interface ICachedHttpResponseRepository : IBasicRepository<CachedHttpResponse>
+    {
+        CachedHttpResponse FindByUrl(string url);
+    }
+
+    public class CachedHttpResponseRepository : BasicRepository<CachedHttpResponse>, ICachedHttpResponseRepository
+    {
+        public CachedHttpResponseRepository(ICacheDatabase database,
+                                            IEventAggregator eventAggregator)
+            : base(database, eventAggregator)
+        {
+        }
+
+        public CachedHttpResponse FindByUrl(string url)
+        {
+            var edition = Query(x => x.Url == url).SingleOrDefault();
+
+            return edition;
+        }
+    }
+}

src/NzbDrone.Core/Http/CachedHttpResponseService.cs

@@ -0,0 +1,58 @@
+using System;
+using System.Net;
+using NzbDrone.Common.Http;
+
+namespace NzbDrone.Core.Http
+{
+    public interface ICachedHttpResponseService
+    {
+        HttpResponse Get(HttpRequest request, TimeSpan ttl);
+    }
+
+    public class CachedHttpResponseService : ICachedHttpResponseService
+    {
+        private readonly ICachedHttpResponseRepository _repo;
+        private readonly IHttpClient _httpClient;
+
+        public CachedHttpResponseService(ICachedHttpResponseRepository httpResponseRepository,
+                                         IHttpClient httpClient)
+        {
+            _repo = httpResponseRepository;
+            _httpClient = httpClient;
+        }
+
+        public HttpResponse Get(HttpRequest request, TimeSpan ttl)
+        {
+            var cached = _repo.FindByUrl(request.Url.ToString());
+
+            if (cached != null && cached.Expiry > DateTime.UtcNow)
+            {
+                return new HttpResponse(request, new HttpHeader(), cached.Value, (HttpStatusCode)cached.StatusCode);
+            }
+
+            var result = _httpClient.Get(request);
+
+            if (!result.HasHttpError)
+            {
+                if (cached == null)
+                {
+                    cached = new CachedHttpResponse
+                    {
+                        Url = request.Url.ToString(),
+                    };
+                }
+
+                var now = DateTime.UtcNow;
+
+                cached.LastRefresh = now;
+                cached.Expiry = now.Add(ttl);
+                cached.Value = result.Content;
+                cached.StatusCode = (int)result.StatusCode;
+
+                _repo.Upsert(cached);
+            }
+
+            return result;
+        }
+    }
+}

src/NzbDrone.Core/MetadataSource/Goodreads/GoodreadsProxy.cs

@@ -10,6 +10,7 @@ using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Books;
 using NzbDrone.Core.Exceptions;
+using NzbDrone.Core.Http;
 using NzbDrone.Core.MediaCover;
 using NzbDrone.Core.Parser;
 
@@ -27,6 +28,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
                                                                RegexOptions.IgnoreCase | RegexOptions.Compiled);
 
         private readonly IHttpClient _httpClient;
+        private readonly ICachedHttpResponseService _cachedHttpClient;
         private readonly Logger _logger;
         private readonly IAuthorService _authorService;
         private readonly IBookService _bookService;
@@ -36,6 +38,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
         private readonly ICached<HashSet<string>> _cache;
 
         public GoodreadsProxy(IHttpClient httpClient,
+                              ICachedHttpResponseService cachedHttpClient,
                               IAuthorService authorService,
                               IBookService bookService,
                               IEditionService editionService,
@@ -43,6 +46,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
                               ICacheManager cacheManager)
         {
             _httpClient = httpClient;
+            _cachedHttpClient = cachedHttpClient;
             _authorService = authorService;
             _bookService = bookService;
             _editionService = editionService;
@@ -80,7 +84,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
             httpRequest.AllowAutoRedirect = true;
             httpRequest.SuppressHttpError = true;
 
-            var httpResponse = _httpClient.Get(httpRequest);
+            var httpResponse = _cachedHttpClient.Get(httpRequest, TimeSpan.FromDays(30));
 
             if (httpResponse.HasHttpError)
             {
@@ -208,7 +212,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
             httpRequest.AllowAutoRedirect = true;
             httpRequest.SuppressHttpError = true;
 
-            var httpResponse = _httpClient.Get(httpRequest);
+            var httpResponse = _cachedHttpClient.Get(httpRequest, TimeSpan.FromDays(7));
 
             if (httpResponse.HasHttpError)
             {
@@ -240,7 +244,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
             httpRequest.AllowAutoRedirect = true;
             httpRequest.SuppressHttpError = true;
 
-            var httpResponse = _httpClient.Get(httpRequest);
+            var httpResponse = _cachedHttpClient.Get(httpRequest, TimeSpan.FromDays(90));
 
             if (httpResponse.HasHttpError)
             {
@@ -314,7 +318,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
             httpRequest.AllowAutoRedirect = true;
             httpRequest.SuppressHttpError = true;
 
-            var httpResponse = _httpClient.Get(httpRequest);
+            var httpResponse = _cachedHttpClient.Get(httpRequest, TimeSpan.FromDays(90));
 
             if (httpResponse.HasHttpError)
             {

src/NzbDrone.Host.Test/ContainerFixture.cs

@@ -31,6 +31,7 @@ namespace NzbDrone.App.Test
             _container = MainAppContainerBuilder.BuildContainer(args);
 
             _container.Register<IMainDatabase>(new MainDatabase(null));
+            _container.Register<ICacheDatabase>(new CacheDatabase(null));
 
             // set up a dummy broadcaster to allow tests to resolve
             var mockBroadcaster = new Mock<IBroadcastSignalRMessage>();