New: Postgres Support

Co-Authored-By: Qstick <376117+Qstick@users.noreply.github.com>
Co-authored-by: ta264 <ta264@users.noreply.github.com>

(cherry picked from commit 80b1aa9a2c81617bdda7ef551c19a2f114e49204)

src/NzbDrone.Core.Test/Datastore/DatabaseFixture.cs

@@ -15,7 +15,7 @@ namespace NzbDrone.Core.Test.Datastore
         public void SingleOrDefault_should_return_null_on_empty_db()
         {
             Mocker.Resolve<IDatabase>()
-                .OpenConnection().Query<Author>("SELECT * FROM Authors")
+                .OpenConnection().Query<Author>("SELECT * FROM \"Authors\"")
                 .SingleOrDefault(c => c.CleanName == "SomeTitle")
                 .Should()
                 .BeNull();

src/NzbDrone.Core.Test/Datastore/LazyLoadingFixture.cs

@@ -78,7 +78,7 @@ namespace NzbDrone.Core.Test.Datastore
         public void should_lazy_load_author_for_trackfile()
         {
             var db = Mocker.Resolve<IDatabase>();
-            var tracks = db.Query<BookFile>(new SqlBuilder()).ToList();
+            var tracks = db.Query<BookFile>(new SqlBuilder(db.DatabaseType)).ToList();
 
             Assert.IsNotEmpty(tracks);
             foreach (var track in tracks)
@@ -94,7 +94,7 @@ namespace NzbDrone.Core.Test.Datastore
         public void should_lazy_load_trackfile_if_not_joined()
         {
             var db = Mocker.Resolve<IDatabase>();
-            var tracks = db.Query<Book>(new SqlBuilder()).ToList();
+            var tracks = db.Query<Book>(new SqlBuilder(db.DatabaseType)).ToList();
 
             foreach (var track in tracks)
             {
@@ -109,7 +109,7 @@ namespace NzbDrone.Core.Test.Datastore
         {
             var db = Mocker.Resolve<IDatabase>();
             var files = MediaFileRepository.Query(db,
-                                                  new SqlBuilder()
+                                                  new SqlBuilder(db.DatabaseType)
                                                   .Join<BookFile, Edition>((t, a) => t.EditionId == a.Id)
                                                   .Join<Edition, Book>((e, b) => e.BookId == b.Id)
                                                   .Join<Book, Author>((book, author) => book.AuthorMetadataId == author.AuthorMetadataId)

src/NzbDrone.Core.Test/Datastore/WhereBuilderPostgresFixture.cs

@@ -0,0 +1,211 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using FluentAssertions;
+using NUnit.Framework;
+using NzbDrone.Core.Books;
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.Datastore
+{
+    [TestFixture]
+    public class WhereBuilderPostgresFixture : CoreTest
+    {
+        private WhereBuilderPostgres _subject;
+
+        [OneTimeSetUp]
+        public void MapTables()
+        {
+            // Generate table mapping
+            Mocker.Resolve<DbFactory>();
+        }
+
+        private WhereBuilderPostgres Where(Expression<Func<Author, bool>> filter)
+        {
+            return new WhereBuilderPostgres(filter, true, 0);
+        }
+
+        private WhereBuilderPostgres WhereMetadata(Expression<Func<AuthorMetadata, bool>> filter)
+        {
+            return new WhereBuilderPostgres(filter, true, 0);
+        }
+
+        [Test]
+        public void postgres_where_equal_const()
+        {
+            _subject = Where(x => x.Id == 10);
+
+            _subject.ToString().Should().Be($"(\"Authors\".\"Id\" = @Clause1_P1)");
+            _subject.Parameters.Get<int>("Clause1_P1").Should().Be(10);
+        }
+
+        [Test]
+        public void postgres_where_equal_variable()
+        {
+            var id = 10;
+            _subject = Where(x => x.Id == id);
+
+            _subject.ToString().Should().Be($"(\"Authors\".\"Id\" = @Clause1_P1)");
+            _subject.Parameters.Get<int>("Clause1_P1").Should().Be(id);
+        }
+
+        [Test]
+        public void postgres_where_equal_property()
+        {
+            var author = new Author { Id = 10 };
+            _subject = Where(x => x.Id == author.Id);
+
+            _subject.Parameters.ParameterNames.Should().HaveCount(1);
+            _subject.ToString().Should().Be($"(\"Authors\".\"Id\" = @Clause1_P1)");
+            _subject.Parameters.Get<int>("Clause1_P1").Should().Be(author.Id);
+        }
+
+        [Test]
+        public void postgres_where_equal_joined_property()
+        {
+            _subject = Where(x => x.QualityProfile.Value.Id == 1);
+
+            _subject.Parameters.ParameterNames.Should().HaveCount(1);
+            _subject.ToString().Should().Be($"(\"QualityProfiles\".\"Id\" = @Clause1_P1)");
+            _subject.Parameters.Get<int>("Clause1_P1").Should().Be(1);
+        }
+
+        [Test]
+        public void postgres_where_throws_without_concrete_condition_if_requiresConcreteCondition()
+        {
+            Expression<Func<Author, Author, bool>> filter = (x, y) => x.Id == y.Id;
+            _subject = new WhereBuilderPostgres(filter, true, 0);
+            Assert.Throws<InvalidOperationException>(() => _subject.ToString());
+        }
+
+        [Test]
+        public void postgres_where_allows_abstract_condition_if_not_requiresConcreteCondition()
+        {
+            Expression<Func<Author, Author, bool>> filter = (x, y) => x.Id == y.Id;
+            _subject = new WhereBuilderPostgres(filter, false, 0);
+            _subject.ToString().Should().Be($"(\"Authors\".\"Id\" = \"Authors\".\"Id\")");
+        }
+
+        [Test]
+        public void postgres_where_string_is_null()
+        {
+            _subject = Where(x => x.CleanName == null);
+
+            _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" IS NULL)");
+        }
+
+        [Test]
+        public void postgres_where_string_is_null_value()
+        {
+            string cleanName = null;
+            _subject = Where(x => x.CleanName == cleanName);
+
+            _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" IS NULL)");
+        }
+
+        [Test]
+        public void postgres_where_equal_null_property()
+        {
+            var author = new Author { CleanName = null };
+            _subject = Where(x => x.CleanName == author.CleanName);
+
+            _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" IS NULL)");
+        }
+
+        [Test]
+        public void postgres_where_column_contains_string()
+        {
+            var test = "small";
+            _subject = Where(x => x.CleanName.Contains(test));
+
+            _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" ILIKE '%' || @Clause1_P1 || '%')");
+            _subject.Parameters.Get<string>("Clause1_P1").Should().Be(test);
+        }
+
+        [Test]
+        public void postgres_where_string_contains_column()
+        {
+            var test = "small";
+            _subject = Where(x => test.Contains(x.CleanName));
+
+            _subject.ToString().Should().Be($"(@Clause1_P1 ILIKE '%' || \"Authors\".\"CleanName\" || '%')");
+            _subject.Parameters.Get<string>("Clause1_P1").Should().Be(test);
+        }
+
+        [Test]
+        public void postgres_where_column_starts_with_string()
+        {
+            var test = "small";
+            _subject = Where(x => x.CleanName.StartsWith(test));
+
+            _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" ILIKE @Clause1_P1 || '%')");
+            _subject.Parameters.Get<string>("Clause1_P1").Should().Be(test);
+        }
+
+        [Test]
+        public void postgres_where_column_ends_with_string()
+        {
+            var test = "small";
+            _subject = Where(x => x.CleanName.EndsWith(test));
+
+            _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" ILIKE '%' || @Clause1_P1)");
+            _subject.Parameters.Get<string>("Clause1_P1").Should().Be(test);
+        }
+
+        [Test]
+        public void postgres_where_in_list()
+        {
+            var list = new List<int> { 1, 2, 3 };
+            _subject = Where(x => list.Contains(x.Id));
+
+            _subject.ToString().Should().Be($"(\"Authors\".\"Id\" = ANY (('{{1, 2, 3}}')))");
+        }
+
+        [Test]
+        public void postgres_where_in_list_2()
+        {
+            var list = new List<int> { 1, 2, 3 };
+            _subject = Where(x => x.CleanName == "test" && list.Contains(x.Id));
+
+            _subject.ToString().Should().Be($"((\"Authors\".\"CleanName\" = @Clause1_P1) AND (\"Authors\".\"Id\" = ANY (('{{1, 2, 3}}'))))");
+        }
+
+        [Test]
+        public void postgres_where_in_string_list()
+        {
+            var list = new List<string> { "first", "second", "third" };
+
+            _subject = Where(x => list.Contains(x.CleanName));
+
+            _subject.ToString().Should().Be($"(\"Authors\".\"CleanName\" = ANY (@Clause1_P1))");
+        }
+
+        [Test]
+        public void enum_as_int()
+        {
+            _subject = WhereMetadata(x => x.Status == AuthorStatusType.Continuing);
+
+            _subject.ToString().Should().Be($"(\"AuthorMetadata\".\"Status\" = @Clause1_P1)");
+        }
+
+        [Test]
+        public void enum_in_list()
+        {
+            var allowed = new List<AuthorStatusType> { AuthorStatusType.Continuing, AuthorStatusType.Ended };
+            _subject = WhereMetadata(x => allowed.Contains(x.Status));
+
+            _subject.ToString().Should().Be($"(\"AuthorMetadata\".\"Status\" = ANY (@Clause1_P1))");
+        }
+
+        [Test]
+        public void enum_in_array()
+        {
+            var allowed = new AuthorStatusType[] { AuthorStatusType.Continuing, AuthorStatusType.Ended };
+            _subject = WhereMetadata(x => allowed.Contains(x.Status));
+
+            _subject.ToString().Should().Be($"(\"AuthorMetadata\".\"Status\" = ANY (@Clause1_P1))");
+        }
+    }
+}

src/NzbDrone.Core.Test/Datastore/WhereBuilderSqliteFixture.cs

@@ -11,9 +11,9 @@ using NzbDrone.Core.Test.Framework;
 namespace NzbDrone.Core.Test.Datastore
 {
     [TestFixture]
-    public class WhereBuilderFixture : CoreTest
+    public class WhereBuilderSqliteFixture : CoreTest
     {
-        private WhereBuilder _subject;
+        private WhereBuilderSqlite _subject;
 
         [OneTimeSetUp]
         public void MapTables()
@@ -22,14 +22,14 @@ namespace NzbDrone.Core.Test.Datastore
             Mocker.Resolve<DbFactory>();
         }
 
-        private WhereBuilder Where(Expression<Func<Author, bool>> filter)
+        private WhereBuilderSqlite Where(Expression<Func<Author, bool>> filter)
         {
-            return new WhereBuilder(filter, true, 0);
+            return new WhereBuilderSqlite(filter, true, 0);
         }
 
-        private WhereBuilder WhereMetadata(Expression<Func<AuthorMetadata, bool>> filter)
+        private WhereBuilderSqlite WhereMetadata(Expression<Func<AuthorMetadata, bool>> filter)
         {
-            return new WhereBuilder(filter, true, 0);
+            return new WhereBuilderSqlite(filter, true, 0);
         }
 
         [Test]
@@ -76,7 +76,7 @@ namespace NzbDrone.Core.Test.Datastore
         public void where_throws_without_concrete_condition_if_requiresConcreteCondition()
         {
             Expression<Func<Author, Author, bool>> filter = (x, y) => x.Id == y.Id;
-            _subject = new WhereBuilder(filter, true, 0);
+            _subject = new WhereBuilderSqlite(filter, true, 0);
             Assert.Throws<InvalidOperationException>(() => _subject.ToString());
         }
 
@@ -84,7 +84,7 @@ namespace NzbDrone.Core.Test.Datastore
         public void where_allows_abstract_condition_if_not_requiresConcreteCondition()
         {
             Expression<Func<Author, Author, bool>> filter = (x, y) => x.Id == y.Id;
-            _subject = new WhereBuilder(filter, false, 0);
+            _subject = new WhereBuilderSqlite(filter, false, 0);
             _subject.ToString().Should().Be($"(\"Authors\".\"Id\" = \"Authors\".\"Id\")");
         }
 

src/NzbDrone.Core.Test/Framework/DbTest.cs

@@ -1,12 +1,18 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Data.SQLite;
+using System.IO;
 using System.Linq;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Npgsql;
 using NUnit.Framework;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Datastore.Migration.Framework;
+using NzbDrone.Test.Common.Datastore;
 
 namespace NzbDrone.Core.Test.Framework
 {
@@ -47,6 +53,7 @@ namespace NzbDrone.Core.Test.Framework
     public abstract class DbTest : CoreTest
     {
         private ITestDatabase _db;
+        private DatabaseType _databaseType;
 
         protected virtual MigrationType MigrationType => MigrationType.Main;
 
@@ -65,8 +72,7 @@ namespace NzbDrone.Core.Test.Framework
 
         protected virtual ITestDatabase WithTestDb(MigrationContext migrationContext)
         {
-            var factory = Mocker.Resolve<DbFactory>();
-            var database = factory.Create(migrationContext);
+            var database = CreateDatabase(migrationContext);
             Mocker.SetConstant(database);
 
             switch (MigrationType)
@@ -98,6 +104,65 @@ namespace NzbDrone.Core.Test.Framework
             return testDb;
         }
 
+        private IDatabase CreateDatabase(MigrationContext migrationContext)
+        {
+            if (_databaseType == DatabaseType.PostgreSQL)
+            {
+                CreatePostgresDb();
+            }
+
+            var factory = Mocker.Resolve<DbFactory>();
+
+            // If a special migration test or log migration then create new
+            if (migrationContext.BeforeMigration != null || _databaseType == DatabaseType.PostgreSQL)
+            {
+                return factory.Create(migrationContext);
+            }
+
+            return CreateSqliteDatabase(factory, migrationContext);
+        }
+
+        private void CreatePostgresDb()
+        {
+            var options = Mocker.Resolve<IOptions<PostgresOptions>>().Value;
+            PostgresDatabase.Create(options, MigrationType);
+        }
+
+        private void DropPostgresDb()
+        {
+            var options = Mocker.Resolve<IOptions<PostgresOptions>>().Value;
+            PostgresDatabase.Drop(options, MigrationType);
+        }
+
+        private IDatabase CreateSqliteDatabase(IDbFactory factory, MigrationContext migrationContext)
+        {
+            // Otherwise try to use a cached migrated db
+            var cachedDb = SqliteDatabase.GetCachedDb(migrationContext.MigrationType);
+            var testDb = GetTestSqliteDb(migrationContext.MigrationType);
+            if (File.Exists(cachedDb))
+            {
+                TestLogger.Info($"Using cached initial database {cachedDb}");
+                File.Copy(cachedDb, testDb);
+                return factory.Create(migrationContext);
+            }
+            else
+            {
+                var db = factory.Create(migrationContext);
+                GC.Collect();
+                GC.WaitForPendingFinalizers();
+                SQLiteConnection.ClearAllPools();
+
+                TestLogger.Info("Caching database");
+                File.Copy(testDb, cachedDb);
+                return db;
+            }
+        }
+
+        private string GetTestSqliteDb(MigrationType type)
+        {
+            return type == MigrationType.Main ? TestFolderInfo.GetDatabase() : TestFolderInfo.GetLogDatabase();
+        }
+
         protected virtual void SetupLogging()
         {
             Mocker.SetConstant<ILoggerProvider>(NullLoggerProvider.Instance);
@@ -108,6 +173,13 @@ namespace NzbDrone.Core.Test.Framework
             WithTempAsAppPath();
             SetupLogging();
 
+            // populate the possible postgres options
+            var postgresOptions = PostgresDatabase.GetTestOptions();
+            _databaseType = postgresOptions.Host.IsNotNullOrWhiteSpace() ? DatabaseType.PostgreSQL : DatabaseType.SQLite;
+
+            // Set up remaining container services
+            Mocker.SetConstant(Options.Create(postgresOptions));
+            Mocker.SetConstant<IConfigFileProvider>(Mocker.Resolve<ConfigFileProvider>());
             Mocker.SetConstant<IConnectionStringFactory>(Mocker.Resolve<ConnectionStringFactory>());
             Mocker.SetConstant<IMigrationController>(Mocker.Resolve<MigrationController>());
 
@@ -127,12 +199,19 @@ namespace NzbDrone.Core.Test.Framework
             // Make sure there are no lingering connections. (When this happens it means we haven't disposed something properly)
             GC.Collect();
             GC.WaitForPendingFinalizers();
+
             SQLiteConnection.ClearAllPools();
+            NpgsqlConnection.ClearAllPools();
 
             if (TestFolderInfo != null)
             {
                 DeleteTempFolder(TestFolderInfo.AppDataFolder);
             }
+
+            if (_databaseType == DatabaseType.PostgreSQL)
+            {
+                DropPostgresDb();
+            }
         }
     }
 }

src/NzbDrone.Core.Test/Framework/TestDatabase.cs

@@ -23,6 +23,7 @@ namespace NzbDrone.Core.Test.Framework
             where T : ModelBase, new();
         IDirectDataMapper GetDirectDataMapper();
         IDbConnection OpenConnection();
+        DatabaseType DatabaseType { get; }
     }
 
     public class TestDatabase : ITestDatabase
@@ -30,6 +31,8 @@ namespace NzbDrone.Core.Test.Framework
         private readonly IDatabase _dbConnection;
         private readonly IEventAggregator _eventAggregator;
 
+        public DatabaseType DatabaseType => _dbConnection.DatabaseType;
+
         public TestDatabase(IDatabase dbConnection)
         {
             _eventAggregator = new Mock<IEventAggregator>().Object;

src/NzbDrone.Core.Test/MusicTests/AlbumRepositoryTests/AlbumRepositoryFixture.cs

@@ -3,8 +3,10 @@ using System.Collections.Generic;
 using System.Linq;
 using FizzWare.NBuilder;
 using FluentAssertions;
+using FluentAssertions.Equivalency;
 using NUnit.Framework;
 using NzbDrone.Core.Books;
+using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Test.Framework;
 
 namespace NzbDrone.Core.Test.MusicTests.BookRepositoryTests
@@ -21,6 +23,13 @@ namespace NzbDrone.Core.Test.MusicTests.BookRepositoryTests
         [SetUp]
         public void Setup()
         {
+            AssertionOptions.AssertEquivalencyUsing(options =>
+            {
+                options.Using<DateTime>(ctx => ctx.Subject.Should().BeCloseTo(ctx.Expectation.ToUniversalTime())).WhenTypeIs<DateTime>();
+                options.Using<DateTime?>(ctx => ctx.Subject.Should().BeCloseTo(ctx.Expectation.Value.ToUniversalTime())).WhenTypeIs<DateTime?>();
+                return options;
+            });
+
             _author = new Author
             {
                 Name = "Alien Ant Farm",
@@ -143,7 +152,7 @@ namespace NzbDrone.Core.Test.MusicTests.BookRepositoryTests
             GivenMultipleBooks();
 
             var result = _bookRepo.GetNextBooks(new[] { _author.AuthorMetadataId });
-            result.Should().BeEquivalentTo(_books.Take(1));
+            result.Should().BeEquivalentTo(_books.Take(1), BookComparerOptions);
         }
 
         [Test]
@@ -152,7 +161,11 @@ namespace NzbDrone.Core.Test.MusicTests.BookRepositoryTests
             GivenMultipleBooks();
 
             var result = _bookRepo.GetLastBooks(new[] { _author.AuthorMetadataId });
-            result.Should().BeEquivalentTo(_books.Skip(2).Take(1));
+            result.Should().BeEquivalentTo(_books.Skip(2).Take(1), BookComparerOptions);
         }
+
+        private EquivalencyAssertionOptions<Book> BookComparerOptions(EquivalencyAssertionOptions<Book> opts) => opts.ComparingByMembers<Book>()
+                .Excluding(ctx => ctx.SelectedMemberInfo.MemberType.IsGenericType && ctx.SelectedMemberInfo.MemberType.GetGenericTypeDefinition() == typeof(LazyLoaded<>))
+                .Excluding(x => x.AuthorId);
     }
 }

src/NzbDrone.Core.Test/MusicTests/ArtistRepositoryTests/ArtistRepositoryFixture.cs

@@ -3,9 +3,11 @@ using System.Collections.Generic;
 using System.Data.SQLite;
 using FizzWare.NBuilder;
 using FluentAssertions;
+using Npgsql;
 using NUnit.Framework;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Books;
+using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Profiles.Metadata;
 using NzbDrone.Core.Profiles.Qualities;
 using NzbDrone.Core.Qualities;
@@ -145,7 +147,14 @@ namespace NzbDrone.Core.Test.MusicTests.AuthorRepositoryTests
             _authorRepo.Insert(author1);
 
             Action insertDupe = () => _authorRepo.Insert(author2);
-            insertDupe.Should().Throw<SQLiteException>();
+            if (Db.DatabaseType == DatabaseType.PostgreSQL)
+            {
+                insertDupe.Should().Throw<PostgresException>();
+            }
+            else
+            {
+                insertDupe.Should().Throw<SQLiteException>();
+            }
         }
     }
 }

src/NzbDrone.Core.Test/MusicTests/RefreshArtistServiceFixture.cs

@@ -106,7 +106,7 @@ namespace NzbDrone.Core.Test.MusicTests
         private void GivenBooksForRefresh(List<Book> books)
         {
             Mocker.GetMock<IBookService>(MockBehavior.Strict)
-                .Setup(s => s.GetBooksForRefresh(It.IsAny<int>(), It.IsAny<IEnumerable<string>>()))
+                .Setup(s => s.GetBooksForRefresh(It.IsAny<int>(), It.IsAny<List<string>>()))
                 .Returns(books);
         }
 
@@ -238,7 +238,7 @@ namespace NzbDrone.Core.Test.MusicTests
 
             Mocker.GetMock<IBookService>(MockBehavior.Strict)
                 .InSequence(seq)
-                .Setup(x => x.GetBooksForRefresh(It.IsAny<int>(), It.IsAny<IEnumerable<string>>()))
+                .Setup(x => x.GetBooksForRefresh(It.IsAny<int>(), It.IsAny<List<string>>()))
                 .Returns(new List<Book>());
 
             // Update called twice for a move/merge
@@ -298,7 +298,7 @@ namespace NzbDrone.Core.Test.MusicTests
 
             Mocker.GetMock<IBookService>(MockBehavior.Strict)
                 .InSequence(seq)
-                .Setup(x => x.GetBooksForRefresh(clash.AuthorMetadataId, It.IsAny<IEnumerable<string>>()))
+                .Setup(x => x.GetBooksForRefresh(clash.AuthorMetadataId, It.IsAny<List<string>>()))
                 .Returns(_books);
 
             // Update called twice for a move/merge