re-adding indexes for series/episodes.

2026-04-19 21:44:30 -04:00 · 2013-09-04 17:06:24 -07:00
parent c90a96c2e0
commit 9390a00f80
19 changed files with 308 additions and 78 deletions
@@ -8,8 +8,8 @@ namespace NzbDrone.Core.Datastore.Migration
    {
        protected override void MainDbUpgrade()
        {
-            SQLiteAlter.DropColumns("Series", new[] { "BacklogSetting" });
-            SQLiteAlter.DropColumns("NamingConfig", new[] { "UseSceneName" });
+            SqLiteAlter.DropColumns("Series", new[] { "BacklogSetting" });
+            SqLiteAlter.DropColumns("NamingConfig", new[] { "UseSceneName" });
        }
    }
 }
@@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration
    {
        protected override void MainDbUpgrade()
        {
-            SQLiteAlter.DropColumns("NamingConfig", new[] { "SeasonFolderFormat" });
+            SqLiteAlter.DropColumns("NamingConfig", new[] { "SeasonFolderFormat" });

            Execute.Sql("UPDATE NamingConfig SET RenameEpisodes = 1 WHERE RenameEpisodes = -1");
            Execute.Sql("UPDATE NamingConfig SET RenameEpisodes = 0 WHERE RenameEpisodes = -2");
@@ -8,8 +8,8 @@ namespace NzbDrone.Core.Datastore.Migration
    {
        protected override void MainDbUpgrade()
        {
-            SQLiteAlter.DropColumns("Episodes", new[] { "Ignored" });
-            SQLiteAlter.DropColumns("Seasons", new[] { "Ignored" });
+            SqLiteAlter.DropColumns("Episodes", new[] { "Ignored" });
+            SqLiteAlter.DropColumns("Seasons", new[] { "Ignored" });
        }
    }
 }
@@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration
    {
        protected override void MainDbUpgrade()
        {
-            SQLiteAlter.DropColumns("Series", new[] { "CustomStartDate" });
+            SqLiteAlter.DropColumns("Series", new[] { "CustomStartDate" });
        }
    }
 }
@@ -8,7 +8,7 @@ namespace NzbDrone.Core.Datastore.Migration
    {
        protected override void MainDbUpgrade()
        {
-            SQLiteAlter.DropColumns("Episodes", new []{ "AirDate" });
+            SqLiteAlter.DropColumns("Episodes", new []{ "AirDate" });
        }
    }
 }
@@ -0,0 +1,60 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+using System.Linq;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(18)]
+    public class remove_duplicates : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            using (var transaction = MigrationHelper.BeginTransaction())
+            {
+                RemoveDuplicateSeries("TvdbId");
+                RemoveDuplicateSeries("TitleSlug");
+
+                var duplicatedEpisodes = MigrationHelper.GetDuplicates<int>("Episodes", "TvDbEpisodeId");
+
+                foreach (var duplicate in duplicatedEpisodes)
+                {
+                    foreach (var episodeId in duplicate.OrderBy(c => c.Key).Skip(1).Select(c => c.Key))
+                    {
+                        RemoveEpisodeRows(episodeId);
+                    }
+                }
+
+                transaction.Commit();
+            }
+        }
+
+        private void RemoveDuplicateSeries(string field)
+        {
+            var duplicatedSeries = MigrationHelper.GetDuplicates<int>("Series", field);
+
+            foreach (var duplicate in duplicatedSeries)
+            {
+                foreach (var seriesId in duplicate.OrderBy(c => c.Key).Skip(1).Select(c => c.Key))
+                {
+                    RemoveSeriesRows(seriesId);
+                }
+            }
+        }
+
+        private void RemoveSeriesRows(int seriesId)
+        {
+            MigrationHelper.ExecuteNonQuery("DELETE FROM Series WHERE Id = {0}", seriesId.ToString());
+            MigrationHelper.ExecuteNonQuery("DELETE FROM Episodes WHERE SeriesId = {0}", seriesId.ToString());
+            MigrationHelper.ExecuteNonQuery("DELETE FROM Seasons WHERE SeriesId = {0}", seriesId.ToString());
+            MigrationHelper.ExecuteNonQuery("DELETE FROM History WHERE SeriesId = {0}", seriesId.ToString());
+            MigrationHelper.ExecuteNonQuery("DELETE FROM EpisodeFiles WHERE SeriesId = {0}", seriesId.ToString());
+        }
+
+        private void RemoveEpisodeRows(int episodeId)
+        {
+            MigrationHelper.ExecuteNonQuery("DELETE FROM Episodes WHERE Id = {0}", episodeId.ToString());
+            MigrationHelper.ExecuteNonQuery("DELETE FROM History WHERE EpisodeId = {0}", episodeId.ToString());
+        }
+
+    }
+}
@@ -0,0 +1,20 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(19)]
+    public class restore_unique_constraints : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            SqLiteAlter.AddIndexes("Series", 
+                new SQLiteIndex { Column = "TvdbId", Table = "Series", Unique = true },
+                new SQLiteIndex { Column = "TitleSlug", Table = "Series", Unique = true });
+
+            SqLiteAlter.AddIndexes("Episodes", 
+                new SQLiteIndex { Column = "TvDbEpisodeId", Table = "Episodes", Unique = true });
+        }
+
+    }
+}
@@ -4,5 +4,6 @@
    {
        public MigrationType MigrationType { get; set; }
        public ISQLiteAlter SQLiteAlter { get; set; }
+        public ISqLiteMigrationHelper MigrationHelper { get; set; }
    }
 }
@@ -15,13 +15,15 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
    {
        private readonly IAnnouncer _announcer;
        private readonly ISQLiteAlter _sqLiteAlter;
+        private readonly ISqLiteMigrationHelper _migrationHelper;

        private static readonly HashSet<string> MigrationCache = new HashSet<string>();

-        public MigrationController(IAnnouncer announcer, ISQLiteAlter sqLiteAlter)
+        public MigrationController(IAnnouncer announcer, ISQLiteAlter sqLiteAlter, ISqLiteMigrationHelper migrationHelper)
        {
            _announcer = announcer;
            _sqLiteAlter = sqLiteAlter;
+            _migrationHelper = migrationHelper;
        }

        public void MigrateToLatest(string connectionString, MigrationType migrationType)
@@ -40,7 +42,8 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
                        ApplicationContext = new MigrationContext
                            {
                                MigrationType = migrationType,
-                                SQLiteAlter = _sqLiteAlter
+                                SQLiteAlter = _sqLiteAlter,
+                                MigrationHelper = _migrationHelper,
                            }
                    };

@@ -1,9 +1,18 @@
 using System;
+using NLog;
+using NzbDrone.Common.Instrumentation;

 namespace NzbDrone.Core.Datastore.Migration.Framework
 {
    public abstract class NzbDroneMigrationBase : FluentMigrator.Migration
    {
+        private Logger _logger;
+
+        protected NzbDroneMigrationBase()
+        {
+            _logger = NzbDroneLogger.GetLogger();
+        }
+
        protected virtual void MainDbUpgrade()
        {
        }
@@ -16,7 +25,8 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
        {
            var context = (MigrationContext)ApplicationContext;

-            SQLiteAlter = context.SQLiteAlter;
+            SqLiteAlter = context.SQLiteAlter;
+            MigrationHelper = context.MigrationHelper;

            switch (context.MigrationType)
            {
@@ -33,7 +43,8 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
            }
        }

-        protected ISQLiteAlter SQLiteAlter { get; private set; }
+        protected ISQLiteAlter SqLiteAlter { get; private set; }
+        protected ISqLiteMigrationHelper MigrationHelper { get; private set; }

        public override void Down()
        {
@@ -0,0 +1,13 @@
+namespace NzbDrone.Core.Datastore.Migration.Framework
+{
+    public class SQLiteColumn
+    {
+        public string Name { get; set; }
+        public string Schema { get; set; }
+
+        public override string ToString()
+        {
+            return string.Format("[{0}] {1}", Name, Schema);
+        }
+    }
+}
@@ -0,0 +1,39 @@
+using System;
+
+namespace NzbDrone.Core.Datastore.Migration.Framework
+{
+    public class SQLiteIndex : IEquatable<SQLiteIndex>
+    {
+        public string Column { get; set; }
+        public string Table { get; set; }
+        public bool Unique { get; set; }
+
+        public bool Equals(SQLiteIndex other)
+        {
+            return IndexName == other.IndexName;
+        }
+
+        public override int GetHashCode()
+        {
+            return IndexName.GetHashCode();
+        }
+
+        public override string ToString()
+        {
+            return string.Format("[{0}] Unique: {1}", Column, Unique);
+        }
+
+        public string IndexName
+        {
+            get
+            {
+                return string.Format("IX_{0}_{1}", Table, Column);
+            }
+        }
+
+        public string CreateSql(string tableName)
+        {
+            return string.Format(@"CREATE UNIQUE INDEX ""{2}"" ON ""{0}"" (""{1}"" ASC)", tableName, Column, IndexName);
+        }
+    }
+}
@@ -7,18 +7,21 @@ using NLog;

 namespace NzbDrone.Core.Datastore.Migration.Framework
 {
-    public interface ISQLiteMigrationHelper
+    public interface ISqLiteMigrationHelper
    {
-        Dictionary<String, SQLiteMigrationHelper.SQLiteColumn> GetColumns(string tableName);
-        void CreateTable(string tableName, IEnumerable<SQLiteMigrationHelper.SQLiteColumn> values, IEnumerable<SQLiteMigrationHelper.SQLiteIndex> indexes);
-        void CopyData(string sourceTable, string destinationTable, IEnumerable<SQLiteMigrationHelper.SQLiteColumn> columns);
+        Dictionary<String, SQLiteColumn> GetColumns(string tableName);
+        void CreateTable(string tableName, IEnumerable<SQLiteColumn> values, IEnumerable<SQLiteIndex> indexes);
+        void CopyData(string sourceTable, string destinationTable, IEnumerable<SQLiteColumn> columns);
        void DropTable(string tableName);
        void RenameTable(string tableName, string newName);
+        IEnumerable<IGrouping<T, KeyValuePair<int, T>>> GetDuplicates<T>(string tableName, string columnName);
        SQLiteTransaction BeginTransaction();
-        List<SQLiteMigrationHelper.SQLiteIndex> GetIndexes(string tableName);
+        List<SQLiteIndex> GetIndexes(string tableName);
+        int ExecuteScalar(string command, params string[] args);
+        void ExecuteNonQuery(string command, params string[] args);
    }

-    public class SQLiteMigrationHelper : ISQLiteMigrationHelper
+    public class SqLiteMigrationHelper : ISqLiteMigrationHelper
    {
        private readonly SQLiteConnection _connection;

@@ -28,7 +31,7 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
        private static readonly Regex IndexRegex = new Regex(@"\(""(?<col>.*)""\s(?<direction>ASC|DESC)\)$",
             RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Multiline);

-        public SQLiteMigrationHelper(IConnectionStringFactory connectionStringFactory, Logger logger)
+        public SqLiteMigrationHelper(IConnectionStringFactory connectionStringFactory, Logger logger)
        {
            try
            {
@@ -69,18 +72,22 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
                                   });
        }

+
+        private static IEnumerable<T> ReadArray<T>(SQLiteDataReader reader)
+        {
+            while (reader.Read())
+            {
+                yield return (T)Convert.ChangeType(reader[0], typeof(T));
+            }
+        }
+
        public List<SQLiteIndex> GetIndexes(string tableName)
        {
            var command = new SQLiteCommand(string.Format("SELECT sql FROM sqlite_master WHERE type='index' AND tbl_name ='{0}'", tableName));
            command.Connection = _connection;

            var reader = command.ExecuteReader();
-            var sqls = new List<string>();
-
-            while (reader.Read())
-            {
-                sqls.Add(reader[0].ToString());
-            }
+            var sqls = ReadArray<string>(reader).ToList();


            var indexes = new List<SQLiteIndex>();
@@ -108,7 +115,7 @@ namespace NzbDrone.Core.Datastore.Migration.Framework

            foreach (var index in indexes)
            {
-                ExecuteNonQuery("DROP INDEX {0}", index.IndexName);
+                ExecuteNonQuery("DROP INDEX IF EXISTS {0}", index.IndexName);
                ExecuteNonQuery(index.CreateSql(tableName));
            }
        }
@@ -146,6 +153,23 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
            renameCommand.ExecuteNonQuery();
        }

+        public IEnumerable<IGrouping<T, KeyValuePair<int, T>>> GetDuplicates<T>(string tableName, string columnName)
+        {
+            var getDuplicates = BuildCommand("select id, {0} from {1}", columnName, tableName);
+
+            var result = new List<KeyValuePair<int, T>>();
+
+            using (var reader = getDuplicates.ExecuteReader())
+            {
+                while (reader.Read())
+                {
+                    result.Add(new KeyValuePair<int, T>(reader.GetInt16(0), (T)Convert.ChangeType(reader[1], typeof(T))));
+                }
+            }
+
+            return result.GroupBy(c => c.Value).Where(g => g.Count() > 1);
+        }
+
        public int GetRowCount(string tableName)
        {
            var countCommand = BuildCommand("SELECT COUNT(*) FROM {0};", tableName);
@@ -166,8 +190,7 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
        }


-
-        private void ExecuteNonQuery(string command, params string[] args)
+        public void ExecuteNonQuery(string command, params string[] args)
        {
            var sqLiteCommand = new SQLiteCommand(string.Format(command, args))
            {
@@ -177,43 +200,17 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
            sqLiteCommand.ExecuteNonQuery();
        }

-
-        public class SQLiteColumn
+        public int ExecuteScalar(string command, params string[] args)
        {
-            public string Name { get; set; }
-            public string Schema { get; set; }
-
-            public override string ToString()
+            var sqLiteCommand = new SQLiteCommand(string.Format(command, args))
            {
-                return string.Format("[{0}] {1}", Name, Schema);
-            }
+                Connection = _connection
+            };
+
+           return (int)sqLiteCommand.ExecuteScalar();
        }

-        public class SQLiteIndex
-        {
-            public string Column { get; set; }
-            public string Table { get; set; }
-            public bool Unique { get; set; }

-            public override string ToString()
-            {
-                return string.Format("[{0}] Unique: {1}", Column, Unique);
-            }
-
-            public string IndexName
-            {
-                get
-                {
-                    return string.Format("IX_{0}_{1}", Table, Column);
-                }
-            }
-
-            public string CreateSql(string tableName)
-            {
-                return string.Format(@"CREATE UNIQUE INDEX ""{2}"" ON ""{0}"" (""{1}"" ASC)", tableName, Column, IndexName);
-            }
-        }
+       
    }
-
-
 }
@@ -6,13 +6,14 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
    public interface ISQLiteAlter
    {
        void DropColumns(string tableName, IEnumerable<string> columns);
+        void AddIndexes(string tableName, params SQLiteIndex[] indexes);
    }

    public class SQLiteAlter : ISQLiteAlter
    {
-        private readonly ISQLiteMigrationHelper _sqLiteMigrationHelper;
+        private readonly ISqLiteMigrationHelper _sqLiteMigrationHelper;

-        public SQLiteAlter(ISQLiteMigrationHelper sqLiteMigrationHelper)
+        public SQLiteAlter(ISqLiteMigrationHelper sqLiteMigrationHelper)
        {
            _sqLiteMigrationHelper = sqLiteMigrationHelper;
        }
@@ -27,18 +28,39 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
                var newColumns = originalColumns.Where(c => !columns.Contains(c.Key)).Select(c => c.Value).ToList();
                var newIndexes = originalIndexes.Where(c => !columns.Contains(c.Column));

-                var tempTableName = tableName + "_temp";
-
-                _sqLiteMigrationHelper.CreateTable(tempTableName, newColumns, newIndexes);
-
-                _sqLiteMigrationHelper.CopyData(tableName, tempTableName, newColumns);
-
-                _sqLiteMigrationHelper.DropTable(tableName);
-
-                _sqLiteMigrationHelper.RenameTable(tempTableName, tableName);
+                CreateTable(tableName, newColumns, newIndexes);

                transaction.Commit();
            }
        }
+
+        public void AddIndexes(string tableName, params SQLiteIndex[] indexes)
+        {
+            using (var transaction = _sqLiteMigrationHelper.BeginTransaction())
+            {
+                var columns = _sqLiteMigrationHelper.GetColumns(tableName).Select(c => c.Value).ToList();
+                var originalIndexes = _sqLiteMigrationHelper.GetIndexes(tableName);
+
+                var newIndexes = originalIndexes.Union(indexes);
+
+
+                CreateTable(tableName, columns, newIndexes);
+
+                transaction.Commit();
+            }
+        }
+
+        private void CreateTable(string tableName, List<SQLiteColumn> newColumns, IEnumerable<SQLiteIndex> newIndexes)
+        {
+            var tempTableName = tableName + "_temp";
+
+            _sqLiteMigrationHelper.CreateTable(tempTableName, newColumns, newIndexes);
+
+            _sqLiteMigrationHelper.CopyData(tableName, tempTableName, newColumns);
+
+            _sqLiteMigrationHelper.DropTable(tableName);
+
+            _sqLiteMigrationHelper.RenameTable(tempTableName, tableName);
+        }
    }
 }