From 2b11fde6a3dd8d86f5e3409ed9117e9bdadebd76 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Wed, 22 Oct 2025 09:02:28 +0200 Subject: [PATCH 01/19] Added test for default value (not reset on ChangeColumn) --- .../Generic/Generic_ChangeColumnTestsBase.cs | 40 +++++++++++++++++++ ...ransformationProvider_ChangeColumnTests.cs | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/Migrator.Tests/Providers/Generic/Generic_ChangeColumnTestsBase.cs b/src/Migrator.Tests/Providers/Generic/Generic_ChangeColumnTestsBase.cs index 6622eaf8..b20ad58b 100644 --- a/src/Migrator.Tests/Providers/Generic/Generic_ChangeColumnTestsBase.cs +++ b/src/Migrator.Tests/Providers/Generic/Generic_ChangeColumnTestsBase.cs @@ -1,4 +1,7 @@ +using System; +using System.Collections.Generic; using System.Data; +using System.Linq; using DotNetProjects.Migrator.Framework; using Migrator.Tests.Providers.Base; using NUnit.Framework; @@ -30,4 +33,41 @@ public void ChangeColumn_NotNullAndNullToNotNull_Success() Assert.That(column1.ColumnProperty.HasFlag(ColumnProperty.NotNull), Is.True); Assert.That(column2.ColumnProperty.HasFlag(ColumnProperty.NotNull), Is.True); } + + [Test, Ignore("Not yet implemented. See issue https://github.com/dotnetprojects/Migrator.NET/issues/139")] + public void ChangeColumn_RemoveDefaultValue_Success() + { + // Arrange + var tableName = "TableName"; + var column1Name = "Column1"; + var column2Name = "Column2"; + + var testTime = new DateTime(2025, 5, 5, 5, 5, 5, DateTimeKind.Utc); + + Provider.AddTable(tableName, + new Column(name: column1Name, type: DbType.Int32, property: ColumnProperty.NotNull), + new Column(name: column2Name, type: DbType.DateTime2, property: ColumnProperty.Null, defaultValue: testTime) + ); + + // Act + Provider.Insert(table: tableName, [column1Name], [1]); + Provider.ChangeColumn(table: tableName, column: new Column(name: column2Name, type: DbType.DateTime2, property: ColumnProperty.Null)); + + // Assert + Provider.Insert(table: tableName, [column1Name], [2]); + + using var cmd = Provider.CreateCommand(); + using var reader = Provider.Select(cmd: cmd, table: tableName, columns: [column1Name, column2Name]); + + List<(int, DateTime)> records = []; + + while (reader.Read()) + { + records.Add((reader.GetInt32(0), reader.GetDateTime(1))); + } + + Assert.That(records.Count, Is.EqualTo(2)); + Assert.That(records.Single(x => x.Item1 == 1).Item2, Is.EqualTo(testTime)); + Assert.That(records.Single(x => x.Item1 == 2).Item2, Is.Null); + } } \ No newline at end of file diff --git a/src/Migrator.Tests/Providers/SQLServer/SQLServerTransformationProvider_ChangeColumnTests.cs b/src/Migrator.Tests/Providers/SQLServer/SQLServerTransformationProvider_ChangeColumnTests.cs index 25199843..754fad52 100644 --- a/src/Migrator.Tests/Providers/SQLServer/SQLServerTransformationProvider_ChangeColumnTests.cs +++ b/src/Migrator.Tests/Providers/SQLServer/SQLServerTransformationProvider_ChangeColumnTests.cs @@ -8,7 +8,7 @@ namespace Migrator.Tests.Providers.SQLServer; [TestFixture] [Category("SqlServer")] -public class SQLServerTransformationProvider_ChangeColumnTests : Generic_AddIndexTestsBase +public class SQLServerTransformationProvider_ChangeColumnTests : Generic_ChangeColumnTestsBase { [SetUp] public async Task SetUpAsync() From 2adefe8fed2ee7fe6c72a958eac76eea68014eea Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Tue, 28 Oct 2025 18:48:06 +0100 Subject: [PATCH 02/19] AddTable now uses GENERATED ALWAYS AS IDENTITY in Oracle / Added data loader for oracle --- .../Generic/Generic_AddTableTestsBase.cs | 63 ++++++++ ...leTransformationProvider_AddTable_Tests.cs | 36 +---- ...SQLTransformationProvider_AddTableTests.cs | 19 +-- ...verTransformationProvider_AddTableTests.cs | 4 +- ...iteTransformationProvider_AddTableTests.cs | 11 +- src/Migrator/Framework/IDialect.cs | 24 ++- .../Providers/ColumnPropertiesMapper.cs | 62 ++++---- src/Migrator/Providers/Dialect.cs | 20 +++ .../FirebirdColumnPropertiesMapper.cs | 4 +- .../Interfaces/IOracleSystemDataLoader.cs | 29 ++++ .../Oracle/Data/OracleSystemDataLoader.cs | 137 ++++++++++++++++++ .../IOracleTransformationProvider.cs | 7 + .../Impl/Oracle/Models/AllTablIdentityCols.cs | 27 ++++ .../Impl/Oracle/Models/PrimaryKeyItem.cs | 29 ++++ .../Oracle/OracleColumnPropertiesMapper.cs | 6 +- .../Providers/Impl/Oracle/OracleDialect.cs | 48 ++---- .../Oracle/OracleTransformationProvider.cs | 110 +++++++------- .../PostgreSQLTransformationProvider.cs | 2 +- .../SQLite/SQLiteColumnPropertiesMapper.cs | 2 +- .../SQLite/SQLiteTransformationProvider.cs | 1 + 20 files changed, 456 insertions(+), 185 deletions(-) create mode 100644 src/Migrator.Tests/Providers/Generic/Generic_AddTableTestsBase.cs create mode 100644 src/Migrator/Providers/Impl/Oracle/Data/Interfaces/IOracleSystemDataLoader.cs create mode 100644 src/Migrator/Providers/Impl/Oracle/Data/OracleSystemDataLoader.cs create mode 100644 src/Migrator/Providers/Impl/Oracle/Interfaces/IOracleTransformationProvider.cs create mode 100644 src/Migrator/Providers/Impl/Oracle/Models/AllTablIdentityCols.cs create mode 100644 src/Migrator/Providers/Impl/Oracle/Models/PrimaryKeyItem.cs diff --git a/src/Migrator.Tests/Providers/Generic/Generic_AddTableTestsBase.cs b/src/Migrator.Tests/Providers/Generic/Generic_AddTableTestsBase.cs new file mode 100644 index 00000000..97c315f9 --- /dev/null +++ b/src/Migrator.Tests/Providers/Generic/Generic_AddTableTestsBase.cs @@ -0,0 +1,63 @@ +using System.Data; +using DotNetProjects.Migrator.Framework; +using Migrator.Tests.Providers.Base; +using NUnit.Framework; + +namespace Migrator.Tests.Providers.Generic; + +[TestFixture] +public abstract class Generic_AddTableTestsBase : TransformationProviderBase +{ + [Test] + public void AddTable_PrimaryKeyWithIdentity_Success() + { + // Arrange + var tableName = "TableName"; + var column1Name = "Column1"; + var column2Name = "Column2"; + + // Act + Provider.AddTable(tableName, + new Column(column1Name, DbType.Int32, ColumnProperty.NotNull | ColumnProperty.PrimaryKeyWithIdentity), + new Column(column2Name, DbType.Int32, ColumnProperty.NotNull) + ); + + // Assert + var column1 = Provider.GetColumnByName(tableName, column1Name); + var column2 = Provider.GetColumnByName(tableName, column2Name); + + Assert.That(column1.ColumnProperty.HasFlag(ColumnProperty.PrimaryKeyWithIdentity), Is.True); + Assert.That(column2.ColumnProperty.HasFlag(ColumnProperty.NotNull), Is.True); + } + + [Test] + public void AddTable_NotNull_Success() + { + // Arrange + var tableName = "TableName"; + var column1Name = "Column1"; + + // Act + Provider.AddTable(tableName, + new Column(column1Name, DbType.Int32, ColumnProperty.NotNull) + ); + + // Assert + var column1 = Provider.GetColumnByName(tableName, column1Name); + + Assert.That(column1.ColumnProperty.HasFlag(ColumnProperty.NotNull), Is.True); + } + + + [Test] + public void AddTableWithCompoundPrimaryKey() + { + Provider.AddTable("Test", + new Column("PersonId", DbType.Int32, ColumnProperty.PrimaryKey), + new Column("AddressId", DbType.Int32, ColumnProperty.PrimaryKey) + ); + + Assert.That(Provider.TableExists("Test"), Is.True, "Table doesn't exist"); + Assert.That(Provider.PrimaryKeyExists("Test", "PK_Test"), Is.True, "Constraint doesn't exist"); + } +} \ No newline at end of file diff --git a/src/Migrator.Tests/Providers/OracleProvider/OracleTransformationProvider_AddTable_Tests.cs b/src/Migrator.Tests/Providers/OracleProvider/OracleTransformationProvider_AddTable_Tests.cs index ca6414eb..c5e9f53e 100644 --- a/src/Migrator.Tests/Providers/OracleProvider/OracleTransformationProvider_AddTable_Tests.cs +++ b/src/Migrator.Tests/Providers/OracleProvider/OracleTransformationProvider_AddTable_Tests.cs @@ -1,48 +1,16 @@ -using System.Data; using System.Threading.Tasks; -using DotNetProjects.Migrator.Framework; -using Migrator.Tests.Providers.Base; +using Migrator.Tests.Providers.Generic; using NUnit.Framework; namespace Migrator.Tests.Providers.OracleProvider; [TestFixture] [Category("Oracle")] -public class OracleTransformationProvider_AddTable_Tests : TransformationProviderBase +public class OracleTransformationProvider_AddTable_Tests : Generic_AddTableTestsBase { [SetUp] public async Task SetUpAsync() { await BeginOracleTransactionAsync(); } - - [Test] - public void AddTableWithCompoundPrimaryKey() - { - Provider.AddTable("Test", - new Column("PersonId", DbType.Int32, ColumnProperty.PrimaryKey), - new Column("AddressId", DbType.Int32, ColumnProperty.PrimaryKey) - ); - - Assert.That(Provider.TableExists("Test"), Is.True, "Table doesn't exist"); - Assert.That(Provider.PrimaryKeyExists("Test", "PK_Test"), Is.True, "Constraint doesn't exist"); - } - - [Test] - public void AddTable_NotNull_Success() - { - // Arrange - var tableName = "TableName"; - var column1Name = "Column1"; - - // Act - Provider.AddTable(tableName, - new Column(column1Name, DbType.Int32, ColumnProperty.NotNull) - ); - - // Assert - var column1 = Provider.GetColumnByName(tableName, column1Name); - - Assert.That(column1.ColumnProperty.HasFlag(ColumnProperty.NotNull), Is.True); - } } \ No newline at end of file diff --git a/src/Migrator.Tests/Providers/PostgreSQL/PostgreSQLTransformationProvider_AddTableTests.cs b/src/Migrator.Tests/Providers/PostgreSQL/PostgreSQLTransformationProvider_AddTableTests.cs index 3efe09c6..d09187b1 100644 --- a/src/Migrator.Tests/Providers/PostgreSQL/PostgreSQLTransformationProvider_AddTableTests.cs +++ b/src/Migrator.Tests/Providers/PostgreSQL/PostgreSQLTransformationProvider_AddTableTests.cs @@ -1,23 +1,16 @@ -using System.Data; -using DotNetProjects.Migrator.Framework; -using Migrator.Tests.Providers.PostgreSQL.Base; +using System.Threading.Tasks; +using Migrator.Tests.Providers.Generic; using NUnit.Framework; namespace Migrator.Tests.Providers.PostgreSQL; [TestFixture] [Category("Postgre")] -public class PostgreSQLTransformationProvider_AddTableTests : PostgreSQLTransformationProviderTestBase +public class PostgreSQLTransformationProvider_AddTableTests : Generic_AddTableTestsBase { - [Test] - public void AddTableWithCompoundPrimaryKey() + [SetUp] + public async Task SetUpAsync() { - Provider.AddTable("Test", - new Column("PersonId", DbType.Int32, ColumnProperty.PrimaryKey), - new Column("AddressId", DbType.Int32, ColumnProperty.PrimaryKey) - ); - - Assert.That(Provider.TableExists("Test"), Is.True, "Table doesn't exist"); - Assert.That(Provider.PrimaryKeyExists("Test", "PK_Test"), Is.True, "Constraint doesn't exist"); + await BeginPostgreSQLTransactionAsync(); } } diff --git a/src/Migrator.Tests/Providers/SQLServer/SQLServerTransformationProvider_AddTableTests.cs b/src/Migrator.Tests/Providers/SQLServer/SQLServerTransformationProvider_AddTableTests.cs index bc423f11..1632e892 100644 --- a/src/Migrator.Tests/Providers/SQLServer/SQLServerTransformationProvider_AddTableTests.cs +++ b/src/Migrator.Tests/Providers/SQLServer/SQLServerTransformationProvider_AddTableTests.cs @@ -1,14 +1,14 @@ using System.Data; using System.Threading.Tasks; using DotNetProjects.Migrator.Framework; -using Migrator.Tests.Providers.Base; +using Migrator.Tests.Providers.Generic; using NUnit.Framework; namespace Migrator.Tests.Providers.SQLServer; [TestFixture] [Category("SqlServer")] -public class SQLServerTransformationProvider_AddTableTests : TransformationProviderBase +public class SQLServerTransformationProvider_AddTableTests : Generic_AddTableTestsBase { [SetUp] public async Task SetUpAsync() diff --git a/src/Migrator.Tests/Providers/SQLite/SQLiteTransformationProvider_AddTableTests.cs b/src/Migrator.Tests/Providers/SQLite/SQLiteTransformationProvider_AddTableTests.cs index deaaef68..269c2661 100644 --- a/src/Migrator.Tests/Providers/SQLite/SQLiteTransformationProvider_AddTableTests.cs +++ b/src/Migrator.Tests/Providers/SQLite/SQLiteTransformationProvider_AddTableTests.cs @@ -1,16 +1,23 @@ using System.Data.SQLite; using System.Linq; +using System.Threading.Tasks; using DotNetProjects.Migrator.Framework; using DotNetProjects.Migrator.Providers.Impl.SQLite; -using Migrator.Tests.Providers.SQLite.Base; +using Migrator.Tests.Providers.Generic; using NUnit.Framework; namespace Migrator.Tests.Providers.SQLite; [TestFixture] [Category("SQLite")] -public class SQLiteTransformationProvider_AddTableTests : SQLiteTransformationProviderTestBase +public class SQLiteTransformationProvider_AddTableTests : Generic_AddTableTestsBase { + [SetUp] + public async Task SetUpAsync() + { + await BeginSQLiteTransactionAsync(); + } + [Test] public void AddTable_UniqueOnlyOnColumnLevel_Obsolete_UniquesListIsEmpty() { diff --git a/src/Migrator/Framework/IDialect.cs b/src/Migrator/Framework/IDialect.cs index 13d25bb9..ffe0cf78 100644 --- a/src/Migrator/Framework/IDialect.cs +++ b/src/Migrator/Framework/IDialect.cs @@ -51,8 +51,9 @@ public interface IDialect DbType GetDbType(string databaseTypeName); void RegisterProperty(ColumnProperty property, string sql); + string SqlForProperty(ColumnProperty property, Column column); - string Quote(string value); + string Default(object defaultValue); /// @@ -61,4 +62,25 @@ public interface IDialect /// The DbType /// True if the database type has an unsigned variant, otherwise false bool IsUnsignedCompatible(DbType type); + + /// + /// Quotes the string. + /// + /// + /// + string Quote(string value); + + /// + /// Quotes the table name if necessary. + /// + /// + /// + string QuoteTableNameIfRequired(string tableName); + + /// + /// Quotes the column name if necessary. + /// + /// + /// + string QuoteColumnNameIfRequired(string columnName); } diff --git a/src/Migrator/Providers/ColumnPropertiesMapper.cs b/src/Migrator/Providers/ColumnPropertiesMapper.cs index 910c15eb..da3e6213 100644 --- a/src/Migrator/Providers/ColumnPropertiesMapper.cs +++ b/src/Migrator/Providers/ColumnPropertiesMapper.cs @@ -12,30 +12,30 @@ public class ColumnPropertiesMapper /// /// the type of the column /// - protected string columnSql; + protected string _ColumnSql; /// /// Sql if this column has a default value /// - protected object defaultVal; + protected object _DefaultVal; - protected Dialect dialect; + protected Dialect _Dialect; /// /// Sql if This column is Indexed /// - protected bool indexed; + protected bool _Indexed; /// The name of the column - protected string name; + protected string _Name; /// The SQL type - public string type { get; private set; } + public string Type { get; private set; } - public ColumnPropertiesMapper(Dialect dialect, string type) + public ColumnPropertiesMapper(Dialect dialect, string typeString) { - this.dialect = dialect; - this.type = type; + _Dialect = dialect; + Type = typeString; } /// @@ -43,33 +43,33 @@ public ColumnPropertiesMapper(Dialect dialect, string type) /// public virtual string ColumnSql { - get { return columnSql; } + get { return _ColumnSql; } } public string Name { - get { return name; } - set { name = value; } + get { return _Name; } + set { _Name = value; } } public object Default { - get { return defaultVal; } - set { defaultVal = value; } + get { return _DefaultVal; } + set { _DefaultVal = value; } } public string QuotedName { - get { return dialect.Quote(Name); } + get { return _Dialect.Quote(Name); } } public string IndexSql { get { - if (dialect.SupportsIndex && indexed) + if (_Dialect.SupportsIndex && _Indexed) { - return string.Format("INDEX({0})", dialect.Quote(name)); + return string.Format("INDEX({0})", _Dialect.Quote(_Name)); } return null; @@ -80,7 +80,7 @@ public virtual void MapColumnProperties(Column column) { Name = column.Name; - indexed = PropertySelected(column.ColumnProperty, ColumnProperty.Indexed); + _Indexed = PropertySelected(column.ColumnProperty, ColumnProperty.Indexed); var vals = new List(); @@ -110,14 +110,14 @@ public virtual void MapColumnProperties(Column column) AddDefaultValue(column, vals); - columnSql = string.Join(" ", vals.ToArray()); + _ColumnSql = string.Join(" ", vals.ToArray()); } public virtual void MapColumnPropertiesWithoutDefault(Column column) { Name = column.Name; - indexed = PropertySelected(column.ColumnProperty, ColumnProperty.Indexed); + _Indexed = PropertySelected(column.ColumnProperty, ColumnProperty.Indexed); var vals = new List(); @@ -145,7 +145,7 @@ public virtual void MapColumnPropertiesWithoutDefault(Column column) AddForeignKey(column, vals); - columnSql = string.Join(" ", vals.ToArray()); + _ColumnSql = string.Join(" ", vals.ToArray()); } protected virtual void AddCaseSensitive(Column column, List vals) @@ -157,7 +157,7 @@ protected virtual void AddDefaultValue(Column column, List vals) { if (column.DefaultValue != null) { - vals.Add(dialect.Default(column.DefaultValue)); + vals.Add(_Dialect.Default(column.DefaultValue)); } } @@ -174,14 +174,14 @@ protected virtual void AddUnique(Column column, List vals) protected virtual void AddIdentityAgain(Column column, List vals) { - if (dialect.IdentityNeedsType) + if (_Dialect.IdentityNeedsType) { AddValueIfSelected(column, ColumnProperty.Identity, vals); } } protected virtual void AddPrimaryKeyNonClustered(Column column, List vals) { - if (dialect.SupportsNonClustered) + if (_Dialect.SupportsNonClustered) { AddValueIfSelected(column, ColumnProperty.PrimaryKeyNonClustered, vals); } @@ -195,7 +195,7 @@ protected virtual void AddNull(Column column, List vals) { if (!PropertySelected(column.ColumnProperty, ColumnProperty.PrimaryKey)) { - if (dialect.NeedsNullForNullableWhenAlteringTable) + if (_Dialect.NeedsNullForNullableWhenAlteringTable) { AddValueIfSelected(column, ColumnProperty.Null, vals); } @@ -204,7 +204,7 @@ protected virtual void AddNull(Column column, List vals) protected virtual void AddNotNull(Column column, List vals) { - if (!PropertySelected(column.ColumnProperty, ColumnProperty.Null) && (!PropertySelected(column.ColumnProperty, ColumnProperty.PrimaryKey) || dialect.NeedsNotNullForIdentity)) + if (!PropertySelected(column.ColumnProperty, ColumnProperty.Null) && (!PropertySelected(column.ColumnProperty, ColumnProperty.PrimaryKey) || _Dialect.NeedsNotNullForIdentity)) { AddValueIfSelected(column, ColumnProperty.NotNull, vals); } @@ -212,7 +212,7 @@ protected virtual void AddNotNull(Column column, List vals) protected virtual void AddUnsigned(Column column, List vals) { - if (dialect.IsUnsignedCompatible(column.Type)) + if (_Dialect.IsUnsignedCompatible(column.Type)) { AddValueIfSelected(column, ColumnProperty.Unsigned, vals); } @@ -220,7 +220,7 @@ protected virtual void AddUnsigned(Column column, List vals) protected virtual void AddIdentity(Column column, List vals) { - if (!dialect.IdentityNeedsType) + if (!_Dialect.IdentityNeedsType) { AddValueIfSelected(column, ColumnProperty.Identity, vals); } @@ -228,19 +228,19 @@ protected virtual void AddIdentity(Column column, List vals) protected virtual void AddType(List vals) { - vals.Add(type); + vals.Add(Type); } protected virtual void AddName(List vals) { - vals.Add(dialect.ColumnNameNeedsQuote || dialect.IsReservedWord(Name) ? QuotedName : Name); + vals.Add(_Dialect.ColumnNameNeedsQuote || _Dialect.IsReservedWord(Name) ? QuotedName : Name); } protected virtual void AddValueIfSelected(Column column, ColumnProperty property, ICollection vals) { if (PropertySelected(column.ColumnProperty, property)) { - vals.Add(dialect.SqlForProperty(property, column)); + vals.Add(_Dialect.SqlForProperty(property, column)); } } diff --git a/src/Migrator/Providers/Dialect.cs b/src/Migrator/Providers/Dialect.cs index fcf12c33..8a3fcf11 100644 --- a/src/Migrator/Providers/Dialect.cs +++ b/src/Migrator/Providers/Dialect.cs @@ -349,6 +349,26 @@ public virtual string Quote(string value) return string.Format(QuoteTemplate, value); } + public virtual string QuoteColumnNameIfRequired(string columnName) + { + if (ColumnNameNeedsQuote || IsReservedWord(columnName)) + { + return Quote(columnName); + } + + return columnName; + } + + public virtual string QuoteTableNameIfRequired(string tableName) + { + if (TableNameNeedsQuote || IsReservedWord(tableName)) + { + return Quote(tableName); + } + + return tableName; + } + public virtual string Default(object defaultValue) { if (defaultValue is string && defaultValue.ToString() == string.Empty) diff --git a/src/Migrator/Providers/Impl/Firebird/FirebirdColumnPropertiesMapper.cs b/src/Migrator/Providers/Impl/Firebird/FirebirdColumnPropertiesMapper.cs index cb007b7d..38ea3a26 100644 --- a/src/Migrator/Providers/Impl/Firebird/FirebirdColumnPropertiesMapper.cs +++ b/src/Migrator/Providers/Impl/Firebird/FirebirdColumnPropertiesMapper.cs @@ -14,7 +14,7 @@ public override void MapColumnProperties(Column column) { Name = column.Name; - indexed = PropertySelected(column.ColumnProperty, ColumnProperty.Indexed); + _Indexed = PropertySelected(column.ColumnProperty, ColumnProperty.Indexed); var vals = new List(); @@ -38,6 +38,6 @@ public override void MapColumnProperties(Column column) AddNull(column, vals); - columnSql = string.Join(" ", vals.ToArray()); + _ColumnSql = string.Join(" ", vals.ToArray()); } } diff --git a/src/Migrator/Providers/Impl/Oracle/Data/Interfaces/IOracleSystemDataLoader.cs b/src/Migrator/Providers/Impl/Oracle/Data/Interfaces/IOracleSystemDataLoader.cs new file mode 100644 index 00000000..2e94583a --- /dev/null +++ b/src/Migrator/Providers/Impl/Oracle/Data/Interfaces/IOracleSystemDataLoader.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; +using DotNetProjects.Migrator.Providers.Impl.Oracle.Models; +using DotNetProjects.Migrator.Providers.Models; + +namespace DotNetProjects.Migrator.Providers.Impl.Oracle.Data.Interfaces; + +public interface IOracleSystemDataLoader +{ + /// + /// Gets s for given table name. + /// + /// + /// + List GetForeignKeyConstraintItems(string tableName); + + /// + /// Gets the USER_TAB_IDENTITY_COLS records for the given table name. + /// + /// + /// + List GetUserTabIdentityCols(string tableName); + + /// + /// Gets the primary key items from user_constraints and user_cons_columns + /// + /// + /// + List GetPrimaryKeyItems(string tableName); +} \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/Oracle/Data/OracleSystemDataLoader.cs b/src/Migrator/Providers/Impl/Oracle/Data/OracleSystemDataLoader.cs new file mode 100644 index 00000000..b2d1b490 --- /dev/null +++ b/src/Migrator/Providers/Impl/Oracle/Data/OracleSystemDataLoader.cs @@ -0,0 +1,137 @@ +using System.Collections.Generic; +using System.Text; +using DotNetProjects.Migrator.Providers.Impl.Oracle.Data.Interfaces; +using DotNetProjects.Migrator.Providers.Impl.Oracle.Interfaces; +using DotNetProjects.Migrator.Providers.Impl.Oracle.Models; +using DotNetProjects.Migrator.Providers.Models; + +namespace DotNetProjects.Migrator.Providers.Impl.Oracle.Data; + +public class OracleSystemDataLoader(IOracleTransformationProvider oracleTransformationProvider) : IOracleSystemDataLoader +{ + private readonly IOracleTransformationProvider _oracleTransformationProvider = oracleTransformationProvider; + + public List GetUserTabIdentityCols(string tableName) + { + List userTabIdentityCols = []; + + var tableNameQuoted = _oracleTransformationProvider.QuoteTableNameIfRequired(tableName); + + var sql = $"SELECT TABLE_NAME, COLUMN_NAME, GENERATION_TYPE, SEQUENCE_NAME FROM USER_TAB_IDENTITY_COLS WHERE TABLE_NAME = '{tableNameQuoted.ToUpperInvariant()}'"; + + using var cmd = _oracleTransformationProvider.CreateCommand(); + using var reader = _oracleTransformationProvider.ExecuteQuery(cmd, sql); + + while (reader.Read()) + { + var tableNameOrdinal = reader.GetOrdinal("TABLE_NAME"); + var columnNameOrdinal = reader.GetOrdinal("COLUMN_NAME"); + var generationTypeOrdinal = reader.GetOrdinal("GENERATION_TYPE"); + var sequenceNameOrdinal = reader.GetOrdinal("SEQUENCE_NAME"); + + var userTablIdentityColsItem = new UserTabIdentityCols + { + ColumnName = reader.GetString(columnNameOrdinal), + GenerationType = reader.GetString(generationTypeOrdinal), + SequenceName = reader.GetString(sequenceNameOrdinal), + TableName = reader.GetString(tableNameOrdinal), + }; + + userTabIdentityCols.Add(userTablIdentityColsItem); + } + + return userTabIdentityCols; + } + + public List GetForeignKeyConstraintItems(string tableName) + { + var tableNameQuoted = _oracleTransformationProvider.QuoteTableNameIfRequired(tableName); + + var sb = new StringBuilder(); + sb.AppendLine("SELECT"); + sb.AppendLine(" a.OWNER AS TABLE_SCHEMA,"); + sb.AppendLine(" c.CONSTRAINT_NAME AS FK_KEY,"); + sb.AppendLine(" a.TABLE_NAME AS CHILD_TABLE,"); + sb.AppendLine(" a.COLUMN_NAME AS CHILD_COLUMN,"); + sb.AppendLine(" c_pk.TABLE_NAME AS PARENT_TABLE,"); + sb.AppendLine(" col_pk.COLUMN_NAME AS PARENT_COLUMN"); + sb.AppendLine("FROM "); + sb.AppendLine(" USER_CONS_COLUMNS a "); + sb.AppendLine("JOIN USER_CONSTRAINTS c"); + sb.AppendLine(" ON a.owner = c.owner AND a.CONSTRAINT_NAME = c.CONSTRAINT_NAME"); + sb.AppendLine("JOIN USER_CONSTRAINTS c_pk"); + sb.AppendLine(" ON c.R_OWNER = c_pk.OWNER AND c.R_CONSTRAINT_NAME = c_pk.CONSTRAINT_NAME"); + sb.AppendLine("JOIN USER_CONS_COLUMNS col_pk"); + sb.AppendLine(" ON c_pk.CONSTRAINT_NAME = col_pk.CONSTRAINT_NAME AND c_pk.OWNER = col_pk.OWNER AND a.POSITION = col_pk.POSITION"); + sb.AppendLine($"WHERE LOWER(a.TABLE_NAME) = LOWER('{tableNameQuoted}') AND c.CONSTRAINT_TYPE = 'R'"); + sb.AppendLine("ORDER BY a.POSITION"); + + var sql = sb.ToString(); + List foreignKeyConstraintItems = []; + + using var cmd = _oracleTransformationProvider.CreateCommand(); + using var reader = _oracleTransformationProvider.ExecuteQuery(cmd, sql); + + while (reader.Read()) + { + var constraintItem = new ForeignKeyConstraintItem + { + SchemaName = reader.GetString(reader.GetOrdinal("TABLE_SCHEMA")), + ForeignKeyName = reader.GetString(reader.GetOrdinal("FK_KEY")), + ChildTableName = reader.GetString(reader.GetOrdinal("CHILD_TABLE")), + ChildColumnName = reader.GetString(reader.GetOrdinal("CHILD_COLUMN")), + ParentTableName = reader.GetString(reader.GetOrdinal("PARENT_TABLE")), + ParentColumnName = reader.GetString(reader.GetOrdinal("PARENT_COLUMN")) + }; + + foreignKeyConstraintItems.Add(constraintItem); + } + + return foreignKeyConstraintItems; + } + + public List GetPrimaryKeyItems(string tableName) + { + var tableNameQuoted = _oracleTransformationProvider.QuoteTableNameIfRequired(tableName); + + var sql = $@" + SELECT + ucc.TABLE_NAME, + ucc.COLUMN_NAME, + ucc.POSITION, + uc.CONSTRAINT_NAME, + uc.STATUS + FROM + USER_CONSTRAINTS uc + JOIN + USER_CONS_COLUMNS ucc + ON uc.CONSTRAINT_NAME = ucc.CONSTRAINT_NAME + WHERE + uc.CONSTRAINT_TYPE = 'P' + AND ucc.TABLE_NAME = '{tableNameQuoted.ToUpperInvariant()}' + ORDER BY + ucc.POSITION + "; + + List primaryKeyItems = []; + + using var cmd = _oracleTransformationProvider.CreateCommand(); + using var reader = _oracleTransformationProvider.ExecuteQuery(cmd, sql); + + while (reader.Read()) + { + var constraintItem = new PrimaryKeyItem + { + TableName = reader.GetString(reader.GetOrdinal("TABLE_NAME")), + ColumnName = reader.GetString(reader.GetOrdinal("COLUMN_NAME")), + Position = reader.GetInt32(reader.GetOrdinal("POSITION")), + ConstraintName = reader.GetString(reader.GetOrdinal("CONSTRAINT_NAME")), + Status = reader.GetString(reader.GetOrdinal("STATUS")) + }; + + primaryKeyItems.Add(constraintItem); + } + + return primaryKeyItems; + } +} \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/Oracle/Interfaces/IOracleTransformationProvider.cs b/src/Migrator/Providers/Impl/Oracle/Interfaces/IOracleTransformationProvider.cs new file mode 100644 index 00000000..6b2aa91e --- /dev/null +++ b/src/Migrator/Providers/Impl/Oracle/Interfaces/IOracleTransformationProvider.cs @@ -0,0 +1,7 @@ +using DotNetProjects.Migrator.Framework; + +namespace DotNetProjects.Migrator.Providers.Impl.Oracle.Interfaces; + +public interface IOracleTransformationProvider : ITransformationProvider +{ +} \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/Oracle/Models/AllTablIdentityCols.cs b/src/Migrator/Providers/Impl/Oracle/Models/AllTablIdentityCols.cs new file mode 100644 index 00000000..b419a04d --- /dev/null +++ b/src/Migrator/Providers/Impl/Oracle/Models/AllTablIdentityCols.cs @@ -0,0 +1,27 @@ +namespace DotNetProjects.Migrator.Providers.Impl.Oracle.Models; + +/// +/// Represents USER_TAB_IDENTITY_COLS partly +/// +public class UserTabIdentityCols +{ + /// + /// Gets or sets the name of the identity column. Column: COLUMN_NAME + /// + public string ColumnName { get; set; } + + /// + /// Gets or sets the generation type of the identity column. Possible values are ALWAYS or BY DEFAULT. Column: GENERATION_TYPE + /// + public string GenerationType { get; set; } + + /// + /// Gets or sets the name of the sequence associated with the identity column. Column: SEQUENCE_NAME + /// + public string SequenceName { get; set; } + + /// + /// Gets or sets the name of the table. Column: TABLE_NAME + /// + public string TableName { get; set; } +} \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/Oracle/Models/PrimaryKeyItem.cs b/src/Migrator/Providers/Impl/Oracle/Models/PrimaryKeyItem.cs new file mode 100644 index 00000000..754a1d62 --- /dev/null +++ b/src/Migrator/Providers/Impl/Oracle/Models/PrimaryKeyItem.cs @@ -0,0 +1,29 @@ +namespace DotNetProjects.Migrator.Providers.Impl.Oracle.Models; + +public class PrimaryKeyItem +{ + /// + /// Gets or sets the table name USER_CONS_COLUMNS.TABLE_NAME + /// + public string TableName { get; set; } + + /// + /// Gets or sets the column name USER_CONS_COLUMNS.COLUMN_NAME + /// + public string ColumnName { get; set; } + + /// + /// Gets or sets USER_CONS_COLUMNS.POSITION + /// + public int Position { get; set; } + + /// + /// Gets or sets USER_CONSTRAINTS.STATUS Enforcement status of the constraint: ENABLED, DISABLED + /// + public string Status { get; set; } + + /// + /// Gets or sets the USER_CONSTRAINTS.CONSTRAINT_NAME + /// + public string ConstraintName { get; set; } +} \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/Oracle/OracleColumnPropertiesMapper.cs b/src/Migrator/Providers/Impl/Oracle/OracleColumnPropertiesMapper.cs index cfa6c023..56f672d4 100644 --- a/src/Migrator/Providers/Impl/Oracle/OracleColumnPropertiesMapper.cs +++ b/src/Migrator/Providers/Impl/Oracle/OracleColumnPropertiesMapper.cs @@ -5,7 +5,7 @@ namespace DotNetProjects.Migrator.Providers.Impl.Oracle; public class OracleColumnPropertiesMapper : ColumnPropertiesMapper { - public OracleColumnPropertiesMapper(Dialect dialect, string type) : base(dialect, type) + public OracleColumnPropertiesMapper(Dialect dialect, string typeString) : base(dialect, typeString) { } @@ -13,7 +13,7 @@ public override void MapColumnProperties(Column column) { Name = column.Name; - indexed = PropertySelected(column.ColumnProperty, ColumnProperty.Indexed); + _Indexed = PropertySelected(column.ColumnProperty, ColumnProperty.Indexed); var vals = new List(); @@ -42,6 +42,6 @@ public override void MapColumnProperties(Column column) AddNull(column, vals); - columnSql = string.Join(" ", vals.ToArray()); + _ColumnSql = string.Join(" ", vals.ToArray()); } } \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/Oracle/OracleDialect.cs b/src/Migrator/Providers/Impl/Oracle/OracleDialect.cs index 0af6fd33..4d5a2b32 100644 --- a/src/Migrator/Providers/Impl/Oracle/OracleDialect.cs +++ b/src/Migrator/Providers/Impl/Oracle/OracleDialect.cs @@ -50,8 +50,10 @@ public OracleDialect() RegisterColumnType(DbType.Guid, "RAW(16)"); RegisterColumnType(MigratorDbType.Interval, "interval day (9) to second (9)"); + RegisterProperty(ColumnProperty.Identity, "GENERATED ALWAYS AS IDENTITY"); + // the original Migrator.Net code had this, but it's a bad idea - when - // apply a "null" migration to a "not-null" field, it just leaves it as "not-null" and silent fails + // apply a "null" migration to a "not-null" field, it just leaves it as "not-null" and it silently fails // because Oracle doesn't consider ALTER TABLE MODIFY (column ) as being a request to make the field null. //RegisterProperty(ColumnProperty.Null, String.Empty); @@ -61,34 +63,14 @@ public OracleDialect() // in Oracle, this: ALTER TABLE EXTERNALSYSTEMREFERENCES MODIFY (TestScriptId RAW(16)) will no make the column nullable, it just leaves it at it's current null/not-null state - public override int MaxFieldNameLength - { - get { return 30; } - } - - public override int MaxKeyLength - { - get { return 767; } - } - - public override bool NeedsNullForNullableWhenAlteringTable - { - get { return true; } - } - - public override bool ColumnNameNeedsQuote - { - get { return false; } - } - public override bool ConstraintNameNeedsQuote - { - get { return false; } - } - public override bool TableNameNeedsQuote - { - get { return false; } - } + public override bool ColumnNameNeedsQuote => false; + public override bool ConstraintNameNeedsQuote => false; + public override bool IdentityNeedsType => false; + public override bool NeedsNullForNullableWhenAlteringTable => true; + public override bool TableNameNeedsQuote => false; + public override int MaxFieldNameLength => 30; + public override int MaxKeyLength => 767; public override ITransformationProvider GetTransformationProvider(Dialect dialect, string connectionString, string defaultSchema, string scope, string providerName) { @@ -104,19 +86,15 @@ public override ITransformationProvider GetTransformationProvider(Dialect dialec public override ColumnPropertiesMapper GetColumnMapper(Column column) { - var type = column.Size > 0 ? GetTypeName(column.Type, column.Size) : GetTypeName(column.Type); + var typeString = column.Size > 0 ? GetTypeName(column.Type, column.Size) : GetTypeName(column.Type); if (column.Precision.HasValue || column.Scale.HasValue) { - type = GetTypeNameParametrized(column.Type, column.Size, column.Precision ?? 0, column.Scale ?? 0); + typeString = GetTypeNameParametrized(column.Type, column.Size, column.Precision ?? 0, column.Scale ?? 0); } - if (!IdentityNeedsType && column.IsIdentity) - { - type = string.Empty; - } - return new OracleColumnPropertiesMapper(this, type); + return new OracleColumnPropertiesMapper(this, typeString); } public override string Default(object defaultValue) diff --git a/src/Migrator/Providers/Impl/Oracle/OracleTransformationProvider.cs b/src/Migrator/Providers/Impl/Oracle/OracleTransformationProvider.cs index bcd0016f..aa53050e 100644 --- a/src/Migrator/Providers/Impl/Oracle/OracleTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/Oracle/OracleTransformationProvider.cs @@ -1,7 +1,9 @@ using DotNetProjects.Migrator.Framework; using DotNetProjects.Migrator.Framework.Models; +using DotNetProjects.Migrator.Providers.Impl.Oracle.Data; +using DotNetProjects.Migrator.Providers.Impl.Oracle.Data.Interfaces; +using DotNetProjects.Migrator.Providers.Impl.Oracle.Interfaces; using DotNetProjects.Migrator.Providers.Impl.Oracle.Models; -using DotNetProjects.Migrator.Providers.Models; using DotNetProjects.Migrator.Providers.Models.Indexes; using System; using System.Collections.Generic; @@ -15,19 +17,22 @@ namespace DotNetProjects.Migrator.Providers.Impl.Oracle; -public class OracleTransformationProvider : TransformationProvider +public class OracleTransformationProvider : TransformationProvider, IOracleTransformationProvider { + private IOracleSystemDataLoader _oracleSystemDataLoader; public const string TemporaryColumnName = "TEMPCOL"; public OracleTransformationProvider(Dialect dialect, string connectionString, string defaultSchema, string scope, string providerName) : base(dialect, connectionString, defaultSchema, scope) { CreateConnection(providerName); + Initialize(); } public OracleTransformationProvider(Dialect dialect, IDbConnection connection, string defaultSchema, string scope, string providerName) : base(dialect, connection, defaultSchema, scope) { + Initialize(); } protected virtual void CreateConnection(string providerName) @@ -54,46 +59,7 @@ public override void DropDatabases(string databaseName) public override ForeignKeyConstraint[] GetForeignKeyConstraints(string table) { var constraints = new List(); - var sb = new StringBuilder(); - sb.AppendLine("SELECT"); - sb.AppendLine(" a.OWNER AS TABLE_SCHEMA,"); - sb.AppendLine(" c.CONSTRAINT_NAME AS FK_KEY,"); - sb.AppendLine(" a.TABLE_NAME AS CHILD_TABLE,"); - sb.AppendLine(" a.COLUMN_NAME AS CHILD_COLUMN,"); - sb.AppendLine(" c_pk.TABLE_NAME AS PARENT_TABLE,"); - sb.AppendLine(" col_pk.COLUMN_NAME AS PARENT_COLUMN"); - sb.AppendLine("FROM "); - sb.AppendLine(" ALL_CONS_COLUMNS a "); - sb.AppendLine("JOIN ALL_CONSTRAINTS c"); - sb.AppendLine(" ON a.owner = c.owner AND a.CONSTRAINT_NAME = c.CONSTRAINT_NAME"); - sb.AppendLine("JOIN ALL_CONSTRAINTS c_pk"); - sb.AppendLine(" ON c.R_OWNER = c_pk.OWNER AND c.R_CONSTRAINT_NAME = c_pk.CONSTRAINT_NAME"); - sb.AppendLine("JOIN ALL_CONS_COLUMNS col_pk"); - sb.AppendLine(" ON c_pk.CONSTRAINT_NAME = col_pk.CONSTRAINT_NAME AND c_pk.OWNER = col_pk.OWNER AND a.POSITION = col_pk.POSITION"); - sb.AppendLine($"WHERE LOWER(a.TABLE_NAME) = LOWER('{table}') AND c.CONSTRAINT_TYPE = 'R'"); - sb.AppendLine("ORDER BY a.POSITION"); - - var sql = sb.ToString(); - List foreignKeyConstraintItems = []; - - using (var cmd = CreateCommand()) - using (var reader = ExecuteQuery(cmd, sql)) - { - while (reader.Read()) - { - var constraintItem = new ForeignKeyConstraintItem - { - SchemaName = reader.GetString(reader.GetOrdinal("TABLE_SCHEMA")), - ForeignKeyName = reader.GetString(reader.GetOrdinal("FK_KEY")), - ChildTableName = reader.GetString(reader.GetOrdinal("CHILD_TABLE")), - ChildColumnName = reader.GetString(reader.GetOrdinal("CHILD_COLUMN")), - ParentTableName = reader.GetString(reader.GetOrdinal("PARENT_TABLE")), - ParentColumnName = reader.GetString(reader.GetOrdinal("PARENT_COLUMN")) - }; - - foreignKeyConstraintItems.Add(constraintItem); - } - } + var foreignKeyConstraintItems = _oracleSystemDataLoader.GetForeignKeyConstraintItems(table); var schemaChildTableGroups = foreignKeyConstraintItems.GroupBy(x => new { x.SchemaName, x.ChildTableName }).Count(); @@ -266,7 +232,7 @@ public override void ChangeColumn(string table, Column column) if (((existingColumn.ColumnProperty & ColumnProperty.NotNull) == ColumnProperty.NotNull) && ((column.ColumnProperty & ColumnProperty.NotNull) == ColumnProperty.NotNull)) { - // was not null, and is being change to not-null - drop the not-null all together + // was not null, and is being change to not-null - drop the not-null all together column.ColumnProperty = column.ColumnProperty & ~ColumnProperty.NotNull; } else if @@ -500,6 +466,9 @@ public override Column[] GetColumns(string table) stringBuilder2.AppendLine(" data_default VARCHAR2(4000) PATH 'DATA_DEFAULT'"); stringBuilder2.AppendLine(") x"); + var userTabIdentityCols = _oracleSystemDataLoader.GetUserTabIdentityCols(tableName: table); + var primaryKeyItems = _oracleSystemDataLoader.GetPrimaryKeyItems(tableName: table); + List userTabColumns = []; using (var cmd = CreateCommand()) @@ -549,6 +518,22 @@ public override Column[] GetColumns(string table) ColumnProperty = isNullable ? ColumnProperty.Null : ColumnProperty.NotNull }; + var isIdentity = userTabIdentityCols.Any(x => x.ColumnName.Equals(columnName, StringComparison.OrdinalIgnoreCase)); + var isPrimaryKey = primaryKeyItems.Any(x => x.ColumnName.Equals(columnName, StringComparison.OrdinalIgnoreCase)); + + if (isIdentity && isPrimaryKey) + { + column.ColumnProperty = column.ColumnProperty.Set(ColumnProperty.PrimaryKeyWithIdentity); + } + else if (isIdentity) + { + column.ColumnProperty.Set(ColumnProperty.Identity); + } + else if (isPrimaryKey) + { + column.ColumnProperty.Set(ColumnProperty.PrimaryKey); + } + // Oracle does not have unsigned types. All NUMBER types can hold positive or negative values so we do not return DbType.UIntX types. if (dataTypeString.StartsWith("NUMBER") || dataTypeString.StartsWith("FLOAT")) { @@ -645,7 +630,8 @@ public override Column[] GetColumns(string table) throw new NotImplementedException($"The data type '{dataTypeString}' is not implemented yet. Please file an issue."); } - if (!string.IsNullOrWhiteSpace(dataDefaultString)) + // dataDefaultString contains ISEQ$$ if the column is an identity column + if (!string.IsNullOrWhiteSpace(dataDefaultString) && !dataDefaultString.Contains("ISEQ$$") && !dataDefaultString.Contains(".nextval")) { // This is only necessary because older versions of this migrator added single quotes for numerics. var singleQuoteStrippedString = dataDefaultString.Replace("'", ""); @@ -887,6 +873,7 @@ public override void RemoveColumnDefaultValue(string table, string column) public override void AddTable(string name, params IDbField[] fields) { GuardAgainstMaximumIdentifierLengthForOracle(name); + name = QuoteTableNameIfRequired(name); var columns = fields.Where(x => x is Column).Cast().ToArray(); @@ -898,29 +885,27 @@ public override void AddTable(string name, params IDbField[] fields) if (columns.Any(c => c.ColumnProperty == ColumnProperty.PrimaryKeyWithIdentity || (c.ColumnProperty.HasFlag(ColumnProperty.Identity) && c.ColumnProperty.HasFlag(ColumnProperty.PrimaryKey)))) { - var identityColumn = columns.First(c => c.ColumnProperty == ColumnProperty.PrimaryKeyWithIdentity || - (c.ColumnProperty.HasFlag(ColumnProperty.Identity) && c.ColumnProperty.HasFlag(ColumnProperty.PrimaryKey))); + var identityColumn = columns.First(x => x.ColumnProperty.HasFlag(ColumnProperty.Identity) && x.ColumnProperty.HasFlag(ColumnProperty.PrimaryKey)); - var seqTName = name.Length > 21 ? name.Substring(0, 21) : name; - if (seqTName.EndsWith("_")) - { - seqTName = seqTName.Substring(0, seqTName.Length - 1); - } + List allowedIdentityDbTypes = [DbType.Int16, DbType.Int32, DbType.Int64]; - // Create a sequence for the table - using (var cmd = CreateCommand()) + if (!allowedIdentityDbTypes.Contains(identityColumn.Type)) { - ExecuteQuery(cmd, string.Format("CREATE SEQUENCE {0}_SEQUENCE NOCACHE", seqTName)); + throw new MigrationException($"Identity columns can only be used with {nameof(DbType.Int16)}, {nameof(DbType.Int32)} and {nameof(DbType.Int64)}"); } - // Create identity trigger (This all has to be in one line (no whitespace), I learned the hard way :) ) - using (var cmd = CreateCommand()) - { - ExecuteQuery(cmd, string.Format( - @"CREATE OR REPLACE TRIGGER {0}_TRIGGER BEFORE INSERT ON {1} FOR EACH ROW BEGIN SELECT {0}_SEQUENCE.NEXTVAL INTO :NEW.{2} FROM DUAL; END;", seqTName, name, identityColumn.Name)); - } + var identityColumnNameQuoted = QuoteColumnNameIfRequired(identityColumn.Name); + + using var cmd = CreateCommand(); + // We use ALWAYS in order to prevent sequence problems in cases of misuse of the column by an unexperienced user. Inserting data will result in an exception. + ExecuteQuery(cmd, $"ALTER TABLE {name} MODIFY {identityColumnNameQuoted} GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1 NOCACHE NOCYCLE)"); + } + else if (columns.Any(x => x.ColumnProperty.HasFlag(ColumnProperty.Identity) && !x.ColumnProperty.HasFlag(ColumnProperty.PrimaryKey))) + { + throw new MigrationException("Identity without Primary is currently not supported by this migrator"); } } + public override void RemoveTable(string name) { base.RemoveTable(name); @@ -1118,4 +1103,9 @@ public override string Concatenate(params string[] strings) { return string.Join(" || ", strings); } + + private void Initialize() + { + _oracleSystemDataLoader = new OracleSystemDataLoader(this); + } } diff --git a/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs b/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs index b6abb333..7571a6e7 100644 --- a/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs @@ -359,7 +359,7 @@ public override void ChangeColumn(string table, Column column) var mapper = _dialect.GetAndMapColumnProperties(column); - var change1 = string.Format("{0} TYPE {1}", QuoteColumnNameIfRequired(mapper.Name), mapper.type); + var change1 = string.Format("{0} TYPE {1}", QuoteColumnNameIfRequired(mapper.Name), mapper.Type); if ((oldColumn.MigratorDbType == MigratorDbType.Int16 || oldColumn.MigratorDbType == MigratorDbType.Int32 || oldColumn.MigratorDbType == MigratorDbType.Int64 || oldColumn.MigratorDbType == MigratorDbType.Decimal) && column.MigratorDbType == MigratorDbType.Boolean) { diff --git a/src/Migrator/Providers/Impl/SQLite/SQLiteColumnPropertiesMapper.cs b/src/Migrator/Providers/Impl/SQLite/SQLiteColumnPropertiesMapper.cs index a645f025..f8c3d72d 100644 --- a/src/Migrator/Providers/Impl/SQLite/SQLiteColumnPropertiesMapper.cs +++ b/src/Migrator/Providers/Impl/SQLite/SQLiteColumnPropertiesMapper.cs @@ -41,6 +41,6 @@ protected override void AddNotNull(Column column, List vals) protected virtual void AddValueIfSelected(Column column, ColumnProperty property, ICollection vals) { - vals.Add(dialect.SqlForProperty(property, column)); + vals.Add(_Dialect.SqlForProperty(property, column)); } } \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/SQLite/SQLiteTransformationProvider.cs b/src/Migrator/Providers/Impl/SQLite/SQLiteTransformationProvider.cs index 83d5aca7..882452c6 100644 --- a/src/Migrator/Providers/Impl/SQLite/SQLiteTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/SQLite/SQLiteTransformationProvider.cs @@ -1304,6 +1304,7 @@ public override Index[] GetIndexes(string table) { if (afterWhereRegex.Match(script) is Match match && match.Success) { + // We cannot use GeneratedRegexAttribute due to old .NET version var andSplitted = Regex.Split(match.Value, " AND "); var filterSingleStrings = andSplitted From 364bcb4d5e73ad1902072ecb301cd6b1b5712212 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Wed, 29 Oct 2025 08:05:41 +0100 Subject: [PATCH 03/19] Moved IndexItems to loader --- .../Interfaces/IOracleSystemDataLoader.cs | 8 ++ .../Oracle/Data/OracleSystemDataLoader.cs | 93 +++++++++++++++---- .../Oracle/OracleTransformationProvider.cs | 69 +++----------- .../PostgreSQLTransformationProvider.cs | 12 ++- .../Providers/TransformationProvider.cs | 26 +----- 5 files changed, 109 insertions(+), 99 deletions(-) diff --git a/src/Migrator/Providers/Impl/Oracle/Data/Interfaces/IOracleSystemDataLoader.cs b/src/Migrator/Providers/Impl/Oracle/Data/Interfaces/IOracleSystemDataLoader.cs index 2e94583a..80d53dff 100644 --- a/src/Migrator/Providers/Impl/Oracle/Data/Interfaces/IOracleSystemDataLoader.cs +++ b/src/Migrator/Providers/Impl/Oracle/Data/Interfaces/IOracleSystemDataLoader.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using DotNetProjects.Migrator.Providers.Impl.Oracle.Models; using DotNetProjects.Migrator.Providers.Models; +using DotNetProjects.Migrator.Providers.Models.Indexes; namespace DotNetProjects.Migrator.Providers.Impl.Oracle.Data.Interfaces; @@ -26,4 +27,11 @@ public interface IOracleSystemDataLoader /// /// List GetPrimaryKeyItems(string tableName); + + /// + /// Gets index items from USER_INDEXES, USER_IND_COLUMNS and USER_CONSTRAINTS + /// + /// + /// + List GetIndexItems(string tableName); } \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/Oracle/Data/OracleSystemDataLoader.cs b/src/Migrator/Providers/Impl/Oracle/Data/OracleSystemDataLoader.cs index b2d1b490..287389a9 100644 --- a/src/Migrator/Providers/Impl/Oracle/Data/OracleSystemDataLoader.cs +++ b/src/Migrator/Providers/Impl/Oracle/Data/OracleSystemDataLoader.cs @@ -4,6 +4,7 @@ using DotNetProjects.Migrator.Providers.Impl.Oracle.Interfaces; using DotNetProjects.Migrator.Providers.Impl.Oracle.Models; using DotNetProjects.Migrator.Providers.Models; +using DotNetProjects.Migrator.Providers.Models.Indexes; namespace DotNetProjects.Migrator.Providers.Impl.Oracle.Data; @@ -95,22 +96,22 @@ public List GetPrimaryKeyItems(string tableName) var tableNameQuoted = _oracleTransformationProvider.QuoteTableNameIfRequired(tableName); var sql = $@" - SELECT - ucc.TABLE_NAME, - ucc.COLUMN_NAME, - ucc.POSITION, - uc.CONSTRAINT_NAME, - uc.STATUS - FROM - USER_CONSTRAINTS uc - JOIN - USER_CONS_COLUMNS ucc - ON uc.CONSTRAINT_NAME = ucc.CONSTRAINT_NAME - WHERE - uc.CONSTRAINT_TYPE = 'P' - AND ucc.TABLE_NAME = '{tableNameQuoted.ToUpperInvariant()}' - ORDER BY - ucc.POSITION + SELECT + ucc.TABLE_NAME, + ucc.COLUMN_NAME, + ucc.POSITION, + uc.CONSTRAINT_NAME, + uc.STATUS + FROM + USER_CONSTRAINTS uc + JOIN + USER_CONS_COLUMNS ucc + ON uc.CONSTRAINT_NAME = ucc.CONSTRAINT_NAME + WHERE + uc.CONSTRAINT_TYPE = 'P' + AND ucc.TABLE_NAME = '{tableNameQuoted.ToUpperInvariant()}' + ORDER BY + ucc.POSITION "; List primaryKeyItems = []; @@ -134,4 +135,64 @@ ORDER BY return primaryKeyItems; } + + public List GetIndexItems(string tableName) + { + var tableNameQuoted = _oracleTransformationProvider.QuoteTableNameIfRequired(tableName); + + var sql = @$" + SELECT + i.table_name, + i.index_name, + i.uniqueness, + ic.column_position, + ic.column_name, + CASE WHEN c.constraint_type = 'P' THEN 'YES' ELSE 'NO' END AS is_primary_key, + CASE WHEN c.constraint_type = 'U' THEN 'YES' ELSE 'NO' END AS is_unique_key + FROM + user_indexes i + JOIN + user_ind_columns ic ON i.index_name = ic.index_name AND + i.table_name = ic.table_name + LEFT JOIN + user_constraints c ON i.index_name = c.index_name AND + i.table_name = c.table_name + WHERE + UPPER(i.table_name) = '{tableNameQuoted.ToUpperInvariant()}' + -- AND + -- i.index_type = 'NORMAL' + ORDER BY + i.table_name, i.index_name, ic.column_position"; + + List indexItems = []; + + using var cmd = _oracleTransformationProvider.CreateCommand(); + using var reader = _oracleTransformationProvider.ExecuteQuery(cmd, sql); + + while (reader.Read()) + { + var tableNameOrdinal = reader.GetOrdinal("table_name"); + var indexNameOrdinal = reader.GetOrdinal("index_name"); + var uniquenessOrdinal = reader.GetOrdinal("uniqueness"); + var columnPositionOrdinal = reader.GetOrdinal("column_position"); + var columnNameOrdinal = reader.GetOrdinal("column_name"); + var isPrimaryKeyOrdinal = reader.GetOrdinal("is_primary_key"); + var isUniqueConstraintOrdinal = reader.GetOrdinal("is_unique_key"); + + var indexItem = new IndexItem + { + ColumnName = reader.GetString(columnNameOrdinal), + ColumnOrder = reader.GetInt32(columnPositionOrdinal), + Name = reader.GetString(indexNameOrdinal), + PrimaryKey = reader.GetString(isPrimaryKeyOrdinal) == "YES", + TableName = reader.GetString(tableNameOrdinal), + Unique = reader.GetString(uniquenessOrdinal) == "UNIQUE", + UniqueConstraint = reader.GetString(isUniqueConstraintOrdinal) == "YES" + }; + + indexItems.Add(indexItem); + } + + return indexItems; + } } \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/Oracle/OracleTransformationProvider.cs b/src/Migrator/Providers/Impl/Oracle/OracleTransformationProvider.cs index aa53050e..afdc8ab9 100644 --- a/src/Migrator/Providers/Impl/Oracle/OracleTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/Oracle/OracleTransformationProvider.cs @@ -190,11 +190,11 @@ private void GuardAgainstMaximumIdentifierLengthForOracle(string name) if (utf8Bytes.Length > 128) { - throw new MigrationException($"The name '{name}' is {utf8Bytes.Length} bytes in length, but maximum length for Oracle identifiers is 128 bytes for Oracle versions 12.1+."); + throw new MigrationException($"The name '{name}' is {utf8Bytes.Length} bytes in length, but maximum length for Oracle identifiers is 128 bytes for Oracle versions 12.1+."); } } - protected override string getPrimaryKeyname(string tableName) + protected override string GetPrimaryKeyname(string tableName) { return tableName.Length > 27 ? "PK_" + tableName.Substring(0, 27) : "PK_" + tableName; } @@ -301,16 +301,17 @@ public override void ChangeColumn(string table, string sqlColumn) { if (string.IsNullOrEmpty(table)) { - throw new ArgumentNullException("table"); + throw new ArgumentNullException(nameof(table)); } if (string.IsNullOrEmpty(table)) { - throw new ArgumentNullException("sqlColumn"); + throw new ArgumentNullException(nameof(sqlColumn)); } table = QuoteTableNameIfRequired(table); sqlColumn = QuoteColumnNameIfRequired(sqlColumn); + ExecuteNonQuery(string.Format("ALTER TABLE {0} MODIFY {1}", table, sqlColumn)); } @@ -319,6 +320,7 @@ public override void AddColumn(string table, string sqlColumn) GuardAgainstMaximumIdentifierLengthForOracle(table); table = QuoteTableNameIfRequired(table); sqlColumn = QuoteColumnNameIfRequired(sqlColumn); + ExecuteNonQuery(string.Format("ALTER TABLE {0} ADD {1}", table, sqlColumn)); } @@ -365,8 +367,10 @@ public override bool ConstraintExists(string table, string name) string.Format( "SELECT COUNT(constraint_name) FROM user_constraints WHERE lower(constraint_name) = '{0}' AND lower(table_name) = '{1}'", name.ToLower(), table.ToLower()); + Logger.Log(sql); var scalar = ExecuteScalar(sql); + return Convert.ToInt32(scalar) == 1; } @@ -909,6 +913,7 @@ public override void AddTable(string name, params IDbField[] fields) public override void RemoveTable(string name) { base.RemoveTable(name); + try { using var cmd = CreateCommand(); @@ -916,9 +921,10 @@ public override void RemoveTable(string name) } catch (Exception) { - // swallow this because sequence may not have originally existed. + // swallow this because sequence may not have existed. } } + private void GuardAgainstMaximumColumnNameLengthForOracle(string name, Column[] columns) { foreach (var column in columns) @@ -1016,58 +1022,7 @@ private string SchemaInfoTableName public override Index[] GetIndexes(string table) { - var sql = @$"SELECT - i.table_name, - i.index_name, - i.uniqueness, - ic.column_position, - ic.column_name, - CASE WHEN c.constraint_type = 'P' THEN 'YES' ELSE 'NO' END AS is_primary_key, - CASE WHEN c.constraint_type = 'U' THEN 'YES' ELSE 'NO' END AS is_unique_key - FROM - user_indexes i - JOIN - user_ind_columns ic ON i.index_name = ic.index_name AND - i.table_name = ic.table_name - LEFT JOIN - user_constraints c ON i.index_name = c.index_name AND - i.table_name = c.table_name - WHERE - UPPER(i.table_name) = '{table.ToUpperInvariant()}' - -- AND - -- i.index_type = 'NORMAL' - ORDER BY - i.table_name, i.index_name, ic.column_position"; - - List indexItems = []; - - using (var cmd = CreateCommand()) - using (var reader = ExecuteQuery(cmd, sql)) - { - while (reader.Read()) - { - var tableNameOrdinal = reader.GetOrdinal("table_name"); - var indexNameOrdinal = reader.GetOrdinal("index_name"); - var uniquenessOrdinal = reader.GetOrdinal("uniqueness"); - var columnPositionOrdinal = reader.GetOrdinal("column_position"); - var columnNameOrdinal = reader.GetOrdinal("column_name"); - var isPrimaryKeyOrdinal = reader.GetOrdinal("is_primary_key"); - var isUniqueConstraintOrdinal = reader.GetOrdinal("is_unique_key"); - - var indexItem = new IndexItem - { - ColumnName = reader.GetString(columnNameOrdinal), - ColumnOrder = reader.GetInt32(columnPositionOrdinal), - Name = reader.GetString(indexNameOrdinal), - PrimaryKey = reader.GetString(isPrimaryKeyOrdinal) == "YES", - TableName = reader.GetString(tableNameOrdinal), - Unique = reader.GetString(uniquenessOrdinal) == "UNIQUE", - UniqueConstraint = reader.GetString(isUniqueConstraintOrdinal) == "YES" - }; - - indexItems.Add(indexItem); - } - } + var indexItems = _oracleSystemDataLoader.GetIndexItems(table); var indexGroups = indexItems.GroupBy(x => new { x.SchemaName, x.TableName, x.Name }); List indexes = []; diff --git a/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs b/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs index 7571a6e7..e4215beb 100644 --- a/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs @@ -361,7 +361,12 @@ public override void ChangeColumn(string table, Column column) var change1 = string.Format("{0} TYPE {1}", QuoteColumnNameIfRequired(mapper.Name), mapper.Type); - if ((oldColumn.MigratorDbType == MigratorDbType.Int16 || oldColumn.MigratorDbType == MigratorDbType.Int32 || oldColumn.MigratorDbType == MigratorDbType.Int64 || oldColumn.MigratorDbType == MigratorDbType.Decimal) && column.MigratorDbType == MigratorDbType.Boolean) + if ( + (oldColumn.MigratorDbType == MigratorDbType.Int16 || + oldColumn.MigratorDbType == MigratorDbType.Int32 || + oldColumn.MigratorDbType == MigratorDbType.Int64 || + oldColumn.MigratorDbType == MigratorDbType.Decimal) && + column.MigratorDbType == MigratorDbType.Boolean) { change1 += string.Format(" USING CASE {0} WHEN 1 THEN true ELSE false END", QuoteColumnNameIfRequired(mapper.Name)); } @@ -370,7 +375,6 @@ public override void ChangeColumn(string table, Column column) change1 += string.Format(" USING CASE {0} WHEN '1' THEN true ELSE false END", QuoteColumnNameIfRequired(mapper.Name)); } - ChangeColumn(table, change1); if (mapper.Default != null) @@ -427,7 +431,7 @@ public override string[] GetTables() tables.Add((string)reader[0]); } } - return tables.ToArray(); + return [.. tables]; } public override int GetColumnContentSize(string table, string columnName) @@ -768,7 +772,7 @@ public override Column[] GetColumns(string table) } else { - throw new NotImplementedException(); + throw new NotImplementedException($"{nameof(DbType)} {column.MigratorDbType} not implemented."); } } diff --git a/src/Migrator/Providers/TransformationProvider.cs b/src/Migrator/Providers/TransformationProvider.cs index 2298eadd..8e836dba 100644 --- a/src/Migrator/Providers/TransformationProvider.cs +++ b/src/Migrator/Providers/TransformationProvider.cs @@ -328,7 +328,6 @@ public virtual void AddView(string name, string tableName, params IViewField[] f ExecuteNonQuery(sql); } - public virtual void AddView(string name, string tableName, params IViewElement[] viewElements) { var selectedColumns = viewElements.Where(x => x is ViewColumn) @@ -383,15 +382,6 @@ public virtual void AddView(string name, string tableName, params IViewElement[] /// /// Table name /// Columns - /// - /// Adds the Test table with two columns: - /// - /// Database.AddTable("Test", - /// new Column("Id", typeof(int), ColumnProperty.PrimaryKey), - /// new Column("Title", typeof(string), 100) - /// ); - /// - /// public virtual void AddTable(string name, params IDbField[] columns) { if (this is not SQLiteTransformationProvider && columns.Any(x => x is CheckConstraint)) @@ -404,20 +394,11 @@ public virtual void AddTable(string name, params IDbField[] columns) } /// - /// Add a new table + /// Adds a new table /// /// Table name /// Columns /// the database storage engine to use - /// - /// Adds the Test table with two columns: - /// - /// Database.AddTable("Test", "INNODB", - /// new Column("Id", typeof(int), ColumnProperty.PrimaryKey), - /// new Column("Title", typeof(string), 100) - /// ); - /// - /// public virtual void AddTable(string name, string engine, params IDbField[] fields) { var columns = fields.Where(x => x is Column).Cast().ToArray(); @@ -441,11 +422,12 @@ public virtual void AddTable(string name, string engine, params IDbField[] field } var columnsAndIndexes = JoinColumnsAndIndexes(columnProviders); + AddTable(name, engine, columnsAndIndexes); if (compoundPrimaryKey) { - AddPrimaryKey(getPrimaryKeyname(name), name, pks.ToArray()); + AddPrimaryKey(GetPrimaryKeyname(name), name, pks.ToArray()); } var indexes = fields.Where(x => x is Index).Cast().ToArray(); @@ -463,7 +445,7 @@ public virtual void AddTable(string name, string engine, params IDbField[] field } } - protected virtual string getPrimaryKeyname(string tableName) + protected virtual string GetPrimaryKeyname(string tableName) { return "PK_" + tableName; } From 1e23ba564a1663371c3e781f86059ff1cb0742a6 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Wed, 29 Oct 2025 15:00:19 +0100 Subject: [PATCH 04/19] Postgre Identity => GENERATED --- src/Migrator/Framework/ColumnProperty.cs | 4 +- .../Interfaces/IPostgreSQLSystemDataLoader.cs | 23 + .../Data/PostgreSQLSystemDataLoader.cs | 129 +++++ .../IPostgreSQLTransformationProvider.cs | 7 + .../Impl/PostgreSQL/Models/ColumnInfo.cs | 73 +++ .../Impl/PostgreSQL/Models/TableConstraint.cs | 29 + .../Impl/PostgreSQL/PostgreSQLDialect.cs | 27 +- .../PostgreSQLTransformationProvider.cs | 503 +++++++++--------- 8 files changed, 527 insertions(+), 268 deletions(-) create mode 100644 src/Migrator/Providers/Impl/PostgreSQL/Data/Interfaces/IPostgreSQLSystemDataLoader.cs create mode 100644 src/Migrator/Providers/Impl/PostgreSQL/Data/PostgreSQLSystemDataLoader.cs create mode 100644 src/Migrator/Providers/Impl/PostgreSQL/Interfaces/IPostgreSQLTransformationProvider.cs create mode 100644 src/Migrator/Providers/Impl/PostgreSQL/Models/ColumnInfo.cs create mode 100644 src/Migrator/Providers/Impl/PostgreSQL/Models/TableConstraint.cs diff --git a/src/Migrator/Framework/ColumnProperty.cs b/src/Migrator/Framework/ColumnProperty.cs index 74cbec63..75daca10 100644 --- a/src/Migrator/Framework/ColumnProperty.cs +++ b/src/Migrator/Framework/ColumnProperty.cs @@ -62,10 +62,10 @@ public enum ColumnProperty /// /// Primary key with identity. This is shorthand for and /// - PrimaryKeyWithIdentity = 1 << 9 | PrimaryKey | Identity, + PrimaryKeyWithIdentity = PrimaryKey | Identity, /// - /// Primary key non clustered. + /// Primary key non clustered. /// PrimaryKeyNonClustered = 1 << 10 | PrimaryKey } diff --git a/src/Migrator/Providers/Impl/PostgreSQL/Data/Interfaces/IPostgreSQLSystemDataLoader.cs b/src/Migrator/Providers/Impl/PostgreSQL/Data/Interfaces/IPostgreSQLSystemDataLoader.cs new file mode 100644 index 00000000..97a35085 --- /dev/null +++ b/src/Migrator/Providers/Impl/PostgreSQL/Data/Interfaces/IPostgreSQLSystemDataLoader.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; +using DotNetProjects.Migrator.Providers.Impl.PostgreSQL.Models; + +namespace DotNetProjects.Migrator.Providers.Impl.PostgreSQL.Data.Interfaces; + +public interface IPostgreSQLSystemDataLoader +{ + /// + /// Gets column infos. + /// + /// + /// + /// + List GetColumnInfos(string tableName, string schemaName = "public"); + + /// + /// Gets table constraints. + /// + /// + /// + /// + List GetTableConstraints(string tableName, string schemaName = "public"); +} \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/PostgreSQL/Data/PostgreSQLSystemDataLoader.cs b/src/Migrator/Providers/Impl/PostgreSQL/Data/PostgreSQLSystemDataLoader.cs new file mode 100644 index 00000000..f646f1bd --- /dev/null +++ b/src/Migrator/Providers/Impl/PostgreSQL/Data/PostgreSQLSystemDataLoader.cs @@ -0,0 +1,129 @@ +using System.Collections.Generic; +using DotNetProjects.Migrator.Providers.Impl.PostgreSQL.Data.Interfaces; +using DotNetProjects.Migrator.Providers.Impl.PostgreSQL.Interfaces; +using DotNetProjects.Migrator.Providers.Impl.PostgreSQL.Models; + +namespace DotNetProjects.Migrator.Providers.Impl.PostgreSQL.Data; + +public class PostgreSQLSystemDataLoader(IPostgreSQLTransformationProvider postgreTransformationProvider) : IPostgreSQLSystemDataLoader +{ + private readonly IPostgreSQLTransformationProvider _postgreSQLTransformationProvider = postgreTransformationProvider; + + public List GetTableConstraints(string tableName, string schemaName = "public") + { + var quotedTableName = _postgreSQLTransformationProvider.QuoteTableNameIfRequired(tableName); + + var sql = $@" + SELECT + tc.TABLE_SCHEMA, + tc.TABLE_NAME, + tc.CONSTRAINT_NAME, + tc.CONSTRAINT_TYPE, + kcu.COLUMN_NAME + FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc + JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu + ON tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME + AND tc.TABLE_SCHEMA = kcu.TABLE_SCHEMA + AND tc.TABLE_NAME = kcu.TABLE_NAME + WHERE + LOWER(tc.table_name) = '{quotedTableName.ToLowerInvariant()}' + AND tc.TABLE_SCHEMA = '{schemaName}' + "; + + List tableConstraints = []; + + using var cmd = _postgreSQLTransformationProvider.CreateCommand(); + using var reader = _postgreSQLTransformationProvider.ExecuteQuery(cmd, sql); + + while (reader.Read()) + { + var constraintNameOrdinal = reader.GetOrdinal("CONSTRAINT_NAME"); + var constraintTypeOrdinal = reader.GetOrdinal("CONSTRAINT_TYPE"); + var columnNameOrdinal = reader.GetOrdinal("COLUMN_NAME"); + var tableNameOrdinal = reader.GetOrdinal("TABLE_NAME"); + var tableSchemaOrdinal = reader.GetOrdinal("TABLE_SCHEMA"); + + var tableConstraint = new TableConstraint + { + ConstraintName = !reader.IsDBNull(constraintNameOrdinal) ? reader.GetString(constraintNameOrdinal) : null, + ConstraintType = !reader.IsDBNull(constraintTypeOrdinal) ? reader.GetString(constraintTypeOrdinal) : null, + ColumnName = reader.GetString(columnNameOrdinal), + TableName = reader.GetString(tableNameOrdinal), + TableSchema = reader.GetString(tableSchemaOrdinal), + }; + + tableConstraints.Add(tableConstraint); + } + + return tableConstraints; + } + + public List GetColumnInfos(string tableName, string schemaName = "public") + { + var quotedTableName = _postgreSQLTransformationProvider.QuoteTableNameIfRequired(tableName); + + var sql = $@" + SELECT + c.CHARACTER_MAXIMUM_LENGTH, + c.COLUMN_DEFAULT, + c.COLUMN_NAME, + c.DATA_TYPE, + c.DATETIME_PRECISION, + c.IDENTITY_GENERATION, + c.IS_IDENTITY, + c.IS_NULLABLE, + c.NUMERIC_PRECISION, + c.NUMERIC_SCALE, + c.ORDINAL_POSITION, + c.TABLE_SCHEMA, + c.TABLE_NAME + FROM information_schema.columns c + WHERE + LOWER(c.table_name) = '{quotedTableName.ToLowerInvariant()}' AND + c.TABLE_SCHEMA = '{schemaName}' + "; + + List columns = []; + + using var cmd = _postgreSQLTransformationProvider.CreateCommand(); + using var reader = _postgreSQLTransformationProvider.ExecuteQuery(cmd, sql); + + while (reader.Read()) + { + var characterMaximumLength = reader.GetOrdinal("CHARACTER_MAXIMUM_LENGTH"); + var columnDefaultOrdinal = reader.GetOrdinal("COLUMN_DEFAULT"); + var columnNameOrdinal = reader.GetOrdinal("COLUMN_NAME"); + var dataTypeOrdinal = reader.GetOrdinal("DATA_TYPE"); + var dateTimePrecision = reader.GetOrdinal("DATETIME_PRECISION"); + var identityGenerationOrdinal = reader.GetOrdinal("IDENTITY_GENERATION"); + var isIdentityOrdinal = reader.GetOrdinal("IS_IDENTITY"); + var isNullableOrdinal = reader.GetOrdinal("IS_NULLABLE"); + var numericPrecisionOrdinal = reader.GetOrdinal("NUMERIC_PRECISION"); + var numericScaleOrdinal = reader.GetOrdinal("NUMERIC_SCALE"); + var ordinalPositionOrdinal = reader.GetOrdinal("ORDINAL_POSITION"); + var tableNameOrdinal = reader.GetOrdinal("TABLE_NAME"); + var tableSchemaOrdinal = reader.GetOrdinal("TABLE_SCHEMA"); + + var columnInfo = new ColumnInfo + { + CharacterMaximumLength = !reader.IsDBNull(columnDefaultOrdinal) ? reader.GetInt32(characterMaximumLength) : null, + ColumnDefault = !reader.IsDBNull(columnDefaultOrdinal) ? reader.GetString(columnDefaultOrdinal) : null, + ColumnName = reader.GetString(columnNameOrdinal), + DataType = reader.GetString(dataTypeOrdinal), + DateTimePrecision = !reader.IsDBNull(columnDefaultOrdinal) ? reader.GetInt32(dateTimePrecision) : null, + IdentityGeneration = !reader.IsDBNull(columnDefaultOrdinal) ? reader.GetString(identityGenerationOrdinal) : null, + IsIdentity = reader.GetString(isIdentityOrdinal), + IsNullable = reader.GetString(isNullableOrdinal), + NumericPrecision = !reader.IsDBNull(columnDefaultOrdinal) ? reader.GetInt32(numericPrecisionOrdinal) : null, + NumericScale = !reader.IsDBNull(columnDefaultOrdinal) ? reader.GetInt32(numericScaleOrdinal) : null, + OrdinalPosition = reader.GetInt32(ordinalPositionOrdinal), + TableName = reader.GetString(tableNameOrdinal), + TableSchema = reader.GetString(tableSchemaOrdinal), + }; + + columns.Add(columnInfo); + } + + return columns; + } +} \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/PostgreSQL/Interfaces/IPostgreSQLTransformationProvider.cs b/src/Migrator/Providers/Impl/PostgreSQL/Interfaces/IPostgreSQLTransformationProvider.cs new file mode 100644 index 00000000..5b98b8bb --- /dev/null +++ b/src/Migrator/Providers/Impl/PostgreSQL/Interfaces/IPostgreSQLTransformationProvider.cs @@ -0,0 +1,7 @@ +using DotNetProjects.Migrator.Framework; + +namespace DotNetProjects.Migrator.Providers.Impl.PostgreSQL.Interfaces; + +public interface IPostgreSQLTransformationProvider : ITransformationProvider +{ +} \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/PostgreSQL/Models/ColumnInfo.cs b/src/Migrator/Providers/Impl/PostgreSQL/Models/ColumnInfo.cs new file mode 100644 index 00000000..5ca2e845 --- /dev/null +++ b/src/Migrator/Providers/Impl/PostgreSQL/Models/ColumnInfo.cs @@ -0,0 +1,73 @@ +namespace DotNetProjects.Migrator.Providers.Impl.PostgreSQL.Models; + +/// +/// Represents the INFORMATIONSCHEMA.COLUMNS +/// +public class ColumnInfo +{ + /// + /// Gets or sets the date time precision. + /// + public int? DateTimePrecision { get; set; } + + /// + /// Gets or sets the character maximum length. + /// If data_type identifies a character or bit string type, the declared maximum length; null for all other data types or if no maximum length was declared. + /// + public int? CharacterMaximumLength { get; set; } + + /// + /// Gets or sets the schema name. + /// + public string TableSchema { get; set; } + + /// + /// Gets or sets the table name. + /// + public string TableName { get; set; } + + /// + /// Gets or sets the column name. + /// + public string ColumnName { get; set; } + + /// + /// Gets or sets the data type. Data type of the column, if it is a built-in type, or ARRAY if it is some array (in that case, see the view element_types), else USER-DEFINED (in that case, the type is identified in udt_name and associated columns). If the column is based on a domain, this column refers to the type underlying the domain (and the domain is identified in domain_name and associated columns). + /// + public string DataType { get; set; } + + /// + /// Gets or sets the is nullable string. + /// + public string IsNullable { get; set; } + + /// + /// Gets or sets the column default. + /// + public string ColumnDefault { get; set; } + + /// + /// Gets or sets the is identity string. YES or NO. + /// + public string IsIdentity { get; set; } + + /// + /// Gets or sets the identity generation. + /// + public string IdentityGeneration { get; set; } + + /// + /// Gets or sets the ordinal position + /// + public int OrdinalPosition { get; set; } + + /// + /// Gets or sets th numeric scale. + /// + public int? NumericScale { get; set; } + + /// + /// Gets or sets the numeric precision. + /// + public int? NumericPrecision { get; set; } +} \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/PostgreSQL/Models/TableConstraint.cs b/src/Migrator/Providers/Impl/PostgreSQL/Models/TableConstraint.cs new file mode 100644 index 00000000..848028bf --- /dev/null +++ b/src/Migrator/Providers/Impl/PostgreSQL/Models/TableConstraint.cs @@ -0,0 +1,29 @@ +namespace DotNetProjects.Migrator.Providers.Impl.PostgreSQL.Models; + +public class TableConstraint +{ + /// + /// Gets or sets the schema name. + /// + public string TableSchema { get; set; } + + /// + /// Gets or sets the table name. + /// + public string TableName { get; set; } + + /// + /// Gets or sets the column name. + /// + public string ColumnName { get; set; } + + /// + /// Gets or sets the constraint name. + /// + public string ConstraintName { get; set; } + + /// + /// Gets or sets the constraint type. + /// + public string ConstraintType { get; set; } +} \ No newline at end of file diff --git a/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLDialect.cs b/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLDialect.cs index d6e8c2a7..1200b8f9 100644 --- a/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLDialect.cs +++ b/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLDialect.cs @@ -88,20 +88,11 @@ public PostgreSQLDialect() "WITHOUT", "WORK", "WRITE", "XMAX", "XMIN", "YEAR", "ZONE"); } - public override bool TableNameNeedsQuote - { - get { return false; } - } + public override bool TableNameNeedsQuote => false; - public override bool ConstraintNameNeedsQuote - { - get { return false; } - } + public override bool ConstraintNameNeedsQuote => false; - //public override bool IdentityNeedsType - //{ - // get { return false; } - //} + public override bool IdentityNeedsType => false; public override ITransformationProvider GetTransformationProvider(Dialect dialect, string connectionString, string defaultSchema, string scope, string providerName) { @@ -113,6 +104,18 @@ public override ITransformationProvider GetTransformationProvider(Dialect dialec return new PostgreSQLTransformationProvider(dialect, connection, defaultSchema, scope, providerName); } + public override ColumnPropertiesMapper GetColumnMapper(Column column) + { + var type = column.Size > 0 ? GetTypeName(column.Type, column.Size) : GetTypeName(column.Type); + + if (column.Precision.HasValue || column.Scale.HasValue) + { + type = GetTypeNameParametrized(column.Type, column.Size, column.Precision ?? 0, column.Scale ?? 0); + } + + return new ColumnPropertiesMapper(this, type); + } + public override string Default(object defaultValue) { if (defaultValue is TimeSpan timeSpan) diff --git a/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs b/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs index e4215beb..f7e750a0 100644 --- a/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/PostgreSQL/PostgreSQLTransformationProvider.cs @@ -13,6 +13,9 @@ using DotNetProjects.Migrator.Framework; using DotNetProjects.Migrator.Framework.Models; +using DotNetProjects.Migrator.Providers.Impl.PostgreSQL.Data; +using DotNetProjects.Migrator.Providers.Impl.PostgreSQL.Data.Interfaces; +using DotNetProjects.Migrator.Providers.Impl.PostgreSQL.Interfaces; using DotNetProjects.Migrator.Providers.Models.Indexes; using DotNetProjects.Migrator.Providers.Models.Indexes.Enums; using System; @@ -20,7 +23,6 @@ using System.Data; using System.Globalization; using System.Linq; -using System.Text; using System.Text.RegularExpressions; using Index = DotNetProjects.Migrator.Framework.Index; @@ -29,13 +31,16 @@ namespace DotNetProjects.Migrator.Providers.Impl.PostgreSQL; /// /// Migration transformations provider for PostgreSql (using NPGSql .Net driver) /// -public class PostgreSQLTransformationProvider : TransformationProvider +public class PostgreSQLTransformationProvider : TransformationProvider, IPostgreSQLTransformationProvider { private Regex stripSingleQuoteRegEx = new("(?<=')[^']*(?=')"); + private IPostgreSQLSystemDataLoader _postgreSQLSystemDataLoader; public PostgreSQLTransformationProvider(Dialect dialect, string connectionString, string defaultSchema, string scope, string providerName) : base(dialect, connectionString, defaultSchema, scope) { + Initialize(); + if (string.IsNullOrEmpty(providerName)) { providerName = "Npgsql"; @@ -50,6 +55,7 @@ public PostgreSQLTransformationProvider(Dialect dialect, string connectionString public PostgreSQLTransformationProvider(Dialect dialect, IDbConnection connection, string defaultSchema, string scope, string providerName) : base(dialect, connection, defaultSchema, scope) { + Initialize(); } protected override string GetPrimaryKeyConstraintName(string table) @@ -465,319 +471,304 @@ public override int GetColumnContentSize(string table, string columnName) public override Column[] GetColumns(string table) { - var stringBuilder = new StringBuilder(); - stringBuilder.AppendLine("SELECT"); - stringBuilder.AppendLine(" COLUMN_NAME,"); - stringBuilder.AppendLine(" IS_NULLABLE,"); - stringBuilder.AppendLine(" COLUMN_DEFAULT,"); - stringBuilder.AppendLine(" DATA_TYPE,"); - stringBuilder.AppendLine(" DATETIME_PRECISION,"); - stringBuilder.AppendLine(" CHARACTER_MAXIMUM_LENGTH,"); - stringBuilder.AppendLine(" NUMERIC_PRECISION,"); - stringBuilder.AppendLine(" NUMERIC_SCALE"); - stringBuilder.AppendLine($"FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = 'public' AND TABLE_NAME = lower('{table}');"); - + var columnInfos = _postgreSQLSystemDataLoader.GetColumnInfos(table, "public"); var columns = new List(); + var tableConstraints = _postgreSQLSystemDataLoader.GetTableConstraints(table); - using (var cmd = CreateCommand()) - using (var reader = ExecuteQuery(cmd, stringBuilder.ToString())) + foreach (var columnInfo in columnInfos) { - while (reader.Read()) + var isNullable = columnInfo.IsNullable == "YES"; + var isIdentity = columnInfo.IsIdentity == "YES"; + var isPrimaryKey = tableConstraints.Any(x => x.ColumnName.Equals(columnInfo.ColumnName, StringComparison.OrdinalIgnoreCase) && x.ConstraintType == "PRIMARY KEY"); + + MigratorDbType dbType = 0; + int? precision = null; + int? scale = null; + int? size = null; + + if (new[] { "timestamptz", "timestamp with time zone" }.Contains(columnInfo.DataType)) { - var defaultValueOrdinal = reader.GetOrdinal("COLUMN_DEFAULT"); - var characterMaximumLengthOrdinal = reader.GetOrdinal("CHARACTER_MAXIMUM_LENGTH"); - var dateTimePrecisionOrdinal = reader.GetOrdinal("DATETIME_PRECISION"); - var numericPrecisionOrdinal = reader.GetOrdinal("NUMERIC_PRECISION"); - var numericScaleOrdinal = reader.GetOrdinal("NUMERIC_SCALE"); - - var columnName = reader.GetString(reader.GetOrdinal("COLUMN_NAME")); - var isNullable = reader.GetString(reader.GetOrdinal("IS_NULLABLE")) == "YES"; - var defaultValueString = reader.IsDBNull(defaultValueOrdinal) ? null : reader.GetString(defaultValueOrdinal); - var dataTypeString = reader.GetString(reader.GetOrdinal("DATA_TYPE")); - var dateTimePrecision = reader.IsDBNull(dateTimePrecisionOrdinal) ? null : (int?)reader.GetInt32(dateTimePrecisionOrdinal); - var characterMaximumLength = reader.IsDBNull(characterMaximumLengthOrdinal) ? null : (int?)reader.GetInt32(characterMaximumLengthOrdinal); - var numericPrecision = reader.IsDBNull(numericPrecisionOrdinal) ? null : (int?)reader.GetInt32(numericPrecisionOrdinal); - var numericScale = reader.IsDBNull(numericScaleOrdinal) ? null : (int?)reader.GetInt32(numericScaleOrdinal); - - MigratorDbType dbType = 0; - int? precision = null; - int? scale = null; - int? size = null; - - if (new[] { "timestamptz", "timestamp with time zone" }.Contains(dataTypeString)) + dbType = MigratorDbType.DateTimeOffset; + precision = columnInfo.DateTimePrecision; + } + else if (columnInfo.DataType == "double precision") + { + dbType = MigratorDbType.Double; + scale = columnInfo.NumericScale; + precision = columnInfo.NumericPrecision; + } + else if (columnInfo.DataType == "timestamp" || columnInfo.DataType == "timestamp without time zone") + { + // 6 is the maximum in PostgreSQL + if (columnInfo.DateTimePrecision > 5) { - dbType = MigratorDbType.DateTimeOffset; - precision = dateTimePrecision; + dbType = MigratorDbType.DateTime2; } - else if (dataTypeString == "double precision") + else { - dbType = MigratorDbType.Double; - scale = numericScale; - precision = numericPrecision; + dbType = MigratorDbType.DateTime; } - else if (dataTypeString == "timestamp" || dataTypeString == "timestamp without time zone") - { - // 6 is the maximum in PostgreSQL - if (dateTimePrecision > 5) - { - dbType = MigratorDbType.DateTime2; - } - else - { - dbType = MigratorDbType.DateTime; - } - precision = dateTimePrecision; - } - else if (dataTypeString == "smallint") - { - dbType = MigratorDbType.Int16; - } - else if (dataTypeString == "integer") - { - dbType = MigratorDbType.Int32; - } - else if (dataTypeString == "bigint") - { - dbType = MigratorDbType.Int64; - } - else if (dataTypeString == "numeric") - { - dbType = MigratorDbType.Decimal; - precision = numericPrecision; - scale = numericScale; - } - else if (dataTypeString == "real") - { - dbType = MigratorDbType.Single; - } - else if (dataTypeString == "interval") - { - dbType = MigratorDbType.Interval; - } - else if (dataTypeString == "money") - { - dbType = MigratorDbType.Currency; - } - else if (dataTypeString == "date") - { - dbType = MigratorDbType.Date; - } - else if (dataTypeString == "byte") - { - dbType = MigratorDbType.Binary; - } - else if (dataTypeString == "uuid") - { - dbType = MigratorDbType.Guid; - } - else if (dataTypeString == "xml") - { - dbType = MigratorDbType.Xml; - } - else if (dataTypeString == "time") - { - dbType = MigratorDbType.Time; - } - else if (dataTypeString == "boolean") - { - dbType = MigratorDbType.Boolean; - } - else if (dataTypeString == "text" || dataTypeString == "character varying") + precision = columnInfo.DateTimePrecision; + } + else if (columnInfo.DataType == "smallint") + { + dbType = MigratorDbType.Int16; + } + else if (columnInfo.DataType == "integer") + { + dbType = MigratorDbType.Int32; + } + else if (columnInfo.DataType == "bigint") + { + dbType = MigratorDbType.Int64; + } + else if (columnInfo.DataType == "numeric") + { + dbType = MigratorDbType.Decimal; + precision = columnInfo.NumericPrecision; + scale = columnInfo.NumericScale; + } + else if (columnInfo.DataType == "real") + { + dbType = MigratorDbType.Single; + } + else if (columnInfo.DataType == "interval") + { + dbType = MigratorDbType.Interval; + } + else if (columnInfo.DataType == "money") + { + dbType = MigratorDbType.Currency; + } + else if (columnInfo.DataType == "date") + { + dbType = MigratorDbType.Date; + } + else if (columnInfo.DataType == "byte") + { + dbType = MigratorDbType.Binary; + } + else if (columnInfo.DataType == "uuid") + { + dbType = MigratorDbType.Guid; + } + else if (columnInfo.DataType == "xml") + { + dbType = MigratorDbType.Xml; + } + else if (columnInfo.DataType == "time") + { + dbType = MigratorDbType.Time; + } + else if (columnInfo.DataType == "boolean") + { + dbType = MigratorDbType.Boolean; + } + else if (columnInfo.DataType == "text" || columnInfo.DataType == "character varying") + { + dbType = MigratorDbType.String; + size = columnInfo.CharacterMaximumLength; + } + else if (columnInfo.DataType == "bytea") + { + dbType = MigratorDbType.Binary; + } + else if (columnInfo.DataType == "character" || columnInfo.DataType.StartsWith("character(")) + { + throw new NotSupportedException("Data type 'character' detected. 'character' is not supported. Use 'text' or 'character varying' instead."); + } + else + { + throw new NotImplementedException("The data type is not implemented. Please file an issue."); + } + + var column = new Column(columnInfo.ColumnName, dbType) + { + Precision = precision, + Scale = scale, + // Size should be nullable + Size = size ?? 0 + }; + + column.ColumnProperty |= isNullable ? ColumnProperty.Null : ColumnProperty.NotNull; + + if (isPrimaryKey) + { + column.ColumnProperty = column.ColumnProperty.Set(ColumnProperty.PrimaryKey); + } + + if (isIdentity) + { + column.ColumnProperty = column.ColumnProperty.Set(ColumnProperty.Identity); + } + + if (columnInfo.ColumnDefault != null) + { + if (column.MigratorDbType == MigratorDbType.Int16 || column.MigratorDbType == MigratorDbType.Int32 || column.MigratorDbType == MigratorDbType.Int64) { - dbType = MigratorDbType.String; - size = characterMaximumLength; + column.DefaultValue = long.Parse(columnInfo.ColumnDefault.ToString()); } - else if (dataTypeString == "bytea") + else if (column.MigratorDbType == MigratorDbType.UInt16 || column.MigratorDbType == MigratorDbType.UInt32 || column.MigratorDbType == MigratorDbType.UInt64) { - dbType = MigratorDbType.Binary; + column.DefaultValue = ulong.Parse(columnInfo.ColumnDefault.ToString()); } - else if (dataTypeString == "character" || dataTypeString.StartsWith("character(")) + else if (column.MigratorDbType == MigratorDbType.Double || column.MigratorDbType == MigratorDbType.Single) { - throw new NotSupportedException("Data type 'character' detected. 'character' is not supported. Use 'text' or 'character varying' instead."); + column.DefaultValue = double.Parse(columnInfo.ColumnDefault.ToString(), CultureInfo.InvariantCulture); } - else + else if (column.MigratorDbType == MigratorDbType.Interval) { - throw new NotImplementedException("The data type is not implemented. Please file an issue."); - } + if (columnInfo.ColumnDefault.StartsWith("'")) + { + var match = stripSingleQuoteRegEx.Match(columnInfo.ColumnDefault); - var column = new Column(columnName, dbType) - { - Precision = precision, - Scale = scale, - // Size should be nullable - Size = size ?? 0 - }; + if (!match.Success) + { + throw new Exception("Postgre default value for interval: Single quotes around the interval string are expected."); + } + + column.DefaultValue = match.Value; + var splitted = match.Value.Split(':'); + if (splitted.Length != 3) + { + throw new NotImplementedException($"Cannot interpret {columnInfo.ColumnDefault} in column '{column.Name}' unexpected pattern."); + } - column.ColumnProperty |= isNullable ? ColumnProperty.Null : ColumnProperty.NotNull; + var hours = int.Parse(splitted[0], CultureInfo.InvariantCulture); + var minutes = int.Parse(splitted[1], CultureInfo.InvariantCulture); + var splitted2 = splitted[2].Split('.'); + var seconds = int.Parse(splitted2[0], CultureInfo.InvariantCulture); + var milliseconds = int.Parse(splitted2[1], CultureInfo.InvariantCulture); - if (defaultValueString != null) + column.DefaultValue = new TimeSpan(0, hours, minutes, seconds, milliseconds); + } + else + { + // We assume that the value was added using this migrator so we do not interpret things like '2 days 01:02:03' if you + // added such format you will run into this exception. + throw new NotImplementedException($"Cannot parse {columnInfo.ColumnDefault} in column '{column.Name}' unexpected pattern."); + } + } + else if (column.MigratorDbType == MigratorDbType.Boolean) { - if (column.MigratorDbType == MigratorDbType.Int16 || column.MigratorDbType == MigratorDbType.Int32 || column.MigratorDbType == MigratorDbType.Int64) + var truthy = new[] { "TRUE", "YES", "'true'", "on", "'on'", "t", "'t'" }; + var falsy = new[] { "FALSE", "NO", "'false'", "off", "'off'", "f", "'f'" }; + + if (truthy.Any(x => x.Equals(columnInfo.ColumnDefault.Trim(), StringComparison.OrdinalIgnoreCase))) { - column.DefaultValue = long.Parse(defaultValueString.ToString()); + column.DefaultValue = true; } - else if (column.MigratorDbType == MigratorDbType.UInt16 || column.MigratorDbType == MigratorDbType.UInt32 || column.MigratorDbType == MigratorDbType.UInt64) + else if (falsy.Any(x => x.Equals(columnInfo.ColumnDefault.Trim(), StringComparison.OrdinalIgnoreCase))) { - column.DefaultValue = ulong.Parse(defaultValueString.ToString()); + column.DefaultValue = false; } - else if (column.MigratorDbType == MigratorDbType.Double || column.MigratorDbType == MigratorDbType.Single) + else { - column.DefaultValue = double.Parse(defaultValueString.ToString(), CultureInfo.InvariantCulture); + throw new NotImplementedException($"Cannot parse {columnInfo.ColumnDefault} in column '{column.Name}'"); } - else if (column.MigratorDbType == MigratorDbType.Interval) + } + else if (column.MigratorDbType == MigratorDbType.DateTime || column.MigratorDbType == MigratorDbType.DateTime2) + { + if (columnInfo.ColumnDefault.StartsWith("'")) { - if (defaultValueString.StartsWith("'")) - { - var match = stripSingleQuoteRegEx.Match(defaultValueString); + var match = stripSingleQuoteRegEx.Match(columnInfo.ColumnDefault); - if (!match.Success) - { - throw new Exception("Postgre default value for interval: Single quotes around the interval string are expected."); - } + if (!match.Success) + { + throw new NotImplementedException($"Cannot parse {columnInfo.ColumnDefault} in column '{column.Name}'"); + } - column.DefaultValue = match.Value; - var splitted = match.Value.Split(':'); - if (splitted.Length != 3) - { - throw new NotImplementedException($"Cannot interpret {defaultValueString} in column '{column.Name}' unexpected pattern."); - } + var timeString = match.Value; - var hours = int.Parse(splitted[0], CultureInfo.InvariantCulture); - var minutes = int.Parse(splitted[1], CultureInfo.InvariantCulture); - var splitted2 = splitted[2].Split('.'); - var seconds = int.Parse(splitted2[0], CultureInfo.InvariantCulture); - var milliseconds = int.Parse(splitted2[1], CultureInfo.InvariantCulture); + // We convert to UTC since we restrict date time default values to UTC on default value definition. + var dateTimeExtracted = DateTime.ParseExact(timeString, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal); - column.DefaultValue = new TimeSpan(0, hours, minutes, seconds, milliseconds); - } - else - { - // We assume that the value was added using this migrator so we do not interpret things like '2 days 01:02:03' if you - // added such format you will run into this exception. - throw new NotImplementedException($"Cannot parse {defaultValueString} in column '{column.Name}' unexpected pattern."); - } + column.DefaultValue = dateTimeExtracted; } - else if (column.MigratorDbType == MigratorDbType.Boolean) + else { - var truthy = new[] { "TRUE", "YES", "'true'", "on", "'on'", "t", "'t'" }; - var falsy = new[] { "FALSE", "NO", "'false'", "off", "'off'", "f", "'f'" }; - - if (truthy.Any(x => x.Equals(defaultValueString.Trim(), StringComparison.OrdinalIgnoreCase))) - { - column.DefaultValue = true; - } - else if (falsy.Any(x => x.Equals(defaultValueString.Trim(), StringComparison.OrdinalIgnoreCase))) - { - column.DefaultValue = false; - } - else - { - throw new NotImplementedException($"Cannot parse {defaultValueString} in column '{column.Name}'"); - } + throw new NotImplementedException($"Cannot parse {columnInfo.ColumnDefault} in column '{column.Name}'"); } - else if (column.MigratorDbType == MigratorDbType.DateTime || column.MigratorDbType == MigratorDbType.DateTime2) + } + else if (column.MigratorDbType == MigratorDbType.Guid) + { + if (columnInfo.ColumnDefault.StartsWith("'")) { - if (defaultValueString.StartsWith("'")) - { - var match = stripSingleQuoteRegEx.Match(defaultValueString); - - if (!match.Success) - { - throw new NotImplementedException($"Cannot parse {defaultValueString} in column '{column.Name}'"); - } - - var timeString = match.Value; + var match = stripSingleQuoteRegEx.Match(columnInfo.ColumnDefault); - // We convert to UTC since we restrict date time default values to UTC on default value definition. - var dateTimeExtracted = DateTime.ParseExact(timeString, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal); - - column.DefaultValue = dateTimeExtracted; - } - else + if (!match.Success) { - throw new NotImplementedException($"Cannot parse {defaultValueString} in column '{column.Name}'"); + throw new NotImplementedException($"Cannot parse {columnInfo.ColumnDefault} in column '{column.Name}'"); } + + column.DefaultValue = Guid.Parse(match.Value); } - else if (column.MigratorDbType == MigratorDbType.Guid) + else { - if (defaultValueString.StartsWith("'")) - { - var match = stripSingleQuoteRegEx.Match(defaultValueString); - - if (!match.Success) - { - throw new NotImplementedException($"Cannot parse {defaultValueString} in column '{column.Name}'"); - } + throw new NotImplementedException($"Cannot parse {columnInfo.ColumnDefault} in column '{column.Name}'"); + } + } + else if (column.MigratorDbType == MigratorDbType.Decimal) + { + column.DefaultValue = decimal.Parse(columnInfo.ColumnDefault, CultureInfo.InvariantCulture); + } + else if (column.MigratorDbType == MigratorDbType.String) + { + if (columnInfo.ColumnDefault.StartsWith("'")) + { + var match = stripSingleQuoteRegEx.Match(columnInfo.ColumnDefault); - column.DefaultValue = Guid.Parse(match.Value); - } - else + if (!match.Success) { - throw new NotImplementedException($"Cannot parse {defaultValueString} in column '{column.Name}'"); + throw new Exception("Postgre default value for date time: Single quotes around the date time string are expected."); } + + column.DefaultValue = match.Value; } - else if (column.MigratorDbType == MigratorDbType.Decimal) + else { - column.DefaultValue = decimal.Parse(defaultValueString, CultureInfo.InvariantCulture); + throw new NotImplementedException(); } - else if (column.MigratorDbType == MigratorDbType.String) + } + else if (column.MigratorDbType == MigratorDbType.Binary) + { + if (columnInfo.ColumnDefault.StartsWith("'")) { - if (defaultValueString.StartsWith("'")) - { - var match = stripSingleQuoteRegEx.Match(defaultValueString); - - if (!match.Success) - { - throw new Exception("Postgre default value for date time: Single quotes around the date time string are expected."); - } + var match = stripSingleQuoteRegEx.Match(columnInfo.ColumnDefault); - column.DefaultValue = match.Value; - } - else + if (!match.Success) { - throw new NotImplementedException(); + throw new NotImplementedException($"Cannot parse {columnInfo.ColumnDefault} in column '{column.Name}'"); } - } - else if (column.MigratorDbType == MigratorDbType.Binary) - { - if (defaultValueString.StartsWith("'")) - { - var match = stripSingleQuoteRegEx.Match(defaultValueString); - if (!match.Success) - { - throw new NotImplementedException($"Cannot parse {defaultValueString} in column '{column.Name}'"); - } - - var singleQuoteString = match.Value; + var singleQuoteString = match.Value; - if (!singleQuoteString.StartsWith("\\x")) - { - throw new Exception(@"Postgre \x notation expected."); - } + if (!singleQuoteString.StartsWith("\\x")) + { + throw new Exception(@"Postgre \x notation expected."); + } - var hexString = singleQuoteString.Substring(2); + var hexString = singleQuoteString.Substring(2); - // Not available in old .NET version: Convert.FromHexString(hexString); + // Not available in old .NET version: Convert.FromHexString(hexString); - column.DefaultValue = Enumerable.Range(0, hexString.Length / 2) - .Select(x => Convert.ToByte(hexString.Substring(x * 2, 2), 16)) - .ToArray(); - } - else - { - throw new NotImplementedException($"Cannot parse {defaultValueString} in column '{column.Name}'"); - } + column.DefaultValue = Enumerable.Range(0, hexString.Length / 2) + .Select(x => Convert.ToByte(hexString.Substring(x * 2, 2), 16)) + .ToArray(); } else { - throw new NotImplementedException($"{nameof(DbType)} {column.MigratorDbType} not implemented."); + throw new NotImplementedException($"Cannot parse {columnInfo.ColumnDefault} in column '{column.Name}'"); } } - - columns.Add(column); + else + { + throw new NotImplementedException($"{nameof(DbType)} {column.MigratorDbType} not implemented."); + } } + + columns.Add(column); } return columns.ToArray(); @@ -944,4 +935,8 @@ protected override void ConfigureParameterWithValue(IDbDataParameter parameter, } } + private void Initialize() + { + _postgreSQLSystemDataLoader = new PostgreSQLSystemDataLoader(this); + } } From eff914f2e2f2f9f9adaaa216eeae6a603e972b8e Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Wed, 29 Oct 2025 15:31:35 +0100 Subject: [PATCH 05/19] Update PostgreSQLSystemDataLoader --- .../PostgreSQL/Data/PostgreSQLSystemDataLoader.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Migrator/Providers/Impl/PostgreSQL/Data/PostgreSQLSystemDataLoader.cs b/src/Migrator/Providers/Impl/PostgreSQL/Data/PostgreSQLSystemDataLoader.cs index f646f1bd..43872ea8 100644 --- a/src/Migrator/Providers/Impl/PostgreSQL/Data/PostgreSQLSystemDataLoader.cs +++ b/src/Migrator/Providers/Impl/PostgreSQL/Data/PostgreSQLSystemDataLoader.cs @@ -94,7 +94,7 @@ FROM information_schema.columns c var columnDefaultOrdinal = reader.GetOrdinal("COLUMN_DEFAULT"); var columnNameOrdinal = reader.GetOrdinal("COLUMN_NAME"); var dataTypeOrdinal = reader.GetOrdinal("DATA_TYPE"); - var dateTimePrecision = reader.GetOrdinal("DATETIME_PRECISION"); + var dateTimePrecisionOrdinal = reader.GetOrdinal("DATETIME_PRECISION"); var identityGenerationOrdinal = reader.GetOrdinal("IDENTITY_GENERATION"); var isIdentityOrdinal = reader.GetOrdinal("IS_IDENTITY"); var isNullableOrdinal = reader.GetOrdinal("IS_NULLABLE"); @@ -106,16 +106,16 @@ FROM information_schema.columns c var columnInfo = new ColumnInfo { - CharacterMaximumLength = !reader.IsDBNull(columnDefaultOrdinal) ? reader.GetInt32(characterMaximumLength) : null, + CharacterMaximumLength = !reader.IsDBNull(characterMaximumLength) ? reader.GetInt32(characterMaximumLength) : null, ColumnDefault = !reader.IsDBNull(columnDefaultOrdinal) ? reader.GetString(columnDefaultOrdinal) : null, ColumnName = reader.GetString(columnNameOrdinal), DataType = reader.GetString(dataTypeOrdinal), - DateTimePrecision = !reader.IsDBNull(columnDefaultOrdinal) ? reader.GetInt32(dateTimePrecision) : null, - IdentityGeneration = !reader.IsDBNull(columnDefaultOrdinal) ? reader.GetString(identityGenerationOrdinal) : null, + DateTimePrecision = !reader.IsDBNull(dateTimePrecisionOrdinal) ? reader.GetInt32(dateTimePrecisionOrdinal) : null, + IdentityGeneration = !reader.IsDBNull(identityGenerationOrdinal) ? reader.GetString(identityGenerationOrdinal) : null, IsIdentity = reader.GetString(isIdentityOrdinal), IsNullable = reader.GetString(isNullableOrdinal), - NumericPrecision = !reader.IsDBNull(columnDefaultOrdinal) ? reader.GetInt32(numericPrecisionOrdinal) : null, - NumericScale = !reader.IsDBNull(columnDefaultOrdinal) ? reader.GetInt32(numericScaleOrdinal) : null, + NumericPrecision = !reader.IsDBNull(numericPrecisionOrdinal) ? reader.GetInt32(numericPrecisionOrdinal) : null, + NumericScale = !reader.IsDBNull(numericScaleOrdinal) ? reader.GetInt32(numericScaleOrdinal) : null, OrdinalPosition = reader.GetInt32(ordinalPositionOrdinal), TableName = reader.GetString(tableNameOrdinal), TableSchema = reader.GetString(tableSchemaOrdinal), From 64dcce98efac38fb8ea8c4ffbe9fcec24f32ba8a Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Wed, 29 Oct 2025 17:09:19 +0100 Subject: [PATCH 06/19] Oracle test handling adjustments --- .editorconfig | 13 +- .github/workflows/sql/oracle.sql | 26 ++-- .../Data/Common/EntityConfiguration.cs | 15 +++ .../Common/Interfaces/IEntityConfiguration.cs | 9 ++ .../Interfaces/IMappingSchemaFactory.cs | 8 ++ .../Data/Common/MappingSchemaFactory.cs | 39 ++++++ .../OracleAllConsColumnsConfiguration.cs | 29 +++++ .../OracleAllConstraintsConfiguration.cs | 35 +++++ .../OracleAllTabColumnsConfiguration.cs | 38 ++++++ .../Oracle/OracleAllUsersConfiguration.cs | 17 +++ .../Oracle/OracleDBADataFilesConfiguration.cs | 20 +++ .../Oracle/OracleVSessionConfiguration.cs | 23 ++++ .../Data/Models/Oracle/AllConsColumns.cs | 32 +++++ .../Data/Models/Oracle/AllConstraints.cs | 55 ++++++++ .../Data/Models/Oracle/AllTabColumns.cs | 48 +++++++ .../Database/Data/Models/Oracle/AllUsers.cs | 12 ++ .../Data/Models/Oracle/DBADataFiles.cs | 17 +++ .../Database/Data/Models/Oracle/VSession.cs | 19 +++ .../DatabaseIntegrationTestServiceBase.cs | 2 +- .../OracleDatabaseIntegrationTestService.cs | 121 ++++++++++-------- ...ostgreSqlDatabaseIntegrationTestService.cs | 2 +- .../SQLiteDatabaseIntegrationTestService.cs | 2 +- ...SqlServerDatabaseIntegrationTestService.cs | 2 +- src/Migrator.Tests/Migrator.Tests.csproj | 4 +- src/Migrator/DotNetProjects.Migrator.csproj | 2 +- .../SQLite/SQLiteTransformationProvider.cs | 1 + 26 files changed, 516 insertions(+), 75 deletions(-) create mode 100644 src/Migrator.Tests/Database/Data/Common/EntityConfiguration.cs create mode 100644 src/Migrator.Tests/Database/Data/Common/Interfaces/IEntityConfiguration.cs create mode 100644 src/Migrator.Tests/Database/Data/Common/Interfaces/IMappingSchemaFactory.cs create mode 100644 src/Migrator.Tests/Database/Data/Common/MappingSchemaFactory.cs create mode 100644 src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllConsColumnsConfiguration.cs create mode 100644 src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllConstraintsConfiguration.cs create mode 100644 src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllTabColumnsConfiguration.cs create mode 100644 src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllUsersConfiguration.cs create mode 100644 src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleDBADataFilesConfiguration.cs create mode 100644 src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleVSessionConfiguration.cs create mode 100644 src/Migrator.Tests/Database/Data/Models/Oracle/AllConsColumns.cs create mode 100644 src/Migrator.Tests/Database/Data/Models/Oracle/AllConstraints.cs create mode 100644 src/Migrator.Tests/Database/Data/Models/Oracle/AllTabColumns.cs create mode 100644 src/Migrator.Tests/Database/Data/Models/Oracle/AllUsers.cs create mode 100644 src/Migrator.Tests/Database/Data/Models/Oracle/DBADataFiles.cs create mode 100644 src/Migrator.Tests/Database/Data/Models/Oracle/VSession.cs diff --git a/.editorconfig b/.editorconfig index 39681c28..4e4275c2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,13 +8,15 @@ spelling_exclusion_path = SpellingExclusions.dic # Code files [*.{cs,csx,vb,vbx}] +end_of_line = lf indent_size = 4 insert_final_newline = true charset = utf-8 # XML project files [*.{csproj,vbproj,vcxproj,vcxproj.filters,proj,projitems,shproj}] -indent_size = 2 +indent_size = 4 +end_of_line = lf # XML config files [*.{props,targets,ruleset,config,nuspec,resx,vsixmanifest,vsct}] @@ -155,6 +157,7 @@ dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion dotnet_style_prefer_compound_assignment = true:suggestion dotnet_style_prefer_simplified_interpolation = true:suggestion dotnet_style_prefer_collection_expression = when_types_loosely_match:suggestion +tab_width = 4 ########################################## # C# Specific settings @@ -223,7 +226,7 @@ csharp_style_prefer_extended_property_pattern = true:suggestion csharp_style_prefer_init_only_properties = true:suggestion # Braces -csharp_prefer_braces = always:warning +csharp_prefer_braces = true:silent csharp_preserve_single_line_blocks = true csharp_preserve_single_line_statements = true dotnet_diagnostic.IDE0011.severity = warning @@ -271,6 +274,12 @@ dotnet_diagnostic.IDE2005.severity = warning dotnet_diagnostic.IDE2006.severity = warning csharp_style_expression_bodied_lambdas = true:silent csharp_style_expression_bodied_local_functions = false:silent +csharp_using_directive_placement = outside_namespace:silent +csharp_prefer_simple_using_statement = true:suggestion +csharp_style_prefer_method_group_conversion = true:silent +csharp_style_prefer_top_level_statements = true:silent +csharp_style_prefer_primary_constructors = true:suggestion +csharp_prefer_system_threading_lock = true:suggestion ########################################## # Exceptions by path diff --git a/.github/workflows/sql/oracle.sql b/.github/workflows/sql/oracle.sql index 87034f22..f4e7eba2 100644 --- a/.github/workflows/sql/oracle.sql +++ b/.github/workflows/sql/oracle.sql @@ -4,30 +4,26 @@ alter session set container = freepdb1; create user k identified by k; -grant - create user -to k; +grant select_catalog_role to k; -grant - drop user -to k; +grant create tablespace to k; -grant - create session -to k with admin option; +grant drop tablespace to k; + +grant create user to k; + +grant drop user to k; + +grant create session to k with admin option; grant resource to k with admin option; grant connect to k with admin option; -grant - unlimited tablespace -to k with admin option; +grant unlimited tablespace to k with admin option; grant select on v_$session to k with grant option -grant - alter system -to k +grant alter system to k exit; \ No newline at end of file diff --git a/src/Migrator.Tests/Database/Data/Common/EntityConfiguration.cs b/src/Migrator.Tests/Database/Data/Common/EntityConfiguration.cs new file mode 100644 index 00000000..2b3a5f18 --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Common/EntityConfiguration.cs @@ -0,0 +1,15 @@ +using LinqToDB.Mapping; +using Migrator.Tests.Database.Data.Common.Interfaces; + +namespace Migrator.Tests.Database.Data.Common; + +public abstract class EntityConfiguration(FluentMappingBuilder fluentMappingBuilder) : IEntityConfiguration where T : class +{ + protected EntityMappingBuilder _EntityMappingBuilder = fluentMappingBuilder.Entity(); + protected FluentMappingBuilder _FluentMappingBuilder = fluentMappingBuilder; + + /// + /// Configures the entity in the fluent migrator of Linq2db + /// + public abstract void ConfigureEntity(); +} \ No newline at end of file diff --git a/src/Migrator.Tests/Database/Data/Common/Interfaces/IEntityConfiguration.cs b/src/Migrator.Tests/Database/Data/Common/Interfaces/IEntityConfiguration.cs new file mode 100644 index 00000000..7ada7a23 --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Common/Interfaces/IEntityConfiguration.cs @@ -0,0 +1,9 @@ +namespace Migrator.Tests.Database.Data.Common.Interfaces; + +public interface IEntityConfiguration +{ + /// + /// Configure the entity mapping. + /// + void ConfigureEntity(); +} \ No newline at end of file diff --git a/src/Migrator.Tests/Database/Data/Common/Interfaces/IMappingSchemaFactory.cs b/src/Migrator.Tests/Database/Data/Common/Interfaces/IMappingSchemaFactory.cs new file mode 100644 index 00000000..7fb68e3b --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Common/Interfaces/IMappingSchemaFactory.cs @@ -0,0 +1,8 @@ +using LinqToDB.Mapping; + +namespace Migrator.Tests.Database.Data.Common.Interfaces; + +public interface IMappingSchemaFactory +{ + MappingSchema CreateOracleMappingSchema(); +} \ No newline at end of file diff --git a/src/Migrator.Tests/Database/Data/Common/MappingSchemaFactory.cs b/src/Migrator.Tests/Database/Data/Common/MappingSchemaFactory.cs new file mode 100644 index 00000000..e044f899 --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Common/MappingSchemaFactory.cs @@ -0,0 +1,39 @@ +using System.Collections.Generic; +using LinqToDB.Mapping; +using Migrator.Tests.Database.Data.Common.Interfaces; +using Migrator.Tests.Database.Data.Mappings.Oracle; + +namespace DotNetProjects.Migrator.Framework.Data.Common; + +public class MappingSchemaFactory() : IMappingSchemaFactory +{ + public MappingSchema CreateOracleMappingSchema() + { + var fluentMappingBuilder = new FluentMappingBuilder(); + + var configs = new List + { + new OracleAllConsColumnsConfiguration(fluentMappingBuilder), + new OracleAllConstraintsConfiguration(fluentMappingBuilder), + new OracleAllTabColumnsConfiguration(fluentMappingBuilder), + new OracleAllTabColumnsConfiguration(fluentMappingBuilder), + new OracleAllUsersConfiguration(fluentMappingBuilder), + new OracleDBADataFilesConfiguration(fluentMappingBuilder), + new OracleVSessionConfiguration(fluentMappingBuilder), + }; + + return Configure(fluentMappingBuilder, configs); + } + + private static MappingSchema Configure(FluentMappingBuilder fluentMappingBuilder, IEnumerable entityConfigurations) + { + foreach (var config in entityConfigurations) + { + config.ConfigureEntity(); + } + + fluentMappingBuilder.Build(); + + return fluentMappingBuilder.MappingSchema; + } +} diff --git a/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllConsColumnsConfiguration.cs b/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllConsColumnsConfiguration.cs new file mode 100644 index 00000000..ea016267 --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllConsColumnsConfiguration.cs @@ -0,0 +1,29 @@ +using DotNetProjects.Migrator.Framework.Data.Models.Oracle; +using LinqToDB.Mapping; +using Migrator.Tests.Database.Data.Common; + +namespace Migrator.Tests.Database.Data.Mappings.Oracle; + +public class OracleAllConsColumnsConfiguration(FluentMappingBuilder fluentMappingBuilder) + : EntityConfiguration(fluentMappingBuilder) +{ + public override void ConfigureEntity() + { + _EntityMappingBuilder!.HasTableName("ALL_CONS_COLUMNS"); + + _EntityMappingBuilder.Property(x => x.ColumnName) + .HasColumnName("COLUMN_NAME"); + + _EntityMappingBuilder.Property(x => x.ConstraintName) + .HasColumnName("CONSTRAINT_NAME"); + + _EntityMappingBuilder.Property(x => x.Owner) + .HasColumnName("OWNER"); + + _EntityMappingBuilder.Property(x => x.Position) + .HasColumnName("POSITION"); + + _EntityMappingBuilder.Property(x => x.TableName) + .HasColumnName("TABLE_NAME"); + } +} \ No newline at end of file diff --git a/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllConstraintsConfiguration.cs b/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllConstraintsConfiguration.cs new file mode 100644 index 00000000..88c22e53 --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllConstraintsConfiguration.cs @@ -0,0 +1,35 @@ +using DotNetProjects.Migrator.Framework.Data.Models.Oracle; +using LinqToDB.Mapping; +using Migrator.Tests.Database.Data.Common; + +namespace Migrator.Tests.Database.Data.Mappings.Oracle; + +public class OracleAllConstraintsConfiguration(FluentMappingBuilder fluentMappingBuilder) + : EntityConfiguration(fluentMappingBuilder) +{ + public override void ConfigureEntity() + { + _EntityMappingBuilder!.HasTableName("ALL_CONSTRAINTS"); + + _EntityMappingBuilder.Property(x => x.ConstraintName) + .HasColumnName("CONSTRAINT_NAME"); + + _EntityMappingBuilder.Property(x => x.RConstraintName) + .HasColumnName("R_CONSTRAINT_NAME"); + + _EntityMappingBuilder.Property(x => x.ROwner) + .HasColumnName("R_OWNER"); + + _EntityMappingBuilder.Property(x => x.ConstraintType) + .HasColumnName("CONSTRAINT_TYPE"); + + _EntityMappingBuilder.Property(x => x.Owner) + .HasColumnName("OWNER"); + + _EntityMappingBuilder.Property(x => x.Status) + .HasColumnName("STATUS"); + + _EntityMappingBuilder.Property(x => x.TableName) + .HasColumnName("TABLE_NAME"); + } +} diff --git a/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllTabColumnsConfiguration.cs b/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllTabColumnsConfiguration.cs new file mode 100644 index 00000000..da72fa1a --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllTabColumnsConfiguration.cs @@ -0,0 +1,38 @@ +using DotNetProjects.Migrator.Framework.Data.Models.Oracle; +using LinqToDB.Mapping; +using Migrator.Tests.Database.Data.Common; + +namespace Migrator.Tests.Database.Data.Mappings.Oracle; + +public class OracleAllTabColumnsConfiguration(FluentMappingBuilder fluentMappingBuilder) + : EntityConfiguration(fluentMappingBuilder) +{ + public override void ConfigureEntity() + { + _EntityMappingBuilder!.HasTableName("ALL_TAB_COLUMNS"); + + _EntityMappingBuilder.Property(x => x.ColumnName) + .HasColumnName("COLUMN_NAME"); + + _EntityMappingBuilder.Property(x => x.DataDefault) + .HasColumnName("DATA_DEFAULT"); + + _EntityMappingBuilder.Property(x => x.DataLength) + .HasColumnName("DATA_LENGTH"); + + _EntityMappingBuilder.Property(x => x.DataType) + .HasColumnName("DATA_TYPE"); + + _EntityMappingBuilder.Property(x => x.IdentityColumn) + .HasColumnName("IDENTITY_COLUMN"); + + _EntityMappingBuilder.Property(x => x.Nullable) + .HasColumnName("NULLABLE"); + + _EntityMappingBuilder.Property(x => x.Owner) + .HasColumnName("OWNER"); + + _EntityMappingBuilder.Property(x => x.TableName) + .HasColumnName("TABLE_NAME"); + } +} \ No newline at end of file diff --git a/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllUsersConfiguration.cs b/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllUsersConfiguration.cs new file mode 100644 index 00000000..acdb6c69 --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleAllUsersConfiguration.cs @@ -0,0 +1,17 @@ +using DotNetProjects.Migrator.Framework.Data.Models.Oracle; +using LinqToDB.Mapping; +using Migrator.Tests.Database.Data.Common; + +namespace Migrator.Tests.Database.Data.Mappings.Oracle; + +public class OracleAllUsersConfiguration(FluentMappingBuilder fluentMappingBuilder) + : EntityConfiguration(fluentMappingBuilder) +{ + public override void ConfigureEntity() + { + _EntityMappingBuilder!.HasTableName("ALL_USERS"); + + _EntityMappingBuilder.Property(x => x.UserName) + .HasColumnName("USERNAME"); + } +} diff --git a/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleDBADataFilesConfiguration.cs b/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleDBADataFilesConfiguration.cs new file mode 100644 index 00000000..f7e5b767 --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleDBADataFilesConfiguration.cs @@ -0,0 +1,20 @@ +using DotNetProjects.Migrator.Framework.Data.Models.Oracle; +using LinqToDB.Mapping; +using Migrator.Tests.Database.Data.Common; + +namespace Migrator.Tests.Database.Data.Mappings.Oracle; + +public class OracleDBADataFilesConfiguration(FluentMappingBuilder fluentMappingBuilder) + : EntityConfiguration(fluentMappingBuilder) +{ + public override void ConfigureEntity() + { + _EntityMappingBuilder!.HasTableName("DBA_DATA_FILES"); + + _EntityMappingBuilder.Property(x => x.FileName) + .HasColumnName("FILE_NAME"); + + _EntityMappingBuilder.Property(x => x.TablespaceName) + .HasColumnName("TABLESPACE_NAME"); + } +} diff --git a/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleVSessionConfiguration.cs b/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleVSessionConfiguration.cs new file mode 100644 index 00000000..a5a08343 --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Mappings/Oracle/OracleVSessionConfiguration.cs @@ -0,0 +1,23 @@ +using DotNetProjects.Migrator.Framework.Data.Models.Oracle; +using LinqToDB.Mapping; +using Migrator.Tests.Database.Data.Common; + +namespace Migrator.Tests.Database.Data.Mappings.Oracle; + +public class OracleVSessionConfiguration(FluentMappingBuilder fluentMappingBuilder) + : EntityConfiguration(fluentMappingBuilder) +{ + public override void ConfigureEntity() + { + _EntityMappingBuilder!.HasTableName("V$SESSION"); + + _EntityMappingBuilder.Property(x => x.SerialHashTag) + .HasColumnName("SERIAL#"); + + _EntityMappingBuilder.Property(x => x.SID) + .HasColumnName("SID"); + + _EntityMappingBuilder.Property(x => x.UserName) + .HasColumnName("USERNAME"); + } +} \ No newline at end of file diff --git a/src/Migrator.Tests/Database/Data/Models/Oracle/AllConsColumns.cs b/src/Migrator.Tests/Database/Data/Models/Oracle/AllConsColumns.cs new file mode 100644 index 00000000..cd75ffc7 --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Models/Oracle/AllConsColumns.cs @@ -0,0 +1,32 @@ +namespace DotNetProjects.Migrator.Framework.Data.Models.Oracle; + +/// +/// Represents the Oracle system table ALL_CONS_COLUMNS +/// +public class AllConsColumns +{ + /// + /// Gets or sets the column name. + /// + public string ColumnName { get; set; } + + /// + /// Gets or sets the name of the constraint definition. + /// + public string ConstraintName { get; set; } + + /// + /// Gets or sets + /// + public int Position { get; set; } + + /// + /// Gets or sets the name of the table with the constraint definition. + /// + public string TableName { get; set; } + + /// + /// Gets or sets the owner of the constraint definition. + /// + public string Owner { get; set; } +} \ No newline at end of file diff --git a/src/Migrator.Tests/Database/Data/Models/Oracle/AllConstraints.cs b/src/Migrator.Tests/Database/Data/Models/Oracle/AllConstraints.cs new file mode 100644 index 00000000..cae0f3aa --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Models/Oracle/AllConstraints.cs @@ -0,0 +1,55 @@ +namespace DotNetProjects.Migrator.Framework.Data.Models.Oracle; + +/// +/// Represents the Oracle system table ALL_CONSTRAINTS +/// +public class AllConstraints +{ + /// + /// Gets or sets the name of the constraint definition. + /// + public string ConstraintName { get; set; } + + /// + /// Gets or sets the name of the unique constraint definition for the referenced table (R_CONSTRAINT_NAME) + /// + public string RConstraintName { get; set; } + + /// + /// Gets or sets the constraint type. + /// + /// Type of the constraint definition: + /// + /// C - Check constraint on a table + /// P - Primary key + /// U - Unique key + /// R - Referential integrity + /// V - With check option, on a view + /// O - With read only, on a view + /// H - Hash expression + /// F - Constraint that involves a REF column + /// S - Supplemental logging + /// + /// + public string ConstraintType { get; set; } + + /// + /// Gets or sets the owner of the constraint definition. + /// + public string Owner { get; set; } + + /// + /// Gets or sets the owner of the table referred to in a referential constraint (R_OWNER) + /// + public string ROwner { get; set; } + + /// + /// Gets or set the status. Enforcement status of the constraint: ENABLED / DISABLED + /// + public string Status { get; set; } + + /// + /// Gets or sets the name associated with the table (or view) with the constraint definition. + /// + public string TableName { get; set; } +} diff --git a/src/Migrator.Tests/Database/Data/Models/Oracle/AllTabColumns.cs b/src/Migrator.Tests/Database/Data/Models/Oracle/AllTabColumns.cs new file mode 100644 index 00000000..c9fa72e8 --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Models/Oracle/AllTabColumns.cs @@ -0,0 +1,48 @@ +namespace DotNetProjects.Migrator.Framework.Data.Models.Oracle; + +/// +/// Represents the Oracle system table ALL_TAB_COLUMNS +/// +public class AllTabColumns +{ + /// + /// Gets or sets the column name + /// + public string ColumnName { get; set; } + + /// + /// Gets or sets the DATA_DEFAULT. This returns sth. like "SCHEMA"."ISEQ$$_1234".nextval + /// + public string DataDefault { get; set; } + + /// + /// Gets or sets the length of the column (in bytes) + /// + public string DataLength { get; set; } + + /// + /// Gets or sets the data type of the column + /// + public string DataType { get; set; } + + /// + /// Indicates whether this is an identity column (YES) or not (NO) + /// + public string IdentityColumn { get; set; } + + /// + /// Indicates whether a column allows NULLs. The value is N if there is a NOT NULL constraint on the column or if the column is part of a + /// PRIMARY KEY. The constraint should be in an ENABLE VALIDATE state. + /// + public string Nullable { get; set; } + + /// + /// Gets or sets the name of the table, view, or cluster + /// + public string TableName { get; set; } + + /// + /// Gets or sets the owner of the table, view, or cluster + /// + public string Owner { get; set; } +} \ No newline at end of file diff --git a/src/Migrator.Tests/Database/Data/Models/Oracle/AllUsers.cs b/src/Migrator.Tests/Database/Data/Models/Oracle/AllUsers.cs new file mode 100644 index 00000000..eadf4c6d --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Models/Oracle/AllUsers.cs @@ -0,0 +1,12 @@ +namespace DotNetProjects.Migrator.Framework.Data.Models.Oracle; + +/// +/// Represents the Oracle system table ALL_USERS. +/// +public class AllUsers +{ + /// + /// Gets or sets the name of the user. + /// + public string UserName { get; set; } +} \ No newline at end of file diff --git a/src/Migrator.Tests/Database/Data/Models/Oracle/DBADataFiles.cs b/src/Migrator.Tests/Database/Data/Models/Oracle/DBADataFiles.cs new file mode 100644 index 00000000..a7aa3775 --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Models/Oracle/DBADataFiles.cs @@ -0,0 +1,17 @@ +namespace DotNetProjects.Migrator.Framework.Data.Models.Oracle; + +/// +/// Represents the Oracle system table DBA_DATA_FILES. +/// +public class DBADataFiles +{ + /// + /// Gets or sets the file name. (FILE_NAME) + /// + public string FileName { get; set; } + + /// + /// Gets or sets the tablespace name. (TABLESPACE_NAME) + /// + public string TablespaceName { get; set; } +} diff --git a/src/Migrator.Tests/Database/Data/Models/Oracle/VSession.cs b/src/Migrator.Tests/Database/Data/Models/Oracle/VSession.cs new file mode 100644 index 00000000..a3709d11 --- /dev/null +++ b/src/Migrator.Tests/Database/Data/Models/Oracle/VSession.cs @@ -0,0 +1,19 @@ +namespace DotNetProjects.Migrator.Framework.Data.Models.Oracle; + +public class VSession +{ + /// + /// Gets or sets the "serial#" + /// + public string SerialHashTag { get; set; } + + /// + /// Gets or sets the session id (SID). + /// + public string SID { get; set; } + + /// + /// Gets or sets the user name. + /// + public string UserName { get; set; } +} \ No newline at end of file diff --git a/src/Migrator.Tests/Database/DatabaseIntegrationTestServiceBase.cs b/src/Migrator.Tests/Database/DatabaseIntegrationTestServiceBase.cs index 0b922a3b..00450410 100644 --- a/src/Migrator.Tests/Database/DatabaseIntegrationTestServiceBase.cs +++ b/src/Migrator.Tests/Database/DatabaseIntegrationTestServiceBase.cs @@ -14,7 +14,7 @@ public abstract class DatabaseIntegrationTestServiceBase(IDatabaseNameService da /// Deletes all integration test databases older than the given time span. /// // TODO CK time span! - protected readonly TimeSpan MinTimeSpanBeforeDatabaseDeletion = TimeSpan.FromMinutes(1); // TimeSpan.FromMinutes(60); + protected readonly TimeSpan _MinTimeSpanBeforeDatabaseDeletion = TimeSpan.FromMinutes(1); // TimeSpan.FromMinutes(60); protected IDatabaseNameService DatabaseNameService { get; private set; } = databaseNameService; diff --git a/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/OracleDatabaseIntegrationTestService.cs b/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/OracleDatabaseIntegrationTestService.cs index 4cecd7a7..bef1c98f 100644 --- a/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/OracleDatabaseIntegrationTestService.cs +++ b/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/OracleDatabaseIntegrationTestService.cs @@ -1,10 +1,15 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using DotNetProjects.Migrator.Framework.Data.Common; +using DotNetProjects.Migrator.Framework.Data.Models.Oracle; using LinqToDB; +using LinqToDB.Async; using LinqToDB.Data; +using LinqToDB.Mapping; using Mapster; using Migrator.Tests.Database.DatabaseName.Interfaces; using Migrator.Tests.Database.Interfaces; @@ -16,15 +21,15 @@ namespace Migrator.Tests.Database.DerivedDatabaseIntegrationTestServices; public class OracleDatabaseIntegrationTestService( TimeProvider timeProvider, - IDatabaseNameService databaseNameService - // IImportExportMappingSchemaFactory importExportMappingSchemaFactory - ) + IDatabaseNameService databaseNameService) : DatabaseIntegrationTestServiceBase(databaseNameService), IDatabaseIntegrationTestService { + private const string TableSpacePrefix = "TS_"; private const string UserStringKey = "User Id"; private const string PasswordStringKey = "Password"; private const string ReplaceString = "RandomStringThatIsNotQuotedByTheBuilderDoNotChange"; - // private readonly IImportExportMappingSchemaFactory _importExportMappingSchemaFactory = importExportMappingSchemaFactory; + private readonly MappingSchema _mappingSchema = new MappingSchemaFactory().CreateOracleMappingSchema(); + private Regex _tablespaceRegex = new("^TS_TESTS_"); /// /// Creates an oracle database for test purposes. @@ -90,7 +95,7 @@ public override async Task CreateTestDatabaseAsync(DatabaseConnect { var creationDate = DatabaseNameService.ReadTimeStampFromString(x); - return creationDate.HasValue && creationDate.Value < timeProvider.GetUtcNow().Subtract(MinTimeSpanBeforeDatabaseDeletion); + return creationDate.HasValue && creationDate.Value < timeProvider.GetUtcNow().Subtract(_MinTimeSpanBeforeDatabaseDeletion); }).ToList(); await Parallel.ForEachAsync( @@ -109,6 +114,25 @@ await Parallel.ForEachAsync( }); + // To be on the safe side we check for table spaces used in tests that have not been deleted for any reason (possible connection issues/concurrent deletion attempts - there is + // no transaction for DDL in Oracle etc.). + var tableSpaceNames = await context.GetTable() + .Select(x => x.TablespaceName) + .ToListAsync(cancellationToken); + + var toBeDeletedTableSpaces = tableSpaceNames + .Where(x => + { + var replacedTablespaceString = _tablespaceRegex.Replace(x, ""); + var creationDate = DatabaseNameService.ReadTimeStampFromString(replacedTablespaceString); + return creationDate.HasValue && creationDate.Value < timeProvider.GetUtcNow().Subtract(_MinTimeSpanBeforeDatabaseDeletion); + }); + + foreach (var toBeDeletedTableSpace in toBeDeletedTableSpaces) + { + await context.ExecuteAsync($"DROP TABLESPACE {toBeDeletedTableSpace} INCLUDING CONTENTS AND DATAFILES", cancellationToken); + } + using (context = new DataConnection(dataOptions)) { await context.ExecuteAsync($"CREATE USER \"{tempUserName}\" IDENTIFIED BY \"{tempUserName}\"", cancellationToken); @@ -145,61 +169,56 @@ public override async Task DropDatabaseAsync(DatabaseInfo databaseInfo, Cancella { var creationDate = ReadTimeStampFromDatabaseName(databaseInfo.SchemaName); - var dataOptions = new DataOptions().UseOracle(databaseInfo.DatabaseConnectionConfigMaster.ConnectionString); - // .UseMappingSchema(_importExportMappingSchemaFactory.CreateOracleMappingSchema()); + var dataOptions = new DataOptions().UseOracle(databaseInfo.DatabaseConnectionConfigMaster.ConnectionString) + .UseMappingSchema(_mappingSchema); using var context = new DataConnection(dataOptions); - // var vSessions = await context.GetTable() - // .Where(x => x.UserName == databaseInfo.SchemaName) - // .ToListAsync(cancellationToken); - - // await Parallel.ForEachAsync( - // vSessions, - // new ParallelOptions { MaxDegreeOfParallelism = 3, CancellationToken = cancellationToken }, - // async (x, cancellationTokenInner) => - // { - // using var killSessionContext = new DataConnection(dataOptions); - - // var killStatement = $"ALTER SYSTEM KILL SESSION '{x.SID},{x.SerialHashTag}' IMMEDIATE"; - // try - // { - // await killSessionContext.ExecuteAsync(killStatement, cancellationToken); - - // // Oracle does not close the session immediately as they pretend so we need to wait a while - // // Since this happens only in very rare cases we accept waiting for a while. - // // If nobody connects to the database this will never happen. - // await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken); - // } - // catch - // { - // // Most probably killed by another parallel running integration test. If not, the DROP USER exception will show the details. - // } - // }); - - try - { - await context.ExecuteAsync($"DROP USER \"{databaseInfo.SchemaName}\" CASCADE", cancellationToken); - } - catch + var maxAttempts = 4; + var delayBetweenAttempts = TimeSpan.FromSeconds(1); + + for (var i = 0; i < maxAttempts; i++) { - await Task.Delay(2000, cancellationToken); + try + { + var vSessions = await context.GetTable() + .Where(x => x.UserName == databaseInfo.SchemaName) + .ToListAsync(cancellationToken); - // In next Linq2db version this can be replaced by ...FromSql().First(); - // https://github.com/linq2db/linq2db/issues/2779 - // TODO CK create issue in Redmine and refer to it here - var countList = await context.QueryToListAsync($"SELECT COUNT(*) FROM all_users WHERE username = '{databaseInfo.SchemaName}'", cancellationToken); - var count = countList.First(); + foreach (var session in vSessions) + { + var killStatement = $"ALTER SYSTEM KILL SESSION '{session.SID},{session.SerialHashTag}' IMMEDIATE"; + await context.ExecuteAsync(killStatement, cancellationToken); + } - if (count == 1) - { - throw; + await context.ExecuteAsync($"DROP USER \"{databaseInfo.SchemaName}\" CASCADE", cancellationToken); } - else + catch { - // The user was removed by another asynchronously running test that kicked in earlier. - // That's ok for us as we have achieved the goal. + if (i + 1 == maxAttempts) + { + throw; + } + + var userExists = await context.GetTable().AnyAsync(x => x.UserName == databaseInfo.SchemaName, token: cancellationToken); + + if (!userExists) + { + break; + } } + + await Task.Delay(delayBetweenAttempts, cancellationToken); + + delayBetweenAttempts = delayBetweenAttempts.Add(TimeSpan.FromSeconds(1)); } + + var tablespaceName = $"{TableSpacePrefix}{databaseInfo.SchemaName}"; + + var tablespaces = await context.GetTable().ToListAsync(cancellationToken); + + await context.ExecuteAsync($"DROP TABLESPACE {tablespaceName} INCLUDING CONTENTS AND DATAFILES", cancellationToken); + + await context.ExecuteAsync($"PURGE RECYCLEBIN", cancellationToken); } } \ No newline at end of file diff --git a/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/PostgreSqlDatabaseIntegrationTestService.cs b/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/PostgreSqlDatabaseIntegrationTestService.cs index 1125e7d5..72847417 100644 --- a/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/PostgreSqlDatabaseIntegrationTestService.cs +++ b/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/PostgreSqlDatabaseIntegrationTestService.cs @@ -38,7 +38,7 @@ public override async Task CreateTestDatabaseAsync(DatabaseConnect { var creationDate = DatabaseNameService.ReadTimeStampFromString(x); - return creationDate.HasValue && creationDate.Value < timeProvider.GetUtcNow().Subtract(MinTimeSpanBeforeDatabaseDeletion); + return creationDate.HasValue && creationDate.Value < timeProvider.GetUtcNow().Subtract(_MinTimeSpanBeforeDatabaseDeletion); }).ToList(); foreach (var databaseName in toBeDeletedDatabaseNames) diff --git a/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/SQLiteDatabaseIntegrationTestService.cs b/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/SQLiteDatabaseIntegrationTestService.cs index a4e8770d..7b25159f 100644 --- a/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/SQLiteDatabaseIntegrationTestService.cs +++ b/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/SQLiteDatabaseIntegrationTestService.cs @@ -58,7 +58,7 @@ public override async Task CreateTestDatabaseAsync(DatabaseConnect var creationDate = DatabaseNameService.ReadTimeStampFromString(fileName); - if (creationDate.HasValue && creationDate.Value < timeProvider.GetUtcNow().Subtract(MinTimeSpanBeforeDatabaseDeletion)) + if (creationDate.HasValue && creationDate.Value < timeProvider.GetUtcNow().Subtract(_MinTimeSpanBeforeDatabaseDeletion)) { var builderExistingFile = new SqliteConnectionStringBuilder { DataSource = filePath }; var dataConnectionConfigExistingFile = databaseConnectionConfig.Adapt(); diff --git a/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/SqlServerDatabaseIntegrationTestService.cs b/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/SqlServerDatabaseIntegrationTestService.cs index 7ec8b2a6..29aaaf6a 100644 --- a/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/SqlServerDatabaseIntegrationTestService.cs +++ b/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/SqlServerDatabaseIntegrationTestService.cs @@ -29,7 +29,7 @@ public override async Task CreateTestDatabaseAsync(DatabaseConnect var toBeDeletedDatabaseNames = databaseNames.Where(x => { var creationDate = DatabaseNameService.ReadTimeStampFromString(x); - return creationDate.HasValue && creationDate.Value < timeProvider.GetUtcNow().Subtract(MinTimeSpanBeforeDatabaseDeletion); + return creationDate.HasValue && creationDate.Value < timeProvider.GetUtcNow().Subtract(_MinTimeSpanBeforeDatabaseDeletion); }).ToList(); foreach (var databaseName in toBeDeletedDatabaseNames) diff --git a/src/Migrator.Tests/Migrator.Tests.csproj b/src/Migrator.Tests/Migrator.Tests.csproj index 9d081733..ddc44c7c 100644 --- a/src/Migrator.Tests/Migrator.Tests.csproj +++ b/src/Migrator.Tests/Migrator.Tests.csproj @@ -19,13 +19,13 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Migrator/DotNetProjects.Migrator.csproj b/src/Migrator/DotNetProjects.Migrator.csproj index a17bed76..fd3f7150 100644 --- a/src/Migrator/DotNetProjects.Migrator.csproj +++ b/src/Migrator/DotNetProjects.Migrator.csproj @@ -21,7 +21,7 @@ 9.0.0.0 - + diff --git a/src/Migrator/Providers/Impl/SQLite/SQLiteTransformationProvider.cs b/src/Migrator/Providers/Impl/SQLite/SQLiteTransformationProvider.cs index 882452c6..422e7724 100644 --- a/src/Migrator/Providers/Impl/SQLite/SQLiteTransformationProvider.cs +++ b/src/Migrator/Providers/Impl/SQLite/SQLiteTransformationProvider.cs @@ -1236,6 +1236,7 @@ public override Column[] GetColumns(string tableName) var hasCompoundPrimaryKey = tableInfoPrimaryKeys.Count > 1; + // Implicit in SQLite if (columnTableInfoItem.Type == "INTEGER" && columnTableInfoItem.Pk == 1 && !hasCompoundPrimaryKey) { column.ColumnProperty |= ColumnProperty.Identity; From 09199b0d7379ed443d08f3a140ac88b896857967 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Wed, 29 Oct 2025 17:09:27 +0100 Subject: [PATCH 07/19] Update --- src/Migrator.Tests/Migrator.Tests.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Migrator.Tests/Migrator.Tests.csproj b/src/Migrator.Tests/Migrator.Tests.csproj index ddc44c7c..456eae07 100644 --- a/src/Migrator.Tests/Migrator.Tests.csproj +++ b/src/Migrator.Tests/Migrator.Tests.csproj @@ -19,13 +19,13 @@ + + - - all runtime; build; native; contentfiles; analyzers; buildtransitive From bd4dc19bc33e4b18b5dcdb8052b58d0b865dd299 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Wed, 29 Oct 2025 17:18:12 +0100 Subject: [PATCH 08/19] Update keys --- .github/workflows/dotnetpull.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dotnetpull.yml b/.github/workflows/dotnetpull.yml index 2ed819de..e2d41445 100644 --- a/.github/workflows/dotnetpull.yml +++ b/.github/workflows/dotnetpull.yml @@ -71,7 +71,7 @@ jobs: - name: Install SQLCMD tools run: | curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list + curl https://packages.microsoft.com/config/ubuntu/25.10/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list sudo apt-get update sudo ACCEPT_EULA=Y apt-get install -y mssql-tools unixodbc-dev echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bashrc From 62a74feec41704a400fc6af42091f7f07596ab58 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Wed, 29 Oct 2025 17:33:07 +0100 Subject: [PATCH 09/19] Install Microsoft GPG apt-key --- .github/workflows/dotnetpull.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dotnetpull.yml b/.github/workflows/dotnetpull.yml index e2d41445..e45bcdfa 100644 --- a/.github/workflows/dotnetpull.yml +++ b/.github/workflows/dotnetpull.yml @@ -68,11 +68,16 @@ jobs: with: dotnet-version: | 9.0.x - - name: Install SQLCMD tools + - name: Install Microsoft GPG apt-key + run: | + curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg + sudo install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ + - name: Add Microsoft SQL Server repo run: | - curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - curl https://packages.microsoft.com/config/ubuntu/25.10/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list + sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/ubuntu/22.04/prod jammy main" > /etc/apt/sources.list/d/mssql-release.list' sudo apt-get update + - name: Install SQLCMD tools + run: | sudo ACCEPT_EULA=Y apt-get install -y mssql-tools unixodbc-dev echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bashrc source ~/.bashrc From eee61d6f64e60e12cebb01100239a8bf6008bd61 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Wed, 29 Oct 2025 17:54:24 +0100 Subject: [PATCH 10/19] Download microsoft.asc --- .github/workflows/dotnetpull.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dotnetpull.yml b/.github/workflows/dotnetpull.yml index e45bcdfa..16ad3bba 100644 --- a/.github/workflows/dotnetpull.yml +++ b/.github/workflows/dotnetpull.yml @@ -70,7 +70,8 @@ jobs: 9.0.x - name: Install Microsoft GPG apt-key run: | - curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg + wget https://packages.microsoft.com/keys/microsoft.asc -O microsoft.gpg + gpg --dearmor < microsoft.asc > microsoft.gpg sudo install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ - name: Add Microsoft SQL Server repo run: | From fc5a4024592f65289c24eae8fcbefaa0ec546732 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Wed, 29 Oct 2025 22:16:06 +0100 Subject: [PATCH 11/19] Update install keys --- .github/workflows/dotnetpull.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dotnetpull.yml b/.github/workflows/dotnetpull.yml index 16ad3bba..6d68bcb5 100644 --- a/.github/workflows/dotnetpull.yml +++ b/.github/workflows/dotnetpull.yml @@ -70,12 +70,13 @@ jobs: 9.0.x - name: Install Microsoft GPG apt-key run: | - wget https://packages.microsoft.com/keys/microsoft.asc -O microsoft.gpg - gpg --dearmor < microsoft.asc > microsoft.gpg - sudo install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ + wget https://packages.microsoft.com/keys/microsoft.asc -O microsoft.asc + gpg --dearmor microsoft.asc + chmod 644 microsoft.asc.gpg + sudo mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/microsoft.gpg - name: Add Microsoft SQL Server repo run: | - sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/ubuntu/22.04/prod jammy main" > /etc/apt/sources.list/d/mssql-release.list' + sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null < Date: Wed, 29 Oct 2025 22:40:13 +0100 Subject: [PATCH 12/19] Update --- .github/workflows/dotnetpull.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dotnetpull.yml b/.github/workflows/dotnetpull.yml index 6d68bcb5..73e1974e 100644 --- a/.github/workflows/dotnetpull.yml +++ b/.github/workflows/dotnetpull.yml @@ -76,7 +76,8 @@ jobs: sudo mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/microsoft.gpg - name: Add Microsoft SQL Server repo run: | - sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null < Date: Wed, 29 Oct 2025 22:49:20 +0100 Subject: [PATCH 13/19] Update --- .github/workflows/dotnetpull.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/dotnetpull.yml b/.github/workflows/dotnetpull.yml index 73e1974e..981f40da 100644 --- a/.github/workflows/dotnetpull.yml +++ b/.github/workflows/dotnetpull.yml @@ -70,13 +70,9 @@ jobs: 9.0.x - name: Install Microsoft GPG apt-key run: | - wget https://packages.microsoft.com/keys/microsoft.asc -O microsoft.asc - gpg --dearmor microsoft.asc - chmod 644 microsoft.asc.gpg - sudo mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/microsoft.gpg + curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - name: Add Microsoft SQL Server repo run: | - sudo apt-key add /etc/apt/trusted.gpg.d/microsoft.gpg curl https://packages.microsoft.com/config/ubuntu/25.10/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list sudo apt-get update - name: Install SQLCMD tools From 96e14b88e2db6d6ecc12a3c475454642598752c8 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Wed, 29 Oct 2025 22:56:06 +0100 Subject: [PATCH 14/19] Downgrade ubuntu version - key not present for latest v --- .github/workflows/dotnetpull.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dotnetpull.yml b/.github/workflows/dotnetpull.yml index 981f40da..f8ca9b0b 100644 --- a/.github/workflows/dotnetpull.yml +++ b/.github/workflows/dotnetpull.yml @@ -73,7 +73,7 @@ jobs: curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - name: Add Microsoft SQL Server repo run: | - curl https://packages.microsoft.com/config/ubuntu/25.10/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list + curl https://packages.microsoft.com/config/ubuntu/24.10/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list sudo apt-get update - name: Install SQLCMD tools run: | From 4201bc48436b899e83ba4c96a495346ef4ddad41 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Wed, 29 Oct 2025 23:25:50 +0100 Subject: [PATCH 15/19] Use ubuntu-22.04 instead of latest due to issues with not yet supported latest version (key) --- .github/workflows/dotnetpull.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dotnetpull.yml b/.github/workflows/dotnetpull.yml index f8ca9b0b..f25eef79 100644 --- a/.github/workflows/dotnetpull.yml +++ b/.github/workflows/dotnetpull.yml @@ -7,7 +7,7 @@ on: branches: [master] jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 services: sqlserver: image: mcr.microsoft.com/mssql/server:2019-latest @@ -70,10 +70,13 @@ jobs: 9.0.x - name: Install Microsoft GPG apt-key run: | - curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - + wget https://packages.microsoft.com/keys/microsoft.asc -O microsoft.asc + gpg --dearmor microsoft.asc + chmod 644 microsoft.asc.gpg + sudo mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/microsoft.gpg - name: Add Microsoft SQL Server repo run: | - curl https://packages.microsoft.com/config/ubuntu/24.10/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list + echo "deb [arch=amd64] https://packages.microsoft.com/config/ubuntu/22.04/prod jammy main" sudo apt-get update - name: Install SQLCMD tools run: | From 2b98a40df46c12c0d1a56c50689730036267839b Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Thu, 30 Oct 2025 08:32:54 +0100 Subject: [PATCH 16/19] Update Oracle --- .../OracleDatabaseIntegrationTestService.cs | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/OracleDatabaseIntegrationTestService.cs b/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/OracleDatabaseIntegrationTestService.cs index bef1c98f..4d2af622 100644 --- a/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/OracleDatabaseIntegrationTestService.cs +++ b/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/OracleDatabaseIntegrationTestService.cs @@ -114,27 +114,27 @@ await Parallel.ForEachAsync( }); - // To be on the safe side we check for table spaces used in tests that have not been deleted for any reason (possible connection issues/concurrent deletion attempts - there is - // no transaction for DDL in Oracle etc.). - var tableSpaceNames = await context.GetTable() - .Select(x => x.TablespaceName) - .ToListAsync(cancellationToken); - - var toBeDeletedTableSpaces = tableSpaceNames - .Where(x => - { - var replacedTablespaceString = _tablespaceRegex.Replace(x, ""); - var creationDate = DatabaseNameService.ReadTimeStampFromString(replacedTablespaceString); - return creationDate.HasValue && creationDate.Value < timeProvider.GetUtcNow().Subtract(_MinTimeSpanBeforeDatabaseDeletion); - }); - - foreach (var toBeDeletedTableSpace in toBeDeletedTableSpaces) - { - await context.ExecuteAsync($"DROP TABLESPACE {toBeDeletedTableSpace} INCLUDING CONTENTS AND DATAFILES", cancellationToken); - } - using (context = new DataConnection(dataOptions)) { + // To be on the safe side we check for table spaces used in tests that have not been deleted for any reason (possible connection issues/concurrent deletion attempts - there is + // no transaction for DDL in Oracle etc.). + var tableSpaceNames = await context.GetTable() + .Select(x => x.TablespaceName) + .ToListAsync(cancellationToken); + + var toBeDeletedTableSpaces = tableSpaceNames + .Where(x => + { + var replacedTablespaceString = _tablespaceRegex.Replace(x, ""); + var creationDate = DatabaseNameService.ReadTimeStampFromString(replacedTablespaceString); + return creationDate.HasValue && creationDate.Value < timeProvider.GetUtcNow().Subtract(_MinTimeSpanBeforeDatabaseDeletion); + }); + + foreach (var toBeDeletedTableSpace in toBeDeletedTableSpaces) + { + await context.ExecuteAsync($"DROP TABLESPACE {toBeDeletedTableSpace} INCLUDING CONTENTS AND DATAFILES", cancellationToken); + } + await context.ExecuteAsync($"CREATE USER \"{tempUserName}\" IDENTIFIED BY \"{tempUserName}\"", cancellationToken); var privileges = new[] @@ -172,11 +172,11 @@ public override async Task DropDatabaseAsync(DatabaseInfo databaseInfo, Cancella var dataOptions = new DataOptions().UseOracle(databaseInfo.DatabaseConnectionConfigMaster.ConnectionString) .UseMappingSchema(_mappingSchema); - using var context = new DataConnection(dataOptions); - var maxAttempts = 4; var delayBetweenAttempts = TimeSpan.FromSeconds(1); + using var context = new DataConnection(dataOptions); + for (var i = 0; i < maxAttempts; i++) { try From fea7d47f2fdd6c57b93bc46f59f330d2a0e65ac7 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Thu, 30 Oct 2025 08:41:49 +0100 Subject: [PATCH 17/19] use schema --- .../OracleDatabaseIntegrationTestService.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/OracleDatabaseIntegrationTestService.cs b/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/OracleDatabaseIntegrationTestService.cs index 4d2af622..9b5d1fee 100644 --- a/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/OracleDatabaseIntegrationTestService.cs +++ b/src/Migrator.Tests/Database/DerivedDatabaseIntegrationTestServices/OracleDatabaseIntegrationTestService.cs @@ -84,7 +84,8 @@ public override async Task CreateTestDatabaseAsync(DatabaseConnect List userNames; - var dataOptions = new DataOptions().UseOracle(databaseConnectionConfig.ConnectionString); + var dataOptions = new DataOptions().UseOracle(databaseConnectionConfig.ConnectionString) + .UseMappingSchema(_mappingSchema); using (context = new DataConnection(dataOptions)) { From db1b8deb79ab4633bb8128ebbe960916f4bec310 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Thu, 30 Oct 2025 08:48:49 +0100 Subject: [PATCH 18/19] cleanup --- .github/workflows/sql/oracle.sql | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/sql/oracle.sql b/.github/workflows/sql/oracle.sql index f4e7eba2..07aee535 100644 --- a/.github/workflows/sql/oracle.sql +++ b/.github/workflows/sql/oracle.sql @@ -5,25 +5,15 @@ alter session set container = freepdb1; create user k identified by k; grant select_catalog_role to k; - grant create tablespace to k; - grant drop tablespace to k; - grant create user to k; - grant drop user to k; - grant create session to k with admin option; - grant resource to k with admin option; - grant connect to k with admin option; - grant unlimited tablespace to k with admin option; - grant select on v_$session to k with grant option - grant alter system to k exit; \ No newline at end of file From 7fea1640e2c41c9c18f722246cc14f3ac38410b8 Mon Sep 17 00:00:00 2001 From: JaBistDuNarrisch Date: Thu, 30 Oct 2025 08:56:41 +0100 Subject: [PATCH 19/19] Update commands --- .github/workflows/sql/oracle.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sql/oracle.sql b/.github/workflows/sql/oracle.sql index 07aee535..5dc26fc6 100644 --- a/.github/workflows/sql/oracle.sql +++ b/.github/workflows/sql/oracle.sql @@ -13,7 +13,7 @@ grant create session to k with admin option; grant resource to k with admin option; grant connect to k with admin option; grant unlimited tablespace to k with admin option; -grant select on v_$session to k with grant option -grant alter system to k +grant select on v_$session to k with grant option; +grant alter system to k; exit; \ No newline at end of file