From f41cd416b7501fa13e3eb5c44c2ff6cb3eef0d51 Mon Sep 17 00:00:00 2001 From: "Erik A. Brandstadmoen" Date: Mon, 10 Apr 2023 22:06:48 +0200 Subject: [PATCH 1/4] Working on refining replacement --- .../BatchSplitterReplacer_.cs | 6 +- .../Statement_Splitting/StatementSplitter_.cs | 19 +++++- .../With_batch_separator.cs | 63 +++++++++++++++++++ .../OracleSplitterContext.cs | 18 +----- grate/Infrastructure/OracleSyntax.cs | 5 +- 5 files changed, 88 insertions(+), 23 deletions(-) create mode 100644 grate.unittests/Oracle/Running_MigrationScripts/With_batch_separator.cs diff --git a/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/BatchSplitterReplacer_.cs b/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/BatchSplitterReplacer_.cs index 0909624b..ba22dbc3 100644 --- a/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/BatchSplitterReplacer_.cs +++ b/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/BatchSplitterReplacer_.cs @@ -264,8 +264,10 @@ public void slash_with_comment_after() [Test] public void slash_with_semicolon_directly_after() { - string sql_to_match = "jalla /;"; - string expected_scrubbed = "jalla " + Batch_terminator_replacement_string + ";"; + string sql_to_match = @"jalla; +/"; + string expected_scrubbed = "jalla" + Batch_terminator_replacement_string + @" +" + Batch_terminator_replacement_string; TestContext.WriteLine(sql_to_match); string sql_statement_scrubbed = Replacer.Replace(sql_to_match); Assert.AreEqual(expected_scrubbed, sql_statement_scrubbed); diff --git a/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/StatementSplitter_.cs b/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/StatementSplitter_.cs index d5bf5a6c..73610a54 100644 --- a/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/StatementSplitter_.cs +++ b/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/StatementSplitter_.cs @@ -1,4 +1,5 @@ -using FluentAssertions; +using System.Linq; +using FluentAssertions; using grate.Infrastructure; using grate.Migration; using Microsoft.Extensions.Logging.Abstractions; @@ -15,7 +16,7 @@ public class StatementSplitter_ private static readonly StatementSplitter Splitter = new(Database.StatementSeparatorRegex); [Test] - public void Splits_and_removes_GO_statements() + public void Splits_and_removes_slashes_and_semicolon() { var original = @" SELECT * FROM v$version WHERE banner LIKE 'Oracle%'; @@ -27,6 +28,20 @@ SELECT 1 var batches = Splitter.Split(original); batches.Should().HaveCount(2); + batches.First().Should().NotEndWith(";"); + } + + [Test] + public void Splits_and_removes_semicolon() + { + var original = @" +SELECT * FROM v$version WHERE banner LIKE 'Oracle%'; +SELECT 1 +"; + var batches = Splitter.Split(original).ToArray(); + + batches.Should().HaveCount(2); + batches.First().Should().NotEndWith(";"); } } diff --git a/grate.unittests/Oracle/Running_MigrationScripts/With_batch_separator.cs b/grate.unittests/Oracle/Running_MigrationScripts/With_batch_separator.cs new file mode 100644 index 00000000..39dff3dd --- /dev/null +++ b/grate.unittests/Oracle/Running_MigrationScripts/With_batch_separator.cs @@ -0,0 +1,63 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using FluentAssertions; +using grate.Configuration; +using grate.Migration; +using grate.unittests.TestInfrastructure; +using NUnit.Framework; +using static grate.Configuration.KnownFolderKeys; + +namespace grate.unittests.Oracle.Running_MigrationScripts; + +[TestFixture] +[Category("Oracle")] +// ReSharper disable once InconsistentNaming +public class With_batch_separator: Generic.Running_MigrationScripts.MigrationsScriptsBase +{ + protected override IGrateTestContext Context => GrateTestContext.Oracle; + + [Test()] + public async Task Separates_multiple_statements() + { + var db = TestConfig.RandomDatabase(); + + var parent = TestConfig.CreateRandomTempDirectory(); + var knownFolders = FoldersConfiguration.Default(null); + GrateMigrator? migrator; + + + var path = new DirectoryInfo(Path.Combine(parent.ToString(), knownFolders[Up]!.Path)); + const string filename = "multiple_statements.sql"; + + const string fileContent = @" +create table table_one ( + col number +); +/ + +create table table_two ( + col number +) +"; + + WriteSql(path, filename, fileContent); + + await using (migrator = Context.GetMigrator(db, parent, knownFolders)) + { + await migrator.Migrate(); + } + + string[] scripts; + string sql = $"SELECT text_of_script FROM {Context.Syntax.TableWithSchema("grate", "ScriptsRun")}"; + + await using (var conn = Context.CreateDbConnection(db)) + { + scripts = (await conn.QueryAsync(sql)).ToArray(); + } + + scripts.Should().HaveCount(2); + } + +} diff --git a/grate.unittests/TestInfrastructure/OracleSplitterContext.cs b/grate.unittests/TestInfrastructure/OracleSplitterContext.cs index 5c5e856d..96c4a3af 100644 --- a/grate.unittests/TestInfrastructure/OracleSplitterContext.cs +++ b/grate.unittests/TestInfrastructure/OracleSplitterContext.cs @@ -30,14 +30,6 @@ public static class FullSplitter /* / */ -BOB7 - -/* - -/ - -*/ - BOB8 -- @@ -121,14 +113,6 @@ INSERT [dbo].[Foo] ([Bar]) VALUES (N'/ speed racer, / speed racer, / speed racer /* / */ -BOB7 - -/* - -/ - -*/ - BOB8 -- @@ -164,7 +148,7 @@ yeppsasd decimal(20, 6) NULL, uhuhhh datetime NULL, slsald varchar(15) NULL, uhasdf varchar(15) NULL, - daf_asdfasdf DECIMAL(20,6) NULL; + daf_asdfasdf DECIMAL(20,6) NULL " + StatementSplitter.BatchTerminatorReplacementString + @" EXEC @ReturnCode = msdb.dbo.sp_add_jobstep @job_id=@jobId, @step_name=N'Daily job', diff --git a/grate/Infrastructure/OracleSyntax.cs b/grate/Infrastructure/OracleSyntax.cs index dbc1a99b..fba90746 100644 --- a/grate/Infrastructure/OracleSyntax.cs +++ b/grate/Infrastructure/OracleSyntax.cs @@ -11,8 +11,9 @@ public string StatementSeparatorRegex const string strings = @"(?'[^']*')"; const string dashComments = @"(?--.*$)"; const string starComments = @"(?/\*[\S\s]*?\*/)"; - const string separator = @"(?^|\s)(?/)(?\s|;|$)"; - return strings + "|" + dashComments + "|" + starComments + "|" + separator; + const string batchSeparator = @"(?.*)(?;)(?\s|$)"; + const string sqlPlusExecuteCommand = @"(?^|\s)(?/)(?\s|$)"; + return strings + "|" + dashComments + "|" + starComments + "|" + batchSeparator + "|" + sqlPlusExecuteCommand; } } From 9b4f5b133432d035455240966fb4f00c5e611b2f Mon Sep 17 00:00:00 2001 From: "Erik A. Brandstadmoen" Date: Sun, 4 Jun 2023 00:33:42 +0200 Subject: [PATCH 2/4] Fixed batch splitter regex, and deleted invalid unit test --- .../Statement_Splitting/StatementSplitter_.cs | 19 ++++++ .../With_batch_separator.cs | 63 ------------------- grate/Infrastructure/OracleSyntax.cs | 2 +- 3 files changed, 20 insertions(+), 64 deletions(-) delete mode 100644 grate.unittests/Oracle/Running_MigrationScripts/With_batch_separator.cs diff --git a/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/StatementSplitter_.cs b/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/StatementSplitter_.cs index 73610a54..73fc3b44 100644 --- a/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/StatementSplitter_.cs +++ b/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/StatementSplitter_.cs @@ -31,6 +31,25 @@ SELECT 1 batches.First().Should().NotEndWith(";"); } + [Test] + public void Splits_and_removes_slashes_and_semicolon_2() + { + var original = @" + create table table_one ( + col number + ); + / + + create table table_two ( + col number + ) +"; + var batches = Splitter.Split(original); + + batches.Should().HaveCount(2); + batches.First().Should().NotEndWith(";"); + } + [Test] public void Splits_and_removes_semicolon() { diff --git a/grate.unittests/Oracle/Running_MigrationScripts/With_batch_separator.cs b/grate.unittests/Oracle/Running_MigrationScripts/With_batch_separator.cs deleted file mode 100644 index 39dff3dd..00000000 --- a/grate.unittests/Oracle/Running_MigrationScripts/With_batch_separator.cs +++ /dev/null @@ -1,63 +0,0 @@ -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using FluentAssertions; -using grate.Configuration; -using grate.Migration; -using grate.unittests.TestInfrastructure; -using NUnit.Framework; -using static grate.Configuration.KnownFolderKeys; - -namespace grate.unittests.Oracle.Running_MigrationScripts; - -[TestFixture] -[Category("Oracle")] -// ReSharper disable once InconsistentNaming -public class With_batch_separator: Generic.Running_MigrationScripts.MigrationsScriptsBase -{ - protected override IGrateTestContext Context => GrateTestContext.Oracle; - - [Test()] - public async Task Separates_multiple_statements() - { - var db = TestConfig.RandomDatabase(); - - var parent = TestConfig.CreateRandomTempDirectory(); - var knownFolders = FoldersConfiguration.Default(null); - GrateMigrator? migrator; - - - var path = new DirectoryInfo(Path.Combine(parent.ToString(), knownFolders[Up]!.Path)); - const string filename = "multiple_statements.sql"; - - const string fileContent = @" -create table table_one ( - col number -); -/ - -create table table_two ( - col number -) -"; - - WriteSql(path, filename, fileContent); - - await using (migrator = Context.GetMigrator(db, parent, knownFolders)) - { - await migrator.Migrate(); - } - - string[] scripts; - string sql = $"SELECT text_of_script FROM {Context.Syntax.TableWithSchema("grate", "ScriptsRun")}"; - - await using (var conn = Context.CreateDbConnection(db)) - { - scripts = (await conn.QueryAsync(sql)).ToArray(); - } - - scripts.Should().HaveCount(2); - } - -} diff --git a/grate/Infrastructure/OracleSyntax.cs b/grate/Infrastructure/OracleSyntax.cs index fba90746..6560069a 100644 --- a/grate/Infrastructure/OracleSyntax.cs +++ b/grate/Infrastructure/OracleSyntax.cs @@ -12,7 +12,7 @@ public string StatementSeparatorRegex const string dashComments = @"(?--.*$)"; const string starComments = @"(?/\*[\S\s]*?\*/)"; const string batchSeparator = @"(?.*)(?;)(?\s|$)"; - const string sqlPlusExecuteCommand = @"(?^|\s)(?/)(?\s|$)"; + const string sqlPlusExecuteCommand = @"(?^|\s)(?\/|;)(?\s|$)"; return strings + "|" + dashComments + "|" + starComments + "|" + batchSeparator + "|" + sqlPlusExecuteCommand; } } From 7b6f8a06be09d1ee4ca0695294f7a4c5a3a4bc25 Mon Sep 17 00:00:00 2001 From: "Erik A. Brandstadmoen" Date: Sun, 4 Jun 2023 01:14:51 +0200 Subject: [PATCH 3/4] Fixed double replacement tokens --- .../Statement_Splitting/BatchSplitterReplacer_.cs | 4 ++-- grate/Infrastructure/BatchSplitterReplacer.cs | 15 +++++++++++++-- grate/Infrastructure/OracleSyntax.cs | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/BatchSplitterReplacer_.cs b/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/BatchSplitterReplacer_.cs index ba22dbc3..3ca6dc11 100644 --- a/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/BatchSplitterReplacer_.cs +++ b/grate.unittests/Basic/Infrastructure/Oracle/Statement_Splitting/BatchSplitterReplacer_.cs @@ -262,11 +262,11 @@ public void slash_with_comment_after() } [Test] - public void slash_with_semicolon_directly_after() + public void slash_with_semicolon_directly_after_yields_just_one_splitter() { string sql_to_match = @"jalla; /"; - string expected_scrubbed = "jalla" + Batch_terminator_replacement_string + @" + string expected_scrubbed = "jalla" + @" " + Batch_terminator_replacement_string; TestContext.WriteLine(sql_to_match); string sql_statement_scrubbed = Replacer.Replace(sql_to_match); diff --git a/grate/Infrastructure/BatchSplitterReplacer.cs b/grate/Infrastructure/BatchSplitterReplacer.cs index 23e38232..c87ddf3b 100644 --- a/grate/Infrastructure/BatchSplitterReplacer.cs +++ b/grate/Infrastructure/BatchSplitterReplacer.cs @@ -14,7 +14,18 @@ public BatchSplitterReplacer(string pattern, string replacement) _regex = new Regex(pattern, IgnoreCase | Multiline); } - public string Replace(string text) => _regex.Replace(text, ReplaceBatchSeparator); + public string Replace(string text) + { + var replace = _regex.Replace(text, ReplaceBatchSeparator); + + // Combine multiple consecutive replacement tokens with one (needed for Oracle, if ; and / are on separate lines) + replace = Regex.Replace( + replace, + Regex.Escape(Replacement) + "(\\s*)" + Regex.Escape(Replacement), + "$1" + Replacement); + + return replace; + } private string ReplaceBatchSeparator(Match match) { @@ -22,4 +33,4 @@ private string ReplaceBatchSeparator(Match match) var replacement = groups["BATCHSPLITTER"].Success ? Replacement : string.Empty; return groups["KEEP1"].Value + replacement + groups["KEEP2"].Value; } -} \ No newline at end of file +} diff --git a/grate/Infrastructure/OracleSyntax.cs b/grate/Infrastructure/OracleSyntax.cs index 6560069a..d6472c84 100644 --- a/grate/Infrastructure/OracleSyntax.cs +++ b/grate/Infrastructure/OracleSyntax.cs @@ -12,7 +12,7 @@ public string StatementSeparatorRegex const string dashComments = @"(?--.*$)"; const string starComments = @"(?/\*[\S\s]*?\*/)"; const string batchSeparator = @"(?.*)(?;)(?\s|$)"; - const string sqlPlusExecuteCommand = @"(?^|\s)(?\/|;)(?\s|$)"; + const string sqlPlusExecuteCommand = @"(?^|\s)(?\/|;)(?\s*|$|\n)"; return strings + "|" + dashComments + "|" + starComments + "|" + batchSeparator + "|" + sqlPlusExecuteCommand; } } From 08f1a1a46a8bd4957d91a94d1acc74496dfc201b Mon Sep 17 00:00:00 2001 From: "Erik A. Brandstadmoen" Date: Sun, 13 Aug 2023 11:47:02 +0200 Subject: [PATCH 4/4] WIP --- grate.unittests/Generic/GenericDatabase.cs | 13 +++++++------ grate/Infrastructure/OracleSyntax.cs | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/grate.unittests/Generic/GenericDatabase.cs b/grate.unittests/Generic/GenericDatabase.cs index aa6586b9..c999571f 100644 --- a/grate.unittests/Generic/GenericDatabase.cs +++ b/grate.unittests/Generic/GenericDatabase.cs @@ -47,15 +47,16 @@ public virtual async Task Is_created_with_custom_script_if_custom_create_databas var customScript = Context.Syntax.CreateDatabase(scriptedDatabase, password); TestConfig.WriteContent(Wrap(config.SqlFilesDirectory, config.Folders?.CreateDatabase?.Path), "createDatabase.sql", customScript); - try - { + //try + //{ await using var migrator = GetMigrator(config); await migrator.Migrate(); - } - catch (DbException) - { + //} + //catch (DbException e) + //{ + //var s = e.Message; //Do nothing because database name is wrong due to custom script - } + //} File.Delete(Path.Join(Wrap(config.SqlFilesDirectory, config.Folders?.CreateDatabase?.Path).ToString(), "createDatabase.sql")); diff --git a/grate/Infrastructure/OracleSyntax.cs b/grate/Infrastructure/OracleSyntax.cs index d6472c84..5ede35b4 100644 --- a/grate/Infrastructure/OracleSyntax.cs +++ b/grate/Infrastructure/OracleSyntax.cs @@ -12,7 +12,8 @@ public string StatementSeparatorRegex const string dashComments = @"(?--.*$)"; const string starComments = @"(?/\*[\S\s]*?\*/)"; const string batchSeparator = @"(?.*)(?;)(?\s|$)"; - const string sqlPlusExecuteCommand = @"(?^|\s)(?\/|;)(?\s*|$|\n)"; + //const string sqlPlusExecuteCommand = @"(?^|\s)(?\/|;)(?\s*|$|\n)"; + const string sqlPlusExecuteCommand = @"(?^|\s)(?\/)(?\s|$)"; return strings + "|" + dashComments + "|" + starComments + "|" + batchSeparator + "|" + sqlPlusExecuteCommand; } }