From 85f624dfe9ce123da2f4813fab0633abd3ccd548 Mon Sep 17 00:00:00 2001 From: Deep1998 Date: Tue, 2 Nov 2021 20:21:36 +0530 Subject: [PATCH 1/4] Updated spanner emulator version --- .github/workflows/integration-tests-against-emulator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-against-emulator.yaml b/.github/workflows/integration-tests-against-emulator.yaml index b28ec38740..9090436f0b 100644 --- a/.github/workflows/integration-tests-against-emulator.yaml +++ b/.github/workflows/integration-tests-against-emulator.yaml @@ -31,7 +31,7 @@ jobs: services: spanner_emulator: - image: gcr.io/cloud-spanner-emulator/emulator:1.2.0 + image: gcr.io/cloud-spanner-emulator/emulator:1.3.0 ports: - 9010:9010 - 9020:9020 From fae4956bc862f811c3d5ec1dd23ed77b1e08fa62 Mon Sep 17 00:00:00 2001 From: Deep1998 Date: Tue, 2 Nov 2021 21:25:26 +0530 Subject: [PATCH 2/4] Add integration tests for json --- .../integration-tests-against-emulator.yaml | 2 +- test_data/mysqldump.test.out | 26 +++++++++++++++++++ testing/mysql/integration_test.go | 24 +++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-against-emulator.yaml b/.github/workflows/integration-tests-against-emulator.yaml index 9090436f0b..96688fda07 100644 --- a/.github/workflows/integration-tests-against-emulator.yaml +++ b/.github/workflows/integration-tests-against-emulator.yaml @@ -31,7 +31,7 @@ jobs: services: spanner_emulator: - image: gcr.io/cloud-spanner-emulator/emulator:1.3.0 + image: gcr.io/cloud-spanner-emulator/emulator:1.4.0 ports: - 9010:9010 - 9020:9020 diff --git a/test_data/mysqldump.test.out b/test_data/mysqldump.test.out index 06c6980718..94b65cca91 100644 --- a/test_data/mysqldump.test.out +++ b/test_data/mysqldump.test.out @@ -88,6 +88,32 @@ LOCK TABLES `products` WRITE; INSERT INTO `products` VALUES ('abc-123','Blue suede shoes',141.99,'2020-06-06'),('axd-673','Antique typewriter',99.99,'2020-06-07'),('zxi-631','Glass vase',55.50,'2020-06-10'); /*!40000 ALTER TABLE `products` ENABLE KEYS */; UNLOCK TABLES; + +-- +-- Table structure for table `customers` +-- + +DROP TABLE IF EXISTS `customers`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `customers` ( + `c_id` varchar(20) NOT NULL, + `customer_profile` JSON DEFAULT NULL, + PRIMARY KEY (`c_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `customers` +-- + +LOCK TABLES `customers` WRITE; +/*!40000 ALTER TABLE `customers` DISABLE KEYS */; +INSERT INTO `customers` VALUES +('svd-124','{"first_name": "Lola", "last_name": "Dog", "location": "NYC", "online" : true, "friends" : 547}'), +('tel-595','{"first_name": "Ernie", "status": "Looking for treats", "location" : "Brooklyn"}'); +/*!40000 ALTER TABLE `customers` ENABLE KEYS */; +UNLOCK TABLES; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; diff --git a/testing/mysql/integration_test.go b/testing/mysql/integration_test.go index c1367183f5..acd12983b9 100644 --- a/testing/mysql/integration_test.go +++ b/testing/mysql/integration_test.go @@ -22,12 +22,14 @@ import ( "log" "os" "path/filepath" + "strings" "testing" "time" "github.com/cloudspannerecosystem/harbourbridge/common/constants" "github.com/cloudspannerecosystem/harbourbridge/common/utils" "github.com/cloudspannerecosystem/harbourbridge/testing/common" + "github.com/stretchr/testify/assert" "cloud.google.com/go/spanner" database "cloud.google.com/go/spanner/admin/database/apiv1" @@ -341,6 +343,7 @@ func checkResults(t *testing.T, dbURI string) { defer client.Close() checkBigInt(ctx, t, client) + checkJson(ctx, t, client, dbURI) } func checkBigInt(ctx context.Context, t *testing.T, client *spanner.Client) { @@ -364,6 +367,27 @@ func checkBigInt(ctx context.Context, t *testing.T, client *spanner.Client) { } } +func checkJson(ctx context.Context, t *testing.T, client *spanner.Client, dbURI string) { + resp, err := databaseAdmin.GetDatabaseDdl(ctx, &databasepb.GetDatabaseDdlRequest{Database: dbURI}) + if err != nil { + t.Fatalf("Could not read DDL from database %s: %v", dbURI, err) + } + for _, stmt := range resp.Statements { + if strings.Contains(stmt, "CREATE TABLE customers") { + assert.True(t, strings.Contains(stmt, "customer_profile JSON")) + } + } + stmt := spanner.Statement{ + SQL: `SELECT COUNT(*) FROM customers`, + } + iter := client.Single().Query(ctx, stmt) + defer iter.Stop() + row, _ := iter.Next() + var rowCount int64 = 0 + row.Columns(&rowCount) + assert.Equal(t, 2, rowCount) +} + func onlyRunForEmulatorTest(t *testing.T) { if os.Getenv("SPANNER_EMULATOR_HOST") == "" { t.Skip("Skipping tests only running against the emulator.") From e89c180f7a81db92526b8bb224643bf236a3d70b Mon Sep 17 00:00:00 2001 From: Deep1998 Date: Wed, 5 Jan 2022 15:11:07 +0530 Subject: [PATCH 3/4] Added flag in test to skip json check --- testing/mysql/integration_test.go | 34 ++++++++++++------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/testing/mysql/integration_test.go b/testing/mysql/integration_test.go index acd12983b9..dcb56e9df6 100644 --- a/testing/mysql/integration_test.go +++ b/testing/mysql/integration_test.go @@ -22,7 +22,6 @@ import ( "log" "os" "path/filepath" - "strings" "testing" "time" @@ -125,7 +124,7 @@ func TestIntegration_MYSQLDUMP_Command(t *testing.T) { // Drop the database later. defer dropDatabase(t, dbURI) - checkResults(t, dbURI) + checkResults(t, dbURI, true) } func TestIntegration_MYSQL_SchemaAndDataSubcommand(t *testing.T) { @@ -150,7 +149,7 @@ func TestIntegration_MYSQL_SchemaAndDataSubcommand(t *testing.T) { // Drop the database later. defer dropDatabase(t, dbURI) - checkResults(t, dbURI) + checkResults(t, dbURI, true) } func TestIntegration_MYSQL_Command(t *testing.T) { @@ -173,7 +172,7 @@ func TestIntegration_MYSQL_Command(t *testing.T) { // Drop the database later. defer dropDatabase(t, dbURI) - checkResults(t, dbURI) + checkResults(t, dbURI, true) } func TestIntegration_MySQLInterleaveTable_DataOnlyWithSessionFile(t *testing.T) { @@ -187,7 +186,7 @@ func TestIntegration_MySQLInterleaveTable_DataOnlyWithSessionFile(t *testing.T) dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName) runDataOnlySubcommandForSessionFile(t, dbName, dbURI, sessionFile) defer dropDatabase(t, dbURI) - checkResults(t, dbURI) + checkResults(t, dbURI, false) } func runSchemaOnly(t *testing.T, dbName, filePrefix, sessionFile, dumpFilePath string) { @@ -243,7 +242,7 @@ func TestIntegration_MySQLDUMP_DataOnly(t *testing.T) { dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName) runDataOnly(t, dbName, dbURI, filePrefix, sessionFile, dumpFilePath) defer dropDatabase(t, dbURI) - checkResults(t, dbURI) + checkResults(t, dbURI, true) } func runSchemaSubcommand(t *testing.T, dbName, filePrefix, sessionFile, dumpFilePath string) { @@ -316,7 +315,7 @@ func TestIntegration_MySQLDUMP_DataSubcommand(t *testing.T) { dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName) runDataSubcommand(t, dbName, dbURI, filePrefix, sessionFile, dumpFilePath) defer dropDatabase(t, dbURI) - checkResults(t, dbURI) + checkResults(t, dbURI, true) } func TestIntegration_MySQLDUMP_SchemaAndDataSubcommand(t *testing.T) { @@ -331,10 +330,10 @@ func TestIntegration_MySQLDUMP_SchemaAndDataSubcommand(t *testing.T) { dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName) runSchemaAndDataSubcommand(t, dbName, dbURI, filePrefix, dumpFilePath) defer dropDatabase(t, dbURI) - checkResults(t, dbURI) + checkResults(t, dbURI, true) } -func checkResults(t *testing.T, dbURI string) { +func checkResults(t *testing.T, dbURI string, json bool) { // Make a query to check results. client, err := spanner.NewClient(ctx, dbURI) if err != nil { @@ -343,7 +342,9 @@ func checkResults(t *testing.T, dbURI string) { defer client.Close() checkBigInt(ctx, t, client) - checkJson(ctx, t, client, dbURI) + if json { + checkJson(ctx, t, client) + } } func checkBigInt(ctx context.Context, t *testing.T, client *spanner.Client) { @@ -367,16 +368,7 @@ func checkBigInt(ctx context.Context, t *testing.T, client *spanner.Client) { } } -func checkJson(ctx context.Context, t *testing.T, client *spanner.Client, dbURI string) { - resp, err := databaseAdmin.GetDatabaseDdl(ctx, &databasepb.GetDatabaseDdlRequest{Database: dbURI}) - if err != nil { - t.Fatalf("Could not read DDL from database %s: %v", dbURI, err) - } - for _, stmt := range resp.Statements { - if strings.Contains(stmt, "CREATE TABLE customers") { - assert.True(t, strings.Contains(stmt, "customer_profile JSON")) - } - } +func checkJson(ctx context.Context, t *testing.T, client *spanner.Client) { stmt := spanner.Statement{ SQL: `SELECT COUNT(*) FROM customers`, } @@ -385,7 +377,7 @@ func checkJson(ctx context.Context, t *testing.T, client *spanner.Client, dbURI row, _ := iter.Next() var rowCount int64 = 0 row.Columns(&rowCount) - assert.Equal(t, 2, rowCount) + assert.Equal(t, int64(2), rowCount) } func onlyRunForEmulatorTest(t *testing.T) { From ec5dd31239919aa238ea23d14781a821a2907c41 Mon Sep 17 00:00:00 2001 From: Deep1998 Date: Thu, 6 Jan 2022 13:13:40 +0530 Subject: [PATCH 4/4] Address comments --- test_data/mysqldump.test.out | 2 +- testing/mysql/integration_test.go | 52 +++++++++++++++++++++---------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/test_data/mysqldump.test.out b/test_data/mysqldump.test.out index 94b65cca91..3a44f3296e 100644 --- a/test_data/mysqldump.test.out +++ b/test_data/mysqldump.test.out @@ -98,7 +98,7 @@ DROP TABLE IF EXISTS `customers`; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `customers` ( `c_id` varchar(20) NOT NULL, - `customer_profile` JSON DEFAULT NULL, + `customer_profile` json DEFAULT NULL, PRIMARY KEY (`c_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; diff --git a/testing/mysql/integration_test.go b/testing/mysql/integration_test.go index dcb56e9df6..c1979b6b88 100644 --- a/testing/mysql/integration_test.go +++ b/testing/mysql/integration_test.go @@ -16,12 +16,14 @@ package mysql_test import ( "context" + "encoding/json" "flag" "fmt" "io/ioutil" "log" "os" "path/filepath" + "strings" "testing" "time" @@ -124,7 +126,7 @@ func TestIntegration_MYSQLDUMP_Command(t *testing.T) { // Drop the database later. defer dropDatabase(t, dbURI) - checkResults(t, dbURI, true) + checkResults(t, dbURI, false) } func TestIntegration_MYSQL_SchemaAndDataSubcommand(t *testing.T) { @@ -186,7 +188,7 @@ func TestIntegration_MySQLInterleaveTable_DataOnlyWithSessionFile(t *testing.T) dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName) runDataOnlySubcommandForSessionFile(t, dbName, dbURI, sessionFile) defer dropDatabase(t, dbURI) - checkResults(t, dbURI, false) + checkResults(t, dbURI, true) } func runSchemaOnly(t *testing.T, dbName, filePrefix, sessionFile, dumpFilePath string) { @@ -242,7 +244,7 @@ func TestIntegration_MySQLDUMP_DataOnly(t *testing.T) { dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName) runDataOnly(t, dbName, dbURI, filePrefix, sessionFile, dumpFilePath) defer dropDatabase(t, dbURI) - checkResults(t, dbURI, true) + checkResults(t, dbURI, false) } func runSchemaSubcommand(t *testing.T, dbName, filePrefix, sessionFile, dumpFilePath string) { @@ -315,7 +317,7 @@ func TestIntegration_MySQLDUMP_DataSubcommand(t *testing.T) { dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName) runDataSubcommand(t, dbName, dbURI, filePrefix, sessionFile, dumpFilePath) defer dropDatabase(t, dbURI) - checkResults(t, dbURI, true) + checkResults(t, dbURI, false) } func TestIntegration_MySQLDUMP_SchemaAndDataSubcommand(t *testing.T) { @@ -330,10 +332,10 @@ func TestIntegration_MySQLDUMP_SchemaAndDataSubcommand(t *testing.T) { dbURI := fmt.Sprintf("projects/%s/instances/%s/databases/%s", projectID, instanceID, dbName) runSchemaAndDataSubcommand(t, dbName, dbURI, filePrefix, dumpFilePath) defer dropDatabase(t, dbURI) - checkResults(t, dbURI, true) + checkResults(t, dbURI, false) } -func checkResults(t *testing.T, dbURI string, json bool) { +func checkResults(t *testing.T, dbURI string, skipJson bool) { // Make a query to check results. client, err := spanner.NewClient(ctx, dbURI) if err != nil { @@ -342,8 +344,8 @@ func checkResults(t *testing.T, dbURI string, json bool) { defer client.Close() checkBigInt(ctx, t, client) - if json { - checkJson(ctx, t, client) + if !skipJson { + checkJson(ctx, t, client, dbURI) } } @@ -368,16 +370,34 @@ func checkBigInt(ctx context.Context, t *testing.T, client *spanner.Client) { } } -func checkJson(ctx context.Context, t *testing.T, client *spanner.Client) { - stmt := spanner.Statement{ - SQL: `SELECT COUNT(*) FROM customers`, +func checkJson(ctx context.Context, t *testing.T, client *spanner.Client, dbURI string) { + resp, err := databaseAdmin.GetDatabaseDdl(ctx, &databasepb.GetDatabaseDdlRequest{Database: dbURI}) + if err != nil { + t.Fatalf("Could not read DDL from database %s: %v", dbURI, err) + } + for _, stmt := range resp.Statements { + if strings.Contains(stmt, "CREATE TABLE customers") { + assert.True(t, strings.Contains(stmt, "customer_profile JSON")) + } } - iter := client.Single().Query(ctx, stmt) + got_profile := spanner.NullJSON{} + iter := client.Single().Read(ctx, "customers", spanner.Key{"tel-595"}, []string{"customer_profile"}) defer iter.Stop() - row, _ := iter.Next() - var rowCount int64 = 0 - row.Columns(&rowCount) - assert.Equal(t, int64(2), rowCount) + for { + row, err := iter.Next() + if err == iterator.Done { + break + } + if err != nil { + t.Fatal(err) + } + if err := row.Columns(&got_profile); err != nil { + t.Fatal(err) + } + } + want_profile := spanner.NullJSON{Valid: true} + json.Unmarshal([]byte("{\"first_name\": \"Ernie\", \"status\": \"Looking for treats\", \"location\" : \"Brooklyn\"}"), &want_profile.Value) + assert.Equal(t, got_profile, want_profile) } func onlyRunForEmulatorTest(t *testing.T) {