From ab7559451007af147bfb73e11d3e308504a9b90a Mon Sep 17 00:00:00 2001 From: Tim Elgersma Date: Fri, 10 Dec 2021 22:11:32 +0000 Subject: [PATCH 1/6] initial implementation --- config/geopartitioned_workload.xml | 19 ++++++++++- .../GeoPartitionedConfigFileOptions.java | 17 +++++++++- .../GeoPartitionedSchemaManager.java | 17 ++++++++-- .../geopartitioned/PartitionedTable.java | 16 ++++++--- .../util/GeoPartitionPolicy.java | 34 +++++++++++++++++-- 5 files changed, 91 insertions(+), 12 deletions(-) diff --git a/config/geopartitioned_workload.xml b/config/geopartitioned_workload.xml index b3c639c..54f3795 100644 --- a/config/geopartitioned_workload.xml +++ b/config/geopartitioned_workload.xml @@ -1,6 +1,7 @@ - false + true + true 2 @@ -45,4 +46,20 @@ + + + tablegroup0 + tablespace0 + + + tablegroup1 + tablespace1 + true + + + tablegroup2 + tablespace2 + true + + diff --git a/src/com/oltpbenchmark/GeoPartitionedConfigFileOptions.java b/src/com/oltpbenchmark/GeoPartitionedConfigFileOptions.java index 33e3d6c..939071c 100644 --- a/src/com/oltpbenchmark/GeoPartitionedConfigFileOptions.java +++ b/src/com/oltpbenchmark/GeoPartitionedConfigFileOptions.java @@ -43,7 +43,7 @@ public GeoPartitionPolicy getGeoPartitionPlacement(final int totalWarehousesAcro } // Verify that the current TPCC client can access only a single partition. - GeoPartitionPolicy policy = new GeoPartitionPolicy(numPartitions, totalWarehousesAcrossShards); + GeoPartitionPolicy policy = new GeoPartitionPolicy(numPartitions, totalWarehousesAcrossShards, isFlagSet("useTablegroups")); int partitionForStartWarehouse = policy.getPartitionForWarehouse(startWarehouseIdForShard); int partitionForEndWarehouse = policy.getPartitionForWarehouse(startWarehouseIdForShard + numWarehouses - 1); @@ -53,6 +53,21 @@ public GeoPartitionPolicy getGeoPartitionPlacement(final int totalWarehousesAcro numPartitions, numWarehouses, startWarehouseIdForShard, totalWarehousesAcrossShards)); } + // Iterate through all the tablegroups in the config. + // The lists in XMLConfiguration are 1-based, so start the + // loop from 1. + for (int ii = 1; ii <= getNumTablespaces(); ++ii) { + final String tablegroupPath = String.format("tablegroups/tablegroup[%d]/", ii); + final String tablegroupName = getRequiredStringOpt(tablegroupPath + "name"); + final String tablespaceName = getRequiredStringOpt(tablegroupPath + "tablespace"); + + if (isFlagSet(tablegroupPath + "storePartitions")) { + policy.addTablegroupForPartition(tablegroupName); + } + + policy.addTablegroup(tablegroupName, tablespaceName); + } + // Iterate through all the tablespaces in the config. // The lists in XMLConfiguration are 1-based, so start the // loop from 1. diff --git a/src/com/oltpbenchmark/schema/geopartitioned/GeoPartitionedSchemaManager.java b/src/com/oltpbenchmark/schema/geopartitioned/GeoPartitionedSchemaManager.java index 2b4a6dc..18e232a 100644 --- a/src/com/oltpbenchmark/schema/geopartitioned/GeoPartitionedSchemaManager.java +++ b/src/com/oltpbenchmark/schema/geopartitioned/GeoPartitionedSchemaManager.java @@ -24,8 +24,8 @@ public GeoPartitionedSchemaManager(GeoPartitionPolicy geoPartitioningPolicy, Con super(conn); this.geoPartitionPolicy = geoPartitioningPolicy; for (TableSchema t : TPCCTableSchemas.tables.values()) { - tables.put(t.name(), - t.name().equals(TPCCConstants.TABLENAME_ITEM) ? new DefaultTable(t, geoPartitioningPolicy.getTablespaceForItemTable()) + tables.put(t.name(), + t.name().equals(TPCCConstants.TABLENAME_ITEM) ? new DefaultTable(t, geoPartitioningPolicy.getTablespaceForItemTable()) : new PartitionedTable(t, geoPartitionPolicy)); } } @@ -41,6 +41,7 @@ public void create() throws SQLException { } createTablespaces(); + createTablegroups(); for (Table t : tables.values()) { execute(t.getCreateDdl()); @@ -73,6 +74,16 @@ private void createTablespaces() throws SQLException { } } + private void createTablegroups() throws SQLException { + if (!geoPartitionPolicy.shouldUseTablegroups()) { + return; + } + for (Map.Entry entry : geoPartitionPolicy.getTablegroups().entrySet()) { + // execute(String.format("DROP TABLEGROUP IF EXISTS %s", entry.getKey())); + execute(String.format("CREATE TABLEGROUP %s TABLESPACE %s", entry.getKey(), entry.getValue())); + } + } + @Override public void enableForeignKeyConstraints() throws SQLException { // Create foreign key relations among the partitions themselves, rather than between @@ -116,7 +127,7 @@ public void enableForeignKeyConstraints() throws SQLException { "(OL_SUPPLY_W_ID, OL_I_ID) REFERENCES STOCK%d (S_W_ID, S_I_ID) NOT VALID", idx, idx, idx)); } } - + @Override public void createSqlProcedures() throws Exception { try (Statement st = db_connection.createStatement()) { diff --git a/src/com/oltpbenchmark/schema/geopartitioned/PartitionedTable.java b/src/com/oltpbenchmark/schema/geopartitioned/PartitionedTable.java index 8df9a40..d4aae1b 100644 --- a/src/com/oltpbenchmark/schema/geopartitioned/PartitionedTable.java +++ b/src/com/oltpbenchmark/schema/geopartitioned/PartitionedTable.java @@ -25,6 +25,10 @@ private String tablespaceForPartition(int idx) { return policy.getTablespaceForPartition(idx); } + private String tablegroupForPartition(int idx) { + return policy.getTablegroupForPartition(idx); + } + @Override public String getCreateDdl() { StringBuilder sb = new StringBuilder(); @@ -38,9 +42,9 @@ public String getCreateDdl() { private String createPartitionedTable() { StringBuilder sb = new StringBuilder(); sb.append("CREATE TABLE ").append(schema.name()).append(" ( "); - + addColDescForCreateDDL(sb); - + // Append the partition key. sb.append(")\n PARTITION BY RANGE ").append(schema.getPartitionKey()); // Append the tablespace name. @@ -63,7 +67,7 @@ private String createPartitionTable(int idx) { // While creating partitions, skip the column declaration. sb.append("\n").append(c.getName()).append(','); } - + // Remove trailing comma added after the last column in above loop. sb.setLength(sb.length() - 1); @@ -71,7 +75,11 @@ private String createPartitionTable(int idx) { sb.append(String.format(",\n PRIMARY KEY %s", schema.getPrimaryKey())); } sb.append(String.format(")\n FOR VALUES FROM (%d) TO (%d)", start, end)); - sb.append(String.format(" TABLESPACE %s;", tablespaceForPartition(idx - 1))); + if (policy.shouldUseTablegroups()) { + sb.append(String.format(" TABLEGROUP %s;", tablegroupForPartition(idx - 1))); + } else { + sb.append(String.format(" TABLESPACE %s;", tablespaceForPartition(idx - 1))); + } return sb.toString(); } diff --git a/src/com/oltpbenchmark/util/GeoPartitionPolicy.java b/src/com/oltpbenchmark/util/GeoPartitionPolicy.java index 63d2249..e4b4f1e 100644 --- a/src/com/oltpbenchmark/util/GeoPartitionPolicy.java +++ b/src/com/oltpbenchmark/util/GeoPartitionPolicy.java @@ -18,32 +18,45 @@ * be replicated and placed. */ public class GeoPartitionPolicy { - private final Map tablespaceToPlacementPolicy = new HashMap();; + private final Map tablespaceToPlacementPolicy = new HashMap<>(); + private final Map tablegroupToTablespace = new HashMap<>(); private String tablespaceForPartitionedTables; private String tablespaceForItemTable; - private final List tablespacesForPartitions = new ArrayList(); + private final List tablespacesForPartitions = new ArrayList<>(); + private final List tablegroupsForPartitions = new ArrayList<>(); private final int numWarehouses; private final int numPartitions; - public GeoPartitionPolicy(int numPartitions, int numWarehouses) { + private final boolean useTablegroups; + + public GeoPartitionPolicy(int numPartitions, int numWarehouses, boolean useTablegroups) { this.numPartitions = numPartitions; this.numWarehouses = numWarehouses; + this.useTablegroups = useTablegroups; } public int getNumPartitions() { return numPartitions; } + public boolean shouldUseTablegroups() { + return useTablegroups; + } + // Getters and setters. public Map getTablespaceToPlacementPolicy() { return tablespaceToPlacementPolicy; } + public Map getTablegroups() { + return tablegroupToTablespace; + } + public String getTablespaceForPartitionedTables() { return tablespaceForPartitionedTables; } @@ -63,6 +76,9 @@ public void setTablespaceForItemTable(String tablespaceForItemTable) { public List getTablespacesForPartitions() { return tablespacesForPartitions; } + public List getTablegroupsForPartitions() { + return tablegroupsForPartitions; + } public int getNumWarehouses() { return numWarehouses; @@ -72,10 +88,18 @@ public void addTablespacePlacementPolicy(String tablespace, PlacementPolicy poli tablespaceToPlacementPolicy.put(tablespace, policy); } + public void addTablegroup(String tablegroup, String tablespace) { + tablegroupToTablespace.put(tablegroup, tablespace); + } + public void addTablespaceForPartition(String tablespace) { tablespacesForPartitions.add(tablespace); } + public void addTablegroupForPartition(String tablegroup) { + tablegroupsForPartitions.add(tablegroup); + } + public String getTablespaceCreationJson(String tablespace) { PlacementPolicy placementPolicy = tablespaceToPlacementPolicy.get(tablespace); // Handle errors. @@ -87,6 +111,10 @@ public String getTablespaceForPartition(int idx) { return tablespacesForPartitions.get(idx); } + public String getTablegroupForPartition(int idx) { + return tablegroupsForPartitions.get(idx); + } + public int getStartWarehouseForPartition(int idx) { return (idx - 1) * numWareHousesPerSplit() + 1; } From 9a95dabb06384e6d65d4bab1531d2e0b795106ed Mon Sep 17 00:00:00 2001 From: Tim Elgersma Date: Mon, 13 Dec 2021 20:38:44 +0000 Subject: [PATCH 2/6] few small fixes to get it to run --- config/geopartitioned_workload.xml | 2 +- src/com/oltpbenchmark/api/BenchmarkModule.java | 3 --- src/com/oltpbenchmark/schema/TPCCTableSchemas.java | 14 +++++++------- .../GeoPartitionedSchemaManager.java | 2 +- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/config/geopartitioned_workload.xml b/config/geopartitioned_workload.xml index 54f3795..f8fa0cf 100644 --- a/config/geopartitioned_workload.xml +++ b/config/geopartitioned_workload.xml @@ -2,7 +2,7 @@ true true - 2 + 1 tablespace0 diff --git a/src/com/oltpbenchmark/api/BenchmarkModule.java b/src/com/oltpbenchmark/api/BenchmarkModule.java index fe8f6e3..c0ad8d4 100644 --- a/src/com/oltpbenchmark/api/BenchmarkModule.java +++ b/src/com/oltpbenchmark/api/BenchmarkModule.java @@ -202,9 +202,6 @@ public final void createDatabase() { /** * Invoke this benchmark's database loader. - * We return the handle to Loader object that we created to do this. - * You probably don't need it and can simply ignore. There are some - * test cases that use it. That's why it's here. */ public final void loadDatabase() { Loader loader = this.makeLoaderImpl(); diff --git a/src/com/oltpbenchmark/schema/TPCCTableSchemas.java b/src/com/oltpbenchmark/schema/TPCCTableSchemas.java index 56dfd2a..b182333 100644 --- a/src/com/oltpbenchmark/schema/TPCCTableSchemas.java +++ b/src/com/oltpbenchmark/schema/TPCCTableSchemas.java @@ -20,14 +20,14 @@ public class TPCCTableSchemas { .column("ol_supply_w_id", "int NOT NULL") .column("ol_quantity", "decimal(2,0) NOT NULL") .column("ol_dist_info", "char(24) NOT NULL") - .primaryKey("((ol_w_id,ol_d_id) HASH,ol_o_id,ol_number)") + .primaryKey("(ol_w_id,ol_d_id,ol_o_id,ol_number)") .partitionKey("(ol_w_id)") .build(), new TableSchemaBuilder(TPCCConstants.TABLENAME_NEWORDER) .column("no_w_id", "int NOT NULL") .column("no_d_id", "int NOT NULL") .column("no_o_id", "int NOT NULL") - .primaryKey("((no_w_id,no_d_id) HASH,no_o_id)") + .primaryKey("(no_w_id,no_d_id,no_o_id)") .partitionKey("(no_w_id)") .build(), new TableSchemaBuilder(TPCCConstants.TABLENAME_STOCK) @@ -48,7 +48,7 @@ public class TPCCTableSchemas { .column("s_dist_08", "char(24) NOT NULL") .column("s_dist_09", "char(24) NOT NULL") .column("s_dist_10", "char(24) NOT NULL") - .primaryKey("(s_w_id HASH, s_i_id ASC)") + .primaryKey("(s_w_id, s_i_id ASC)") .partitionKey("(s_w_id)") .build(), new TableSchemaBuilder(TPCCConstants.TABLENAME_OPENORDER) @@ -60,7 +60,7 @@ public class TPCCTableSchemas { .column("o_ol_cnt", "decimal(2,0) NOT NULL") .column("o_all_local", "decimal(1,0) NOT NULL") .column("o_entry_d", "timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP") - .primaryKey("((o_w_id,o_d_id) HASH,o_id)") + .primaryKey("(o_w_id,o_d_id,o_id)") .partitionKey("(o_w_id)") .build(), new TableSchemaBuilder(TPCCConstants.TABLENAME_HISTORY) @@ -96,7 +96,7 @@ public class TPCCTableSchemas { .column("c_since", "timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP") .column("c_middle", "char(2) NOT NULL") .column("c_data", "varchar(500) NOT NULL") - .primaryKey("((c_w_id,c_d_id) HASH,c_id)") + .primaryKey("(c_w_id,c_d_id,c_id)") .partitionKey("(c_w_id)") .build(), new TableSchemaBuilder(TPCCConstants.TABLENAME_DISTRICT) @@ -111,7 +111,7 @@ public class TPCCTableSchemas { .column("d_city", "varchar(20) NOT NULL") .column("d_state", "char(2) NOT NULL") .column("d_zip", "char(9) NOT NULL") - .primaryKey("((d_w_id,d_id) HASH)") + .primaryKey("(d_w_id,d_id)") .partitionKey("(d_w_id)") .build(), new TableSchemaBuilder(TPCCConstants.TABLENAME_ITEM) @@ -136,7 +136,7 @@ public class TPCCTableSchemas { .partitionKey("(w_id)") .build() ).collect(Collectors.toMap(TableSchema::name, e -> e))); - + public static TableSchema getTableSchema(String tablename) { return tables.get(tablename); } diff --git a/src/com/oltpbenchmark/schema/geopartitioned/GeoPartitionedSchemaManager.java b/src/com/oltpbenchmark/schema/geopartitioned/GeoPartitionedSchemaManager.java index 18e232a..67d3cb9 100644 --- a/src/com/oltpbenchmark/schema/geopartitioned/GeoPartitionedSchemaManager.java +++ b/src/com/oltpbenchmark/schema/geopartitioned/GeoPartitionedSchemaManager.java @@ -79,7 +79,7 @@ private void createTablegroups() throws SQLException { return; } for (Map.Entry entry : geoPartitionPolicy.getTablegroups().entrySet()) { - // execute(String.format("DROP TABLEGROUP IF EXISTS %s", entry.getKey())); + execute(String.format("DROP TABLEGROUP IF EXISTS %s", entry.getKey())); execute(String.format("CREATE TABLEGROUP %s TABLESPACE %s", entry.getKey(), entry.getValue())); } } From 6b07b22329acbe212ad9a2099e6bf09046da186e Mon Sep 17 00:00:00 2001 From: Tim Elgersma Date: Tue, 14 Dec 2021 13:39:29 +0000 Subject: [PATCH 3/6] no tgs --- config/geopartitioned_workload.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/geopartitioned_workload.xml b/config/geopartitioned_workload.xml index f8fa0cf..a4afe4e 100644 --- a/config/geopartitioned_workload.xml +++ b/config/geopartitioned_workload.xml @@ -1,8 +1,8 @@ true - true - 1 + false + 2 tablespace0 From 5455ab44c0f190cdfa3dfe939eed4b2f1472d050 Mon Sep 17 00:00:00 2001 From: Tim Elgersma Date: Tue, 14 Dec 2021 14:06:11 +0000 Subject: [PATCH 4/6] one partition --- config/geopartitioned_workload.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/geopartitioned_workload.xml b/config/geopartitioned_workload.xml index a4afe4e..b7e2c35 100644 --- a/config/geopartitioned_workload.xml +++ b/config/geopartitioned_workload.xml @@ -2,7 +2,7 @@ true false - 2 + 1 tablespace0 From 3ce3915a1e082d6b39345f2f02fe097e81a95aad Mon Sep 17 00:00:00 2001 From: Tim Elgersma Date: Tue, 14 Dec 2021 14:47:01 +0000 Subject: [PATCH 5/6] 2abc --- config/geopartitioned_workload.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/geopartitioned_workload.xml b/config/geopartitioned_workload.xml index b7e2c35..5d58344 100644 --- a/config/geopartitioned_workload.xml +++ b/config/geopartitioned_workload.xml @@ -14,7 +14,7 @@ aws us-west-2 - us-west-2a0 + us-west-2a 1 @@ -27,7 +27,7 @@ aws us-west-2 - us-west-2a1 + us-west-2b 1 @@ -40,7 +40,7 @@ aws us-west-2 - us-west-2a2 + us-west-2c 1 From 7ebed008800e2b17cd279b1ab5694e5492bd1799 Mon Sep 17 00:00:00 2001 From: Tim Elgersma Date: Tue, 14 Dec 2021 16:02:44 +0000 Subject: [PATCH 6/6] useTgs --- config/geopartitioned_workload.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/geopartitioned_workload.xml b/config/geopartitioned_workload.xml index 5d58344..388a800 100644 --- a/config/geopartitioned_workload.xml +++ b/config/geopartitioned_workload.xml @@ -1,7 +1,7 @@ true - false + true 1