From 5a593ee5f4da6dc025363eff664b80716eaa1691 Mon Sep 17 00:00:00 2001 From: Saransh Gupta Date: Wed, 26 Jul 2023 23:48:13 +0000 Subject: [PATCH 1/5] Added ALTER operations to set and remove table properties --- .../java/iceberg_cli/IcebergConnector.java | 54 +++++++++++++++++++ .../src/main/java/iceberg_cli/cli/Parser.java | 2 +- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java b/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java index 165a3b1..283d142 100644 --- a/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java +++ b/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java @@ -51,6 +51,7 @@ import org.apache.iceberg.TableScan; import org.apache.iceberg.Transaction; import org.apache.iceberg.UpdateSchema; +import org.apache.iceberg.UpdateProperties; import org.apache.iceberg.aws.s3.S3FileIO; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; @@ -194,6 +195,18 @@ public boolean createTable(Schema schema, PartitionSpec spec, boolean overwrite) * { * "drop":["c1","c2"] * } + * + * The property changes are expected in the following format: + * SETTING key/value property + * { + * "set_prop":[ + * {"key":"p1","value":"v1"} + * ] + * } + * REMOVING property key + * { + * "rm_prop":["p1","p2"] + * } * * * There are more ALTER operations that can be performed according to the documentation, @@ -205,6 +218,8 @@ public boolean alterTable(String newSchema) throws Exception { final int OP_ADD = 1; final int OP_DROP = 2; final int OP_RENAME = 4; + final int OP_SET_PROP = 8; + final int OP_RM_PROP = 16; loadTable(); UpdateSchema updateSchema = iceberg_table.updateSchema(); JSONObject schemaSpecs = new JSONObject(newSchema); @@ -279,6 +294,45 @@ public boolean alterTable(String newSchema) throws Exception { // all good - commit changes updateSchema.commit(); + + // check for updates to table properties + UpdateProperties updateProperties = iceberg_table.updateProperties(); + try { + JSONArray setProps = schemaSpecs.getJSONArray("set_prop"); + for (int i = 0; i < setProps.length(); i++) { + try { + JSONObject jo = setProps.getJSONObject(i); + String key = jo.getString("key"); + String value = jo.getString("value"); + updateProperties.set(key, value); + op |= OP_SET_PROP; + } catch (JSONException e) { + System.out.println("Invalid key/value property."); + return false; + } + } + } catch (JSONException e) { + // no properties to set, move on + } + + try { + JSONArray rmProps = schemaSpecs.getJSONArray("rm_prop"); + for (int i = 0; i < rmProps.length(); i++) { + try { + String key = rmProps.getString(i); + updateProperties.remove(key); + op |= OP_RM_PROP; + } catch (JSONException e) { + System.out.println("Invalid property key."); + return false; + } + } + } catch (JSONException e) { + // no properties to remove, move on + } + + updateProperties.commit(); + return true; } diff --git a/tools/java-iceberg-cli/src/main/java/iceberg_cli/cli/Parser.java b/tools/java-iceberg-cli/src/main/java/iceberg_cli/cli/Parser.java index 477c6f9..4ba9026 100644 --- a/tools/java-iceberg-cli/src/main/java/iceberg_cli/cli/Parser.java +++ b/tools/java-iceberg-cli/src/main/java/iceberg_cli/cli/Parser.java @@ -90,7 +90,7 @@ private void initializeCommands() { Command alter = new Command("alter", "Alter a table"); alter.addOption("--help", "Show this help message and exit"); alter.addArgument("identifier", "Table or namespace identifier", true); - create.addArgument("schema", "Alter a table using this schema"); + alter.addArgument("schema", "Alter a table using this schema"); m_commands.put("alter", alter); Command drop = new Command("drop", "Drop a table or a namespace"); From bc621deba2ef0b308837152bc506192093fab3db Mon Sep 17 00:00:00 2001 From: Saransh Gupta Date: Thu, 27 Jul 2023 16:40:16 +0000 Subject: [PATCH 2/5] Added ALTER operations to set and remove table properties --- .../src/main/java/iceberg_cli/IcebergConnector.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java b/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java index 283d142..0d5c978 100644 --- a/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java +++ b/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java @@ -280,12 +280,6 @@ public boolean alterTable(String newSchema) throws Exception { // no columns to rename, move on } - // have we altered anything? - if (op == OP_NONE) { - System.out.println("Unrecognized ALTER operation."); - return false; - } - // confirm DROP wasn't bundled with any other ALTERs if ((op & OP_DROP) == OP_DROP && op != OP_DROP) { System.out.println("Cannot perform DROP along with other ALTER operations."); @@ -333,6 +327,12 @@ public boolean alterTable(String newSchema) throws Exception { updateProperties.commit(); + // have we altered anything? + if (op == OP_NONE) { + System.out.println("Unrecognized ALTER operation."); + return false; + } + return true; } From dcdb6e2d02bf7c024b6b2068a305ae50b60295e4 Mon Sep 17 00:00:00 2001 From: Saransh Gupta Date: Wed, 27 Sep 2023 19:04:20 +0000 Subject: [PATCH 3/5] Applying comments from review * Expanded operation names * Conditional commit of updateSchema and updateProperties to avoid empty commit exception --- .../java/iceberg_cli/IcebergConnector.java | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java b/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java index 0d5c978..ecddf3a 100644 --- a/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java +++ b/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java @@ -220,6 +220,8 @@ public boolean alterTable(String newSchema) throws Exception { final int OP_RENAME = 4; final int OP_SET_PROP = 8; final int OP_RM_PROP = 16; + final int UPDATE_SCHEMA = OP_ADD | OP_DROP | OP_RENAME; + final int UPDATE_PROP = OP_SET_PROP | OP_RM_PROP; loadTable(); UpdateSchema updateSchema = iceberg_table.updateSchema(); JSONObject schemaSpecs = new JSONObject(newSchema); @@ -280,19 +282,10 @@ public boolean alterTable(String newSchema) throws Exception { // no columns to rename, move on } - // confirm DROP wasn't bundled with any other ALTERs - if ((op & OP_DROP) == OP_DROP && op != OP_DROP) { - System.out.println("Cannot perform DROP along with other ALTER operations."); - return false; - } - - // all good - commit changes - updateSchema.commit(); - // check for updates to table properties UpdateProperties updateProperties = iceberg_table.updateProperties(); try { - JSONArray setProps = schemaSpecs.getJSONArray("set_prop"); + JSONArray setProps = schemaSpecs.getJSONArray("set_property"); for (int i = 0; i < setProps.length(); i++) { try { JSONObject jo = setProps.getJSONObject(i); @@ -310,7 +303,7 @@ public boolean alterTable(String newSchema) throws Exception { } try { - JSONArray rmProps = schemaSpecs.getJSONArray("rm_prop"); + JSONArray rmProps = schemaSpecs.getJSONArray("remove_property"); for (int i = 0; i < rmProps.length(); i++) { try { String key = rmProps.getString(i); @@ -324,15 +317,27 @@ public boolean alterTable(String newSchema) throws Exception { } catch (JSONException e) { // no properties to remove, move on } - - updateProperties.commit(); + // confirm DROP wasn't bundled with any other ALTERs + if ((op & OP_DROP) == OP_DROP && op != OP_DROP) { + System.out.println("Cannot perform DROP along with other ALTER operations."); + return false; + } + // have we altered anything? if (op == OP_NONE) { System.out.println("Unrecognized ALTER operation."); return false; } + // all good - commit changes + if (op & UPDATE_SCHEMA) { + updateSchema.commit(); + } + if (op & UPDATE_PROP) { + updateProperties.commit(); + } + return true; } From 6db0913ba2051a2ac466870ab6967dfd2796ea2b Mon Sep 17 00:00:00 2001 From: Saransh Gupta Date: Wed, 27 Sep 2023 20:11:29 +0000 Subject: [PATCH 4/5] Updating function description --- .../src/main/java/iceberg_cli/IcebergConnector.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java b/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java index ecddf3a..7bf1e94 100644 --- a/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java +++ b/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java @@ -199,13 +199,13 @@ public boolean createTable(Schema schema, PartitionSpec spec, boolean overwrite) * The property changes are expected in the following format: * SETTING key/value property * { - * "set_prop":[ + * "set_property":[ * {"key":"p1","value":"v1"} * ] * } * REMOVING property key * { - * "rm_prop":["p1","p2"] + * "remove_property":["p1","p2"] * } * * From 8f91436f056bce6d507357fe4cda532077ee2c79 Mon Sep 17 00:00:00 2001 From: Saransh Gupta Date: Wed, 27 Sep 2023 20:28:04 +0000 Subject: [PATCH 5/5] update commit condition --- .../src/main/java/iceberg_cli/IcebergConnector.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java b/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java index 7bf1e94..0574be9 100644 --- a/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java +++ b/tools/java-iceberg-cli/src/main/java/iceberg_cli/IcebergConnector.java @@ -331,10 +331,10 @@ public boolean alterTable(String newSchema) throws Exception { } // all good - commit changes - if (op & UPDATE_SCHEMA) { + if ((op & UPDATE_SCHEMA) != 0) { updateSchema.commit(); } - if (op & UPDATE_PROP) { + if ((op & UPDATE_PROP) != 0) { updateProperties.commit(); }