From 6cf84dadfa18780ff622c0458b6c216f99462059 Mon Sep 17 00:00:00 2001 From: i1befree Date: Fri, 21 Mar 2014 11:25:19 +0900 Subject: [PATCH 1/5] bug fix : hive 0.11 inner join bug fix --- .../java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g index 8e5936132077..ecb25a92acb9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g @@ -172,8 +172,8 @@ tableSample tableSource @init { gParent.msgs.push("table source"); } @after { gParent.msgs.pop(); } - : tabname=tableName (ts=tableSample)? (alias=identifier)? - -> ^(TOK_TABREF $tabname $ts? $alias?) + : tabname=tableName (props=tableProperties)? (ts=tableSample)? (KW_AS? alias=Identifier)? + -> ^(TOK_TABREF $tabname $props? $ts? $alias?) ; tableName From aee88399845b6add01104737e5a0337b534620e8 Mon Sep 17 00:00:00 2001 From: i1befree Date: Tue, 25 Mar 2014 14:47:26 +0900 Subject: [PATCH 2/5] catalog schema return value for obzen olap tool --- .../cli/operation/GetCatalogsOperation.java | 16 ++++++++++++++++ .../cli/operation/GetColumnsOperation.java | 2 +- .../cli/operation/GetTablesOperation.java | 18 +++++++++++++++--- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/service/src/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java b/service/src/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java index 581e69c63e49..7f22a2aa0b64 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java @@ -18,6 +18,8 @@ package org.apache.hive.service.cli.operation; +import java.util.List; + import org.apache.hive.service.cli.FetchOrientation; import org.apache.hive.service.cli.HiveSQLException; import org.apache.hive.service.cli.OperationState; @@ -46,6 +48,20 @@ protected GetCatalogsOperation(HiveSession parentSession) { @Override public void run() throws HiveSQLException { setState(OperationState.RUNNING); + try{ + List databaseNames = getParentSession().getMetaStoreClient().getAllDatabases(); + + if(databaseNames != null && databaseNames.size() > 0){ + for(String databaseName : databaseNames){ + Object rowData[] = new Object[] {databaseName}; + rowSet.addRow(RESULT_SET_SCHEMA, rowData); + } + } + } catch (Exception e) { + setState(OperationState.ERROR); + throw new HiveSQLException(e); + } + setState(OperationState.FINISHED); } diff --git a/service/src/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java b/service/src/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java index af87a90e5e3b..08e2120d4de3 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java @@ -140,7 +140,7 @@ public void run() throws HiveSQLException { continue; } Object[] rowData = new Object[] { - null, // TABLE_CAT + table.getDbName(), // TABLE_CAT table.getDbName(), // TABLE_SCHEM table.getTableName(), // TABLE_NAME column.getName(), // COLUMN_NAME diff --git a/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java b/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java index df8b5b31c22d..21dbb4466582 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.HashMap; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.Table; @@ -50,6 +52,15 @@ public class GetTablesOperation extends MetadataOperation { .addStringColumn("TABLE_NAME", "Table name.") .addStringColumn("TABLE_TYPE", "The table type, e.g. \"TABLE\", \"VIEW\", etc.") .addStringColumn("REMARKS", "Comments about the table."); + + private static Map typeMap; + + static{ + typeMap = new HashMap(); + + typeMap.put("MANAGED_TABLE", "TABLE"); + typeMap.put("MANAGED_VIEW", "VIEW"); + } protected GetTablesOperation(HiveSession parentSession, String catalogName, String schemaName, String tableName, @@ -77,13 +88,14 @@ public void run() throws HiveSQLException { List tableNames = metastoreClient.getTables(dbName, tablePattern); for (Table table : metastoreClient.getTableObjectsByName(dbName, tableNames)) { Object[] rowData = new Object[] { - DEFAULT_HIVE_CATALOG, + dbName, table.getDbName(), table.getTableName(), - table.getTableType(), + typeMap.get(table.getTableType()), table.getParameters().get("comment") }; - if (tableTypes.isEmpty() || tableTypes.contains(table.getTableType())) { + + if (tableTypes.isEmpty() || tableTypes.contains( typeMap.get(table.getTableType()) )) { rowSet.addRow(RESULT_SET_SCHEMA, rowData); } } From 62d51f03955b4b73cd7b3ec0e9d174f1524507b4 Mon Sep 17 00:00:00 2001 From: i1befree Date: Thu, 27 Mar 2014 11:14:14 +0900 Subject: [PATCH 3/5] return table type EXTERNAL_TABLE -> TABLE --- .../hive/service/cli/operation/GetTablesOperation.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java b/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java index 21dbb4466582..dbbef5a7c934 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java @@ -59,6 +59,7 @@ public class GetTablesOperation extends MetadataOperation { typeMap = new HashMap(); typeMap.put("MANAGED_TABLE", "TABLE"); + typeMap.put("EXTERNAL_TABLE", "TABLE"); typeMap.put("MANAGED_VIEW", "VIEW"); } @@ -91,11 +92,11 @@ public void run() throws HiveSQLException { dbName, table.getDbName(), table.getTableName(), - typeMap.get(table.getTableType()), + typeMap.containsKey(table.getTableType())? typeMap.get(table.getTableType()) : table.getTableType(), table.getParameters().get("comment") }; - if (tableTypes.isEmpty() || tableTypes.contains( typeMap.get(table.getTableType()) )) { + if (tableTypes.isEmpty() || tableTypes.contains( (typeMap.containsKey(table.getTableType())? typeMap.get(table.getTableType()) : table.getTableType()) )) { rowSet.addRow(RESULT_SET_SCHEMA, rowData); } } From c735289bedf7deb3c455d18fabe566d7f70db413 Mon Sep 17 00:00:00 2001 From: i1befree Date: Mon, 31 Mar 2014 18:59:51 +0900 Subject: [PATCH 4/5] support non-ascii string for tables --- .../hive/ql/parse/BaseSemanticAnalyzer.java | 6 ++++++ .../hadoop/hive/ql/parse/FromClauseParser.g | 9 ++++++++ .../apache/hadoop/hive/ql/parse/HiveLexer.g | 1 + .../hive/ql/parse/SemanticAnalyzer.java | 6 ++++-- .../hive/ql/parse/TypeCheckProcFactory.java | 21 +++++++++++++++++++ 5 files changed, 41 insertions(+), 2 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index e5321c82a327..ebcef7bccb7f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -401,9 +401,15 @@ public static String unescapeIdentifier(String val) { if (val == null) { return null; } + if (val.charAt(0) == '`' && val.charAt(val.length() - 1) == '`') { val = val.substring(1, val.length() - 1); } + + if (val.charAt(0) == '"' && val.charAt(val.length() - 1) == '"') { + val = val.substring(1, val.length() - 1); + } + return val; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g index ecb25a92acb9..e1b513e71c54 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g @@ -183,6 +183,15 @@ tableName db=identifier DOT tab=identifier -> ^(TOK_TABNAME $db $tab) | + db=identifier DOT tab2=StringLiteral + -> ^(TOK_TABNAME $db Identifier[$tab2.text.substring(1,$tab2.text.length()-1)]) + | + db2=StringLiteral DOT tab=identifier + -> ^(TOK_TABNAME Identifier[$db2.text.substring(1,$db2.text.length()-1)] $tab) + | + db2=StringLiteral DOT tab2=StringLiteral + -> ^(TOK_TABNAME Identifier[$db2.text.substring(1,$db2.text.length()-1)] Identifier[$tab2.text.substring(1,$tab2.text.length()-1)]) + | tab=identifier -> ^(TOK_TABNAME $tab) ; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g index f72f47645b8c..7d05ff438e15 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g @@ -295,6 +295,7 @@ BITWISEOR : '|'; BITWISEXOR : '^'; QUESTION : '?'; DOLLAR : '$'; +DOUBLEQUOTE : '\"' // LITERALS fragment diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index ef66d4ba5d89..be2ded2bfd6b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2619,9 +2619,11 @@ && isRegex(unescapeIdentifier(expr.getChild(0).getText()))) { pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()), null, expr, col_list, inputRR, pos, out_rwsch, qb.getAliases(), subQuery); } else if (expr.getType() == HiveParser.DOT - && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL + && ((expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0) - .getChild(0).getText().toLowerCase())) && !hasAsClause + .getChild(0).getText().toLowerCase()))) || (expr.getChild(0).getType() == HiveParser.StringLiteral + && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0).getText().toLowerCase()))) ) + && !hasAsClause && !inputRR.getIsExprResolver() && isRegex(unescapeIdentifier(expr.getChild(1).getText()))) { // In case the expression is TABLE.COL (col can be regex). diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 99686358a76d..62b8819e1c1b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -956,6 +956,27 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, .getIsVirtualCol()); } + if (expr.getType() == HiveParser.DOT + && expr.getChild(0).getType() == HiveParser.StringLiteral + && nodeOutputs[0] != null) { + + RowResolver input = ctx.getInputRR(); + String tableAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr + .getChild(0).getText()); + // NOTE: tableAlias must be a valid non-ambiguous table alias, + // because we've checked that in TOK_TABLE_OR_COL's process method. + ColumnInfo colInfo = input.get(tableAlias, + ((ExprNodeConstantDesc) nodeOutputs[1]).getValue().toString()); + + if (colInfo == null) { + ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr); + return null; + } + return new ExprNodeColumnDesc(colInfo.getType(), colInfo + .getInternalName(), colInfo.getTabAlias(), colInfo + .getIsVirtualCol()); + } + // Return nulls for conversion operators if (conversionFunctionTextHashMap.keySet().contains(expr.getType()) || specialFunctionTextHashMap.keySet().contains(expr.getType()) From b3fc32174081e43d0b103925eef93f72f2ed8989 Mon Sep 17 00:00:00 2001 From: i1befree Date: Tue, 1 Apr 2014 14:17:01 +0900 Subject: [PATCH 5/5] using connection url, change database. ex) "jdbc:hive2://localhost:10000/mytest" will use mytest database --- jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java index 01d4abbf3617..254b0055420b 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java @@ -116,6 +116,9 @@ private void configureConnection(Utils.JdbcConnectionParams connParams) } else { // for remote JDBC client, try to set the conf var using 'set foo=bar' Statement stmt = createStatement(); + + stmt.execute("use " + connParams.getDbName()); + for (Entry hiveConf : connParams.getHiveConfs().entrySet()) { stmt.execute("set " + hiveConf.getKey() + "=" + hiveConf.getValue()); stmt.close();