diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 4b89c8e274..ed5d0672b1 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -121,7 +121,7 @@ void initPropertiesAndConfigs() { // Injection of equality testing code of logical plan (de)serialization conf.setClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, LogicalPlanTestRuleProvider.class); conf.setClassVar(ConfVars.GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, GlobalPlanTestRuleProvider.class); - conf.setLongVar(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD, 1024 * 1024); // 1GB + conf.setLongVar(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_LIMIT, 1024 * 1024); // 1GB conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES.varname, 4); conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB.varname, 2000); diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index ab00a41749..15d57a6b61 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -94,10 +94,10 @@ public enum SessionVars implements ConfigKey { Validators.bool()), // for distributed query strategies - BROADCAST_NON_CROSS_JOIN_THRESHOLD(ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD, + BROADCAST_NON_CROSS_JOIN_THRESHOLD(ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_LIMIT, "restriction for the total size of broadcasted table for non-cross join (kb)", DEFAULT, Long.class, Validators.min("0")), - BROADCAST_CROSS_JOIN_THRESHOLD(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD, + BROADCAST_CROSS_JOIN_THRESHOLD(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_LIMIT, "restriction for the total size of broadcasted table for cross join (kb)", DEFAULT, Long.class, Validators.min("0")), @@ -125,13 +125,13 @@ public enum SessionVars implements ConfigKey { Integer.class, Validators.min("0")), HASH_SHUFFLE_BUFFER_SIZE(ConfVars.$EXECUTOR_HASH_SHUFFLE_BUFFER_SIZE, "hash-shuffle buffer size for local disk I/O (mb)" , DEFAULT, Integer.class, Validators.min("1")), - HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash join (mb)", DEFAULT, + HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT, "limited size for hash join (mb)", DEFAULT, Long.class, Validators.min("0")), - INNER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_INNER_HASH_JOIN_SIZE_THRESHOLD, + INNER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_INNER_HASH_JOIN_SIZE_LIMIT, "limited size for hash inner join (mb)", DEFAULT, Long.class, Validators.min("0")), - OUTER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_OUTER_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash outer join (mb)", + OUTER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_OUTER_HASH_JOIN_SIZE_LIMIT, "limited size for hash outer join (mb)", DEFAULT, Long.class, Validators.min("0")), - HASH_GROUPBY_SIZE_LIMIT(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD, "limited size for hash groupby (mb)", + HASH_GROUPBY_SIZE_LIMIT(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_LIMIT, "limited size for hash groupby (mb)", DEFAULT, Long.class, Validators.min("0")), MAX_OUTPUT_FILE_SIZE(ConfVars.$MAX_OUTPUT_FILE_SIZE, "Maximum per-output file size (mb). 0 means infinite.", DEFAULT, Long.class, Validators.min("0")), @@ -145,7 +145,7 @@ public enum SessionVars implements ConfigKey { // for index INDEX_ENABLED(ConfVars.$INDEX_ENABLED, "index scan enabled", DEFAULT), - INDEX_SELECTIVITY_THRESHOLD(ConfVars.$INDEX_SELECTIVITY_THRESHOLD, "the selectivity threshold for index scan", + INDEX_SELECTIVITY_LIMIT(ConfVars.$INDEX_SELECTIVITY_LIMIT, "the selectivity threshold for index scan", DEFAULT), // for partition overwrite @@ -176,7 +176,7 @@ public enum SessionVars implements ConfigKey { TEST_FILTER_PUSHDOWN_ENABLED(ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED, "filter push down enabled", TEST_VAR), TEST_MIN_TASK_NUM(ConfVars.$TEST_MIN_TASK_NUM, "(test only) min task num", TEST_VAR), TEST_PLAN_SHAPE_FIX_ENABLED(ConfVars.$TEST_PLAN_SHAPE_FIX_ENABLED, "(test only) plan shape fix enabled", TEST_VAR), - TEST_TIM_SORT_THRESHOLD_FOR_RADIX_SORT(ConfVars.$TEST_TIM_SORT_THRESHOLD_FOR_RADIX_SORT, "(test only) Tim sort threshold for radix sort", TEST_VAR) + TEST_TIM_SORT_LIMIT_FOR_RADIX_SORT(ConfVars.$TEST_TIM_SORT_LIMIT_FOR_RADIX_SORT, "(test only) Tim sort threshold for radix sort", TEST_VAR) ; public static final Map SESSION_VARS = Maps.newHashMap(); diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 6d24ee3d38..25bd5d5703 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -189,7 +189,7 @@ public static enum ConfVars implements ConfigKey { WORKER_RESOURCE_AVAILABLE_DISK_PARALLEL_NUM("tajo.worker.resource.disk.parallel-execution.num", 2, Validators.min("1")), - WORKER_HEARTBEAT_QUEUE_THRESHOLD_RATE("tajo.worker.heartbeat.queue.threshold-rate", 0.1f, Validators.min("0")),//10% + WORKER_HEARTBEAT_QUEUE_LIMIT_RATE("tajo.worker.heartbeat.queue.limit-rate", 0.1f, Validators.min("0")),//10% WORKER_HEARTBEAT_IDLE_INTERVAL("tajo.worker.heartbeat.idle.interval", 10 * 1000), // 10 sec WORKER_HEARTBEAT_ACTIVE_INTERVAL("tajo.worker.heartbeat.active.interval", 1000), // 1 sec @@ -333,9 +333,9 @@ public static enum ConfVars implements ConfigKey { // for distributed query strategies - $DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD("tajo.dist-query.broadcast.non-cross-join.threshold-kb", 5 * 1024l, + $DIST_QUERY_BROADCAST_NON_CROSS_JOIN_LIMIT("tajo.dist-query.broadcast.non-cross-join.limit-kb", 5 * 1024l, Validators.min("0")), // 5 MB - $DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD("tajo.dist-query.broadcast.cross-join.threshold-kb", 1 * 1024l, + $DIST_QUERY_BROADCAST_CROSS_JOIN_LIMIT("tajo.dist-query.broadcast.cross-join.limit-kb", 1 * 1024l, Validators.min("0")), // 1 MB $DIST_QUERY_JOIN_TASK_VOLUME("tajo.dist-query.join.task-volume-mb", 64), @@ -351,12 +351,12 @@ public static enum ConfVars implements ConfigKey { // for physical Executors $EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE("tajo.executor.external-sort.buffer-mb", 200), - $EXECUTOR_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.common.in-memory-hash-threshold-mb", 64l, Validators.min("0")), - $EXECUTOR_INNER_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.inner.in-memory-hash-threshold-mb", 64l, + $EXECUTOR_HASH_JOIN_SIZE_LIMIT("tajo.executor.join.common.in-memory-hash-limit-mb", 64l, Validators.min("0")), + $EXECUTOR_INNER_HASH_JOIN_SIZE_LIMIT("tajo.executor.join.inner.in-memory-hash-limit-mb", 64l, Validators.min("0")), - $EXECUTOR_OUTER_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.outer.in-memory-hash-threshold-mb", 64l, + $EXECUTOR_OUTER_HASH_JOIN_SIZE_LIMIT("tajo.executor.join.outer.in-memory-hash-limit-mb", 64l, Validators.min("0")), - $EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD("tajo.executor.groupby.in-memory-hash-threshold-mb", 64l, + $EXECUTOR_GROUPBY_INMEMORY_HASH_LIMIT("tajo.executor.groupby.in-memory-hash-limit-mb", 64l, Validators.min("0")), $EXECUTOR_HASH_SHUFFLE_BUFFER_SIZE("tajo.executor.hash-shuffle.buffer-mb", 100, Validators.min("1")), $MAX_OUTPUT_FILE_SIZE("tajo.query.max-outfile-size-mb", 0), // zero means infinite @@ -368,7 +368,7 @@ public static enum ConfVars implements ConfigKey { // for index $INDEX_ENABLED("tajo.query.index.enabled", false), - $INDEX_SELECTIVITY_THRESHOLD("tajo.query.index.selectivity.threshold", 0.05f), + $INDEX_SELECTIVITY_LIMIT("tajo.query.index.selectivity.limit", 0.05f), // Client ----------------------------------------------------------------- $CLIENT_SESSION_EXPIRY_TIME("tajo.client.session.expiry-time-sec", 3600), // default time is one hour. @@ -397,7 +397,7 @@ public static enum ConfVars implements ConfigKey { $TEST_FILTER_PUSHDOWN_ENABLED("tajo.test.plan.filter-pushdown.enabled", true), $TEST_MIN_TASK_NUM("tajo.test.min-task-num", -1), $TEST_PLAN_SHAPE_FIX_ENABLED("tajo.test.plan.shape.fix.enabled", false), // used for explain statement test - $TEST_TIM_SORT_THRESHOLD_FOR_RADIX_SORT("tajo.test.executor.radix-sort.tim-sort-threshold", 65536), + $TEST_TIM_SORT_LIMIT_FOR_RADIX_SORT("tajo.test.executor.radix-sort.tim-sort-limit", 65536), // Behavior Control --------------------------------------------------------- $BEHAVIOR_ARITHMETIC_ABORT("tajo.behavior.arithmetic-abort", false), diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRadixSort.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRadixSort.java index ce17ff0771..83ac32bf49 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRadixSort.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRadixSort.java @@ -63,7 +63,7 @@ public class TestRadixSort { static { queryContext = new QueryContext(new TajoConf()); - queryContext.setInt(SessionVars.TEST_TIM_SORT_THRESHOLD_FOR_RADIX_SORT, 0); + queryContext.setInt(SessionVars.TEST_TIM_SORT_LIMIT_FOR_RADIX_SORT, 0); schema = SchemaBuilder.builder().addAll(new Column[]{ new Column("col0", Type.INT8), diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java index f049aeb076..5b94f4651a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java @@ -39,7 +39,7 @@ public TestIndexScan() throws ServiceException, SQLException, NoSuchSessionVaria super(TajoConstants.DEFAULT_DATABASE_NAME); Map sessionVars = new HashMap<>(); sessionVars.put(SessionVars.INDEX_ENABLED.keyname(), "true"); - sessionVars.put(SessionVars.INDEX_SELECTIVITY_THRESHOLD.keyname(), "0.01f"); + sessionVars.put(SessionVars.INDEX_SELECTIVITY_LIMIT.keyname(), "0.01f"); client.updateSessionVariables(sessionVars); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java index 125a2ec677..9692add843 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java @@ -58,46 +58,46 @@ public TestJoinQuery(String joinOption) throws Exception { super(TajoConstants.DEFAULT_DATABASE_NAME, joinOption); testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "true"); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_LIMIT.varname, "" + 5); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_LIMIT.varname, 1024 * 1024 + ""); testingCluster.setAllTajoDaemonConfValue( - ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, - ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal); + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT.varname, + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT.defaultVal); testingCluster.setAllTajoDaemonConfValue(ConfVars.$JOIN_HASH_TABLE_SIZE.keyname(), "100"); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, - ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, - ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT.varname, + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_LIMIT.varname, + ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_LIMIT.defaultVal); if (joinOption.indexOf("NoBroadcast") >= 0) { - testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_LIMIT.varname, 1024 * 1024 + ""); testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false"); } if (joinOption.indexOf("Hash") >= 0) { testingCluster.setAllTajoDaemonConfValue( - ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, String.valueOf(256)); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT.varname, String.valueOf(256)); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT.varname, String.valueOf(256)); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_LIMIT.varname, String.valueOf(256)); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_LIMIT.varname, 1024 * 1024 + ""); } if (joinOption.indexOf("Sort") >= 0) { testingCluster.setAllTajoDaemonConfValue( - ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, String.valueOf(1)); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT.varname, String.valueOf(1)); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT.varname, String.valueOf(0)); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_LIMIT.varname, String.valueOf(0)); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_LIMIT.varname, 1024 * 1024 + ""); } } @@ -121,19 +121,19 @@ public static void setup() throws Exception { public static void classTearDown() throws SQLException { testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, ConfVars.$TEST_BROADCAST_JOIN_ENABLED.defaultVal); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD.varname, - ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD.defaultVal); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_LIMIT.varname, + ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_LIMIT.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_LIMIT.varname, 1024 * 1024 + ""); testingCluster.setAllTajoDaemonConfValue( - ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, - ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal); + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT.varname, + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT.defaultVal); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, - ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal); - testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, - ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT.varname, + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_LIMIT.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_LIMIT.varname, + ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_LIMIT.defaultVal); if (--reference == 0) { dropCommonTables(); diff --git a/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result b/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result index 5a6198e08a..d44ebef721 100644 --- a/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result +++ b/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result @@ -42,7 +42,7 @@ Available Session Variables: \set SORT_LIST_SIZE [int value] - The initial size of list for in-memory sort \set JOIN_HASH_TABLE_SIZE [int value] - The initial size of hash table for in-memory hash join \set INDEX_ENABLED [true or false] - index scan enabled -\set INDEX_SELECTIVITY_THRESHOLD [real value] - the selectivity threshold for index scan +\set INDEX_SELECTIVITY_LIMIT [real value] - the selectivity threshold for index scan \set PARTITION_NO_RESULT_OVERWRITE_ENABLED [true or false] - If true, a partitioned table is overwritten even if a sub query leads to no result. Otherwise, the table data will be kept if there is no result \set ARITHABORT [true or false] - If true, a running query will be terminated when an overflow or divide-by-zero occurs. \set FETCH_ROWNUM [int value] - The number of rows to be fetched from Master at a time diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RadixSort.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RadixSort.java index 780ef54a92..78c9f548c2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RadixSort.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RadixSort.java @@ -122,7 +122,7 @@ public static List sort(QueryContext queryContext, UnSafeTupleList Comparator comp) { UnSafeTuple[] in = list.toArray(new UnSafeTuple[list.size()]); RadixSortContext context = new RadixSortContext(in, schema, sortSpecs, comp, - queryContext.getInt(SessionVars.TEST_TIM_SORT_THRESHOLD_FOR_RADIX_SORT)); + queryContext.getInt(SessionVars.TEST_TIM_SORT_LIMIT_FOR_RADIX_SORT)); long before = System.currentTimeMillis(); recursiveCallForNextKey(context, 0, context.in.length, 0); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java index ab7e6edc06..eb64c85ae7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java @@ -94,7 +94,7 @@ public void serviceStart() throws Exception { NodeResources.createResource(systemConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY))); // if resource changed over than 30%, send reports - float queueingRate = systemConf.getFloatVar(TajoConf.ConfVars.WORKER_HEARTBEAT_QUEUE_THRESHOLD_RATE); + float queueingRate = systemConf.getFloatVar(TajoConf.ConfVars.WORKER_HEARTBEAT_QUEUE_LIMIT_RATE); this.queueingThreshold = Math.max((int) Math.floor(maxContainer * queueingRate), 1); LOG.info("Queueing threshold:" + queueingThreshold); diff --git a/tajo-docs/src/main/sphinx/configuration/tajo-site-xml.rst b/tajo-docs/src/main/sphinx/configuration/tajo-site-xml.rst index 2280897089..fc4e3fe4c7 100644 --- a/tajo-docs/src/main/sphinx/configuration/tajo-site-xml.rst +++ b/tajo-docs/src/main/sphinx/configuration/tajo-site-xml.rst @@ -27,10 +27,10 @@ A flag to enable or disable the use of broadcast join. true -.. _tajo.dist-query.broadcast.non-cross-join.threshold-kb: +.. _tajo.dist-query.broadcast.non-cross-join.limit-kb: """"""""""""""""""""""""""""""""""""""""""""""""""""""""""" -`tajo.dist-query.broadcast.non-cross-join.threshold-kb` +`tajo.dist-query.broadcast.non-cross-join.limit-kb` """"""""""""""""""""""""""""""""""""""""""""""""""""""""""" A threshold for non-cross joins. When a non-cross join query is executed with the broadcast join, the whole size of broadcasted tables won't exceed this threshold. @@ -43,14 +43,14 @@ A threshold for non-cross joins. When a non-cross join query is executed with th .. code-block:: xml - tajo.dist-query.broadcast.non-cross-join.threshold-kb + tajo.dist-query.broadcast.non-cross-join.limit-kb 5120 -.. _tajo.dist-query.broadcast.cross-join.threshold-kb: +.. _tajo.dist-query.broadcast.cross-join.limit-kb: """"""""""""""""""""""""""""""""""""""""""""""""""""""""""" -`tajo.dist-query.broadcast.cross-join.threshold-kb` +`tajo.dist-query.broadcast.cross-join.limit-kb` """"""""""""""""""""""""""""""""""""""""""""""""""""""""""" A threshold for cross joins. When a cross join query is executed, the whole size of broadcasted tables won't exceed this threshold. @@ -63,7 +63,7 @@ A threshold for cross joins. When a cross join query is executed, the whole size .. code-block:: xml - tajo.dist-query.broadcast.cross-join.threshold-kb + tajo.dist-query.broadcast.cross-join.limit-kb 1024 @@ -112,10 +112,10 @@ this value indicates the output size of each task at the first stage, which dete 128 -.. _tajo.executor.join.common.in-memory-hash-threshold-mb: +.. _tajo.executor.join.common.in-memory-hash-limit-mb: """""""""""""""""""""""""""""""""""""""""""""""""""""""" -`tajo.executor.join.common.in-memory-hash-threshold-mb` +`tajo.executor.join.common.in-memory-hash-limit-mb` """""""""""""""""""""""""""""""""""""""""""""""""""""""" This value provides the criterion to decide the algorithm to perform a join in a task. @@ -130,7 +130,7 @@ Otherwise, the sort-merge join is used. .. code-block:: xml - tajo.executor.join.common.in-memory-hash-threshold-mb + tajo.executor.join.common.in-memory-hash-limit-mb 64 @@ -139,10 +139,10 @@ Otherwise, the sort-merge join is used. its actual size is usually much larger than the configured value, which means that too large threshold can cause unexpected OutOfMemory errors. This value should be tuned carefully. -.. _tajo.executor.join.inner.in-memory-hash-threshold-mb: +.. _tajo.executor.join.inner.in-memory-hash-limit-mb: """""""""""""""""""""""""""""""""""""""""""""""""""""""" -`tajo.executor.join.inner.in-memory-hash-threshold-mb` +`tajo.executor.join.inner.in-memory-hash-limit-mb` """""""""""""""""""""""""""""""""""""""""""""""""""""""" This value provides the criterion to decide the algorithm to perform an inner join in a task. @@ -157,7 +157,7 @@ Otherwise, the sort-merge join is used. .. code-block:: xml - tajo.executor.join.inner.in-memory-hash-threshold-mb + tajo.executor.join.inner.in-memory-hash-limit-mb 64 @@ -166,10 +166,10 @@ Otherwise, the sort-merge join is used. its actual size is usually much larger than the configured value, which means that too large threshold can cause unexpected OutOfMemory errors. This value should be tuned carefully. -.. _tajo.executor.join.outer.in-memory-hash-threshold-mb: +.. _tajo.executor.join.outer.in-memory-hash-limit-mb: """""""""""""""""""""""""""""""""""""""""""""""""""""""" -`tajo.executor.join.outer.in-memory-hash-threshold-mb` +`tajo.executor.join.outer.in-memory-hash-limit-mb` """""""""""""""""""""""""""""""""""""""""""""""""""""""" This value provides the criterion to decide the algorithm to perform an outer join in a task. @@ -184,7 +184,7 @@ Otherwise, the sort-merge join is used. .. code-block:: xml - tajo.executor.join.outer.in-memory-hash-threshold-mb + tajo.executor.join.outer.in-memory-hash-limit-mb 64 @@ -342,10 +342,10 @@ As a result, it determines the degree of the parallel processing of the aggregat 64 -.. _tajo.executor.groupby.in-memory-hash-threshold-mb: +.. _tajo.executor.groupby.in-memory-hash-limit-mb: """""""""""""""""""""""""""""""""""""""""""""""""""""""" -`tajo.executor.groupby.in-memory-hash-threshold-mb` +`tajo.executor.groupby.in-memory-hash-limit-mb` """""""""""""""""""""""""""""""""""""""""""""""""""""""" This value provides the criterion to decide the algorithm to perform an aggregation in a task. @@ -360,7 +360,7 @@ Otherwise, the sort-based aggregation is used. .. code-block:: xml - tajo.executor.groupby.in-memory-hash-threshold-mb + tajo.executor.groupby.in-memory-hash-limit-mb 64 diff --git a/tajo-docs/src/main/sphinx/sql_language/joins.rst b/tajo-docs/src/main/sphinx/sql_language/joins.rst index 7402063dd3..05fcd97708 100644 --- a/tajo-docs/src/main/sphinx/sql_language/joins.rst +++ b/tajo-docs/src/main/sphinx/sql_language/joins.rst @@ -157,4 +157,4 @@ Tajo provides a session variable for broadcast join configuration. (For more det * ``BROADCAST_NON_CROSS_JOIN_THRESHOLD`` and ``BROADCAST_CROSS_JOIN_THRESHOLD`` are thresholds for broadcast join. Only the relations who are larger than this threshold can be broadcasted. -You can also apply this configuration system widely by setting ``tajo.dist-query.broadcast.non-cross-join.threshold-kb`` or ``tajo.dist-query.broadcast.cross-join.threshold-kb`` in ``${TAJO_HOME}/conf/tajo-site.xml``. \ No newline at end of file +You can also apply this configuration system widely by setting ``tajo.dist-query.broadcast.non-cross-join.limit-kb`` or ``tajo.dist-query.broadcast.cross-join.limit-kb`` in ``${TAJO_HOME}/conf/tajo-site.xml``. \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/tsql/variables.rst b/tajo-docs/src/main/sphinx/tsql/variables.rst index 3f787e23f5..21989e8982 100644 --- a/tajo-docs/src/main/sphinx/tsql/variables.rst +++ b/tajo-docs/src/main/sphinx/tsql/variables.rst @@ -36,7 +36,7 @@ Currently, tajo provides the following session variables. A threshold for non-cross joins. When a non-cross join query is executed with the broadcast join, the whole size of broadcasted tables won't exceed this threshold. - * Configuration name: :ref:`tajo.dist-query.broadcast.non-cross-join.threshold-kb` + * Configuration name: :ref:`tajo.dist-query.broadcast.non-cross-join.limit-kb` * Property value: Integer * Unit: KB * Default value: 5120 @@ -50,7 +50,7 @@ A threshold for non-cross joins. When a non-cross join query is executed with th A threshold for cross joins. When a cross join query is executed, the whole size of broadcasted tables won't exceed this threshold. - * Configuration name: :ref:`tajo.dist-query.broadcast.cross-join.threshold-kb` + * Configuration name: :ref:`tajo.dist-query.broadcast.cross-join.limit-kb` * Property value: Integer * Unit: KB * Default value: 1024 @@ -99,7 +99,7 @@ This value provides the criterion to decide the algorithm to perform a join in a If the input data is smaller than this value, join is performed with the in-memory hash join. Otherwise, the sort-merge join is used. - * Configuration name: :ref:`tajo.executor.join.common.in-memory-hash-threshold-mb` + * Configuration name: :ref:`tajo.executor.join.common.in-memory-hash-limit-mb` * Property value: Integer * Unit: MB * Default value: 64 @@ -120,7 +120,7 @@ This value provides the criterion to decide the algorithm to perform an inner jo If the input data is smaller than this value, the inner join is performed with the in-memory hash join. Otherwise, the sort-merge join is used. - * Configuration name: :ref:`tajo.executor.join.inner.in-memory-hash-threshold-mb` + * Configuration name: :ref:`tajo.executor.join.inner.in-memory-hash-limit-mb` * Property value: Integer * Unit: MB * Default value: 64 @@ -141,7 +141,7 @@ This value provides the criterion to decide the algorithm to perform an outer jo If the input data is smaller than this value, the outer join is performed with the in-memory hash join. Otherwise, the sort-merge join is used. - * Configuration name: :ref:`tajo.executor.join.outer.in-memory-hash-threshold-mb` + * Configuration name: :ref:`tajo.executor.join.outer.in-memory-hash-limit-mb` * Property value: Integer * Unit: MB * Default value: 64 @@ -261,7 +261,7 @@ This value provides the criterion to decide the algorithm to perform an aggregat If the input data is smaller than this value, the aggregation is performed with the in-memory hash aggregation. Otherwise, the sort-based aggregation is used. - * Configuration name: :ref:`tajo.executor.groupby.in-memory-hash-threshold-mb` + * Configuration name: :ref:`tajo.executor.groupby.in-memory-hash-limit-mb` * Property value: Integer * Unit: MB * Default value: 64 diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java index 7ba5f8b941..b40d8225a5 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java @@ -102,7 +102,7 @@ public Object visitScan(Object object, LogicalPlan plan, LogicalPlan.QueryBlock // estimation selectivity and choose the better path // TODO: improve the selectivity estimation double estimateSelectivity = 0.001; - double selectivityThreshold = conf.getFloat(SessionVars.INDEX_SELECTIVITY_THRESHOLD); + double selectivityThreshold = conf.getFloat(SessionVars.INDEX_SELECTIVITY_LIMIT); LOG.info("Selectivity threshold: " + selectivityThreshold); LOG.info("Estimated selectivity: " + estimateSelectivity); if (estimateSelectivity < selectivityThreshold) { diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java index 34c9238396..5b7483444f 100644 --- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java +++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java @@ -93,9 +93,9 @@ public void testTPCH_Q2_Part_MixedStorage() throws Exception { try { testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "true"); - testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_LIMIT.varname, "" + (5 * 1024)); - testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_LIMIT.varname, 1024 * 1024 + ""); runSimpleTests(); @@ -103,9 +103,9 @@ public void testTPCH_Q2_Part_MixedStorage() throws Exception { } finally { testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.defaultVal); - testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD.varname, - TajoConf.ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD.defaultVal); - testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_LIMIT.varname, + TajoConf.ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_LIMIT.defaultVal); + testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_LIMIT.varname, 1024 * 1024 + ""); } }