From 8d0161cd52f6045b5a9a3b7e6efc81792ebe1deb Mon Sep 17 00:00:00 2001 From: akashshelke-6sense Date: Thu, 19 Feb 2026 20:14:25 +0530 Subject: [PATCH 1/2] Adding timestamps for console logging --- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 7 +++++++ .../hive/ql/exec/tez/monitoring/RenderStrategy.java | 11 +++++++---- .../apache/hadoop/hive/ql/session/SessionState.java | 10 +++++++++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index d055288e7369..fd513c2a19ef 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3741,6 +3741,10 @@ public static enum ConfVars { "hive.tez.exec.inplace.progress", true, "Updates tez job execution progress in-place in the terminal when hive-cli is used."), + HIVE_SESSION_STATE_TIMESTAMP( + "hive.session.state.timestamp", + false, + "If true, adds timestamp prefix to status messages like 'Launching Job' and 'Moving data to directory'."), HIVE_SERVER2_INPLACE_PROGRESS( "hive.server2.in.place.progress", true, @@ -3748,6 +3752,9 @@ public static enum ConfVars { + " only if the execution engine is tez."), TEZ_DAG_STATUS_CHECK_INTERVAL("hive.tez.dag.status.check.interval", "500ms", new TimeValidator(TimeUnit.MILLISECONDS), "Interval between subsequent DAG status invocation."), + TEZ_PROGRESS_PRINT_INTERVAL("hive.tez.progress.print.interval", "3s", + new TimeValidator(TimeUnit.MILLISECONDS), + "Interval between printing progress status for Map/Reducer tasks. Progress is printed at this fixed interval regardless of status changes."), SPARK_EXEC_INPLACE_PROGRESS("hive.spark.exec.inplace.progress", true, "Updates spark job execution progress in-place in the terminal."), TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION("hive.tez.container.max.java.heap.fraction", 0.8f, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java index 7cf8ed91e6c1..a788aba061c5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; +import java.util.concurrent.TimeUnit; class RenderStrategy { @@ -39,10 +40,9 @@ interface UpdateFunction { } private abstract static class BaseUpdateFunction implements UpdateFunction { - private static final int PRINT_INTERVAL = 3000; - final TezJobMonitor monitor; private final PerfLogger perfLogger; + private final long printInterval; private long lastPrintTime = 0L; private String lastReport = null; @@ -50,6 +50,10 @@ private abstract static class BaseUpdateFunction implements UpdateFunction { BaseUpdateFunction(TezJobMonitor monitor) { this.monitor = monitor; perfLogger = SessionState.getPerfLogger(); + this.printInterval = HiveConf.getTimeVar( + SessionState.get().getConf(), + HiveConf.ConfVars.TEZ_PROGRESS_PRINT_INTERVAL, + TimeUnit.MILLISECONDS); } @Override @@ -64,8 +68,7 @@ public void update(DAGStatus status, Map vertexProgressMap) { } private boolean showReport(String report) { - return !report.equals(lastReport) - || System.currentTimeMillis() >= lastPrintTime + PRINT_INTERVAL; + return System.currentTimeMillis() >= lastPrintTime + printInterval; } /* diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 711d4603db25..94f89ac7429d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -1226,7 +1226,15 @@ public void printInfo(String info, String detail) { */ public void printInfo(String info, String detail, boolean isSilent) { if (!isSilent) { - getInfoStream().println(info); + SessionState ss = SessionState.get(); + boolean addTimestamp = ss != null && ss.getConf() != null + && HiveConf.getBoolVar(ss.getConf(), HiveConf.ConfVars.HIVE_SESSION_STATE_TIMESTAMP); + if (addTimestamp) { + String timestamp = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new java.util.Date()); + getInfoStream().println(timestamp + "\t" + info); + } else { + getInfoStream().println(info); + } } LOG.info(info + StringUtils.defaultString(detail)); } From 61388d8e17b25a9bb7ab16328c2c60f63fc558ef Mon Sep 17 00:00:00 2001 From: akashshelke-6sense Date: Thu, 19 Feb 2026 21:02:57 +0530 Subject: [PATCH 2/2] print counters for hive cli too --- .../hadoop/hive/ql/exec/tez/TezTask.java | 21 +++++++++++++---- .../hadoop/hive/ql/session/SessionState.java | 23 +++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index a15482f19c43..ed1bce8d3fa1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -270,14 +270,25 @@ public int execute(DriverContext driverContext) { } } - if (LOG.isInfoEnabled() && counters != null + if (counters != null && console != null && (HiveConf.getBoolVar(conf, HiveConf.ConfVars.TEZ_EXEC_SUMMARY) || Utilities.isPerfOrAboveLogging(conf))) { - for (CounterGroup group: counters) { - LOG.info(group.getDisplayName() +":"); - for (TezCounter counter: group) { - LOG.info(" "+counter.getDisplayName()+": "+counter.getValue()); + try { + for (CounterGroup group : counters) { + if (group != null) { + String groupName = group.getDisplayName(); + console.printInfoNoTimestamp(groupName != null ? groupName + ":" : "Unknown Group:"); + for (TezCounter counter : group) { + if (counter != null) { + String counterName = counter.getDisplayName(); + console.printInfoNoTimestamp(" " + (counterName != null ? counterName : "Unknown") + + ": " + counter.getValue()); + } + } + } } + } catch (Exception e) { + LOG.warn("Failed to print counters to console. Ignoring.", e); } } } catch (Exception e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 94f89ac7429d..ff133c16c91f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -1239,6 +1239,29 @@ public void printInfo(String info, String detail, boolean isSilent) { LOG.info(info + StringUtils.defaultString(detail)); } + /** + * Logs info into the log file and console without timestamp prefix. + * Use this for output like counters where timestamps are not desired. + * @param info The log message + */ + public void printInfoNoTimestamp(String info) { + printInfoNoTimestamp(info, null, getIsSilent()); + } + + /** + * Logs info into the log file and console without timestamp prefix. + * Use this for output like counters where timestamps are not desired. + * @param info The log message + * @param detail Extra detail to log which will be not printed if null + * @param isSilent If true then the message will not be printed to the info stream + */ + public void printInfoNoTimestamp(String info, String detail, boolean isSilent) { + if (!isSilent) { + getInfoStream().println(info); + } + LOG.info(info + StringUtils.defaultString(detail)); + } + /** * Logs an error into the log file, and into the HiveServer2 or HiveCli error stream too. * BeeLine uses the operation log file to show the logs to the user, so depending on the