Skip to content

[GLUTEN-8063][VL] Export cpuNanos metric for InputIteratorTransformer #11724

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
n0r0shi wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[GLUTEN-8063][VL] Export cpuNanos metric for InputIteratorTransformer #11724

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions backends-velox/src/main/java/org/apache/gluten/metrics/Metrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ public class Metrics implements IMetrics {
public long[] outputBytes;
public long[] cpuCount;
public long[] wallNanos;
public long[] cpuNanos;
public long[] scanTime;
public long[] peakMemoryBytes;
public long[] numMemoryAllocations;
Expand Down Expand Up @@ -79,6 +80,7 @@ public Metrics(
long[] outputBytes,
long[] cpuCount,
long[] wallNanos,
long[] cpuNanos,
long veloxToArrow,
long[] peakMemoryBytes,
long[] numMemoryAllocations,
Expand Down Expand Up @@ -122,6 +124,7 @@ public Metrics(
this.outputBytes = outputBytes;
this.cpuCount = cpuCount;
this.wallNanos = wallNanos;
this.cpuNanos = cpuNanos;
this.scanTime = scanTime;
this.singleMetric.veloxToArrow = veloxToArrow;
this.peakMemoryBytes = peakMemoryBytes;
Expand Down Expand Up @@ -174,6 +177,7 @@ public OperatorMetrics getOperatorMetrics(int index) {
outputBytes[index],
cpuCount[index],
wallNanos[index],
cpuNanos[index],
peakMemoryBytes[index],
numMemoryAllocations[index],
spilledInputBytes[index],
Expand Down
3 changes: 3 additions & 0 deletions backends-velox/src/main/java/org/apache/gluten/metrics/OperatorMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ public class OperatorMetrics implements IOperatorMetrics {
public long outputBytes;
public long cpuCount;
public long wallNanos;
public long cpuNanos;
public long scanTime;
public long peakMemoryBytes;
public long numMemoryAllocations;
Expand Down Expand Up @@ -73,6 +74,7 @@ public OperatorMetrics(
long outputBytes,
long cpuCount,
long wallNanos,
long cpuNanos,
long peakMemoryBytes,
long numMemoryAllocations,
long spilledInputBytes,
Expand Down Expand Up @@ -114,6 +116,7 @@ public OperatorMetrics(
this.outputBytes = outputBytes;
this.cpuCount = cpuCount;
this.wallNanos = wallNanos;
this.cpuNanos = cpuNanos;
this.scanTime = scanTime;
this.peakMemoryBytes = peakMemoryBytes;
this.numMemoryAllocations = numMemoryAllocations;
Expand Down
3 changes: 2 additions & 1 deletion backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,8 @@ class VeloxMetricsApi extends MetricsApi with Logging {

Map(
"cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time count"),
"wallNanos" -> wallNanosMetric
"wallNanos" -> wallNanosMetric,
"cpuNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "cpu time")
) ++ outputMetrics
}

Expand Down
1 change: 1 addition & 0 deletions backends-velox/src/main/scala/org/apache/gluten/metrics/InputIteratorMetricsUpdater.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ case class InputIteratorMetricsUpdater(metrics: Map[String, SQLMetric], forBroad
val operatorMetrics = opMetrics.asInstanceOf[OperatorMetrics]
metrics("cpuCount") += operatorMetrics.cpuCount
metrics("wallNanos") += operatorMetrics.wallNanos
metrics("cpuNanos") += operatorMetrics.cpuNanos
if (!forBroadcast) {
if (operatorMetrics.outputRows == 0 && operatorMetrics.outputVectors == 0) {
// Sometimes, velox does not update metrics for intermediate operator,
Expand Down
3 changes: 2 additions & 1 deletion cpp/core/jni/JniWrapper.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) {
env,
metricsBuilderClass,
"<init>",
"([J[J[J[J[J[J[J[J[J[JJ[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[JLjava/lang/String;)V");
"([J[J[J[J[J[J[J[J[J[J[JJ[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[J[JLjava/lang/String;)V");

nativeColumnarToRowInfoClass =
createGlobalClassReferenceOrError(env, "Lorg/apache/gluten/vectorized/NativeColumnarToRowInfo;");
Expand Down Expand Up @@ -578,6 +578,7 @@ JNIEXPORT jobject JNICALL Java_org_apache_gluten_metrics_IteratorMetricsJniWrapp
longArray[Metrics::kOutputBytes],
longArray[Metrics::kCpuCount],
longArray[Metrics::kWallNanos],
longArray[Metrics::kCpuNanos],
metrics ? metrics->veloxToArrow : -1,
longArray[Metrics::kPeakMemoryBytes],
longArray[Metrics::kNumMemoryAllocations],
Expand Down
1 change: 1 addition & 0 deletions cpp/core/utils/Metrics.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ struct Metrics {
// CpuWallTiming.
kCpuCount,
kWallNanos,
kCpuNanos,

kPeakMemoryBytes,
kNumMemoryAllocations,
Expand Down
2 changes: 2 additions & 0 deletions cpp/velox/compute/WholeStageResultIterator.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,7 @@ void WholeStageResultIterator::collectMetrics() {
metrics_->get(Metrics::kOutputBytes)[metricIndex] = 0;
metrics_->get(Metrics::kCpuCount)[metricIndex] = 0;
metrics_->get(Metrics::kWallNanos)[metricIndex] = 0;
metrics_->get(Metrics::kCpuNanos)[metricIndex] = 0;
metrics_->get(Metrics::kPeakMemoryBytes)[metricIndex] = 0;
metrics_->get(Metrics::kNumMemoryAllocations)[metricIndex] = 0;
metricIndex += 1;
Expand All @@ -477,6 +478,7 @@ void WholeStageResultIterator::collectMetrics() {
metrics_->get(Metrics::kOutputBytes)[metricIndex] = second->outputBytes;
metrics_->get(Metrics::kCpuCount)[metricIndex] = second->cpuWallTiming.count;
metrics_->get(Metrics::kWallNanos)[metricIndex] = second->cpuWallTiming.wallNanos;
metrics_->get(Metrics::kCpuNanos)[metricIndex] = second->cpuWallTiming.cpuNanos;
metrics_->get(Metrics::kPeakMemoryBytes)[metricIndex] = second->peakMemoryBytes;
metrics_->get(Metrics::kNumMemoryAllocations)[metricIndex] = second->numMemoryAllocations;
metrics_->get(Metrics::kSpilledInputBytes)[metricIndex] = second->spilledInputBytes;
Expand Down