diff --git a/app/src/main/java/com/facebook/encapp/MainActivity.java b/app/src/main/java/com/facebook/encapp/MainActivity.java index 08bcd4a4..7c3da171 100644 --- a/app/src/main/java/com/facebook/encapp/MainActivity.java +++ b/app/src/main/java/com/facebook/encapp/MainActivity.java @@ -173,6 +173,12 @@ protected void onCreate(Bundle savedInstanceState) { // need to check permission strategy getTestSettings(); CliSettings.setWorkDir(this, mExtraData); + + // Check if performance tracing is enabled + if (mExtraData != null && mExtraData.containsKey(CliSettings.ENABLE_TRACING)) { + CliSettings.setEnableTracing(mExtraData.getBoolean(CliSettings.ENABLE_TRACING, false)); + } + boolean useNewMethod = true; if (mExtraData != null && mExtraData.size() > 0) { useNewMethod = !mExtraData.getBoolean(CliSettings.OLD_AUTH_METHOD, false); diff --git a/app/src/main/java/com/facebook/encapp/SurfaceEncoder.java b/app/src/main/java/com/facebook/encapp/SurfaceEncoder.java index b278e3db..93158799 100644 --- a/app/src/main/java/com/facebook/encapp/SurfaceEncoder.java +++ b/app/src/main/java/com/facebook/encapp/SurfaceEncoder.java @@ -21,7 +21,9 @@ import com.facebook.encapp.proto.PixFmt; import com.facebook.encapp.proto.Test; -import com.facebook.encapp.utils.FakeInputReader; +import com.facebook.encapp.utils.CliSettings; +import com.facebook.encapp.utils.ClockTimes; +import com.facebook.encapp.utils.FakeGLRenderer; import com.facebook.encapp.utils.FileReader; import com.facebook.encapp.utils.FpsMeasure; import com.facebook.encapp.utils.FrameswapControl; @@ -38,6 +40,7 @@ import java.lang.NullPointerException; import java.nio.ByteBuffer; import java.util.Locale; +import java.util.concurrent.CountDownLatch; /** @@ -53,7 +56,7 @@ class SurfaceEncoder extends Encoder implements VsyncListener { boolean mIsRgbaSource = false; boolean mIsCameraSource = false; boolean mIsFakeInput = false; - FakeInputReader mFakeInputReader; + FakeGLRenderer mFakeGLRenderer; // GL-based fake input (fast!) boolean mUseCameraTimestamp = true; OutputMultiplier mOutputMult; Bundle mKeyFrameBundle; @@ -114,7 +117,8 @@ public String encode( mRefFramesizeInBytes = width * height * 4; } else if (mTest.getInput().getFilepath().equals("fake_input")) { mIsFakeInput = true; - Log.d(TAG, "Using fake input for performance testing"); + // Use GL rendering for fake input - ZERO CPU overhead! + Log.d(TAG, "Using fake input with GL rendering for performance testing"); } else if (mTest.getInput().getFilepath().equals("camera")) { mIsCameraSource = true; //TODO: handle other fps (i.e. try to set lower or higher fps) @@ -152,26 +156,35 @@ public String encode( } if (!mIsCameraSource) { - mBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); + if (!mIsFakeInput) { + // Only create bitmap for non-GL paths + mBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); + } if (mIsFakeInput) { - mFakeInputReader = new FakeInputReader(); - if (!mFakeInputReader.openFile(mTest.getInput().getFilepath(), mTest.getInput().getPixFmt(), width, height)) { - return "Could not initialize fake input"; - } + // Initialize FakeGLRenderer (will be set up later on GL thread) + mFakeGLRenderer = new FakeGLRenderer(); + mFakeGLRenderer.setPatternType(FakeGLRenderer.PatternType.TEXTURE); + Log.d(TAG, "Created FakeGLRenderer for GL-based fake input"); + // Initialize on GL thread after OutputMultiplier is ready } else { mYuvReader = new FileReader(); if (!mYuvReader.openFile(mTest.getInput().getFilepath(), mTest.getInput().getPixFmt())) { return "Could not open file"; } } - } MediaFormat format; try { + // Surface encoding requires OutputMultiplier - create one if not provided + if (mOutputMult == null) { + Log.d(TAG, "Creating OutputMultiplier for surface encoding (no display output)"); + mOutputMult = new OutputMultiplier(mVsyncHandler); + } + // Unless we have a mime, do lookup if (mTest.getConfigure().getMime().length() == 0) { Log.d(TAG, "codec id: " + mTest.getConfigure().getCodec()); @@ -254,6 +267,17 @@ public String encode( int current_loop = 1; ByteBuffer byteBuffer = ByteBuffer.allocate(mRefFramesizeInBytes); boolean done = false; + + // For file input, we're immediately stable (no warmup needed) + // For fake input with GL, initialization will happen on first frame render + if (!mIsCameraSource && mYuvReader != null) { + mStable = true; + } else if (mIsFakeInput && mFakeGLRenderer != null) { + // GL renderer will be initialized on first frame (on GL thread) + mStable = true; + Log.i(TAG, "FakeGLRenderer ready (will init on GL thread)"); + } + synchronized (this) { Log.d(TAG, "Wait for synchronized start"); try { @@ -264,6 +288,7 @@ public String encode( } } mStats.start(); + int errorCounter = 0; while (!done) { if (mFramesAdded % 100 == 0 && MainActivity.isStable()) { @@ -320,7 +345,7 @@ public String encode( } else { mFrameSwapSurface.dropNext(false); long ptsUsec = 0; - if (mUseCameraTimestamp) { + if (mUseCameraTimestamp && mIsCameraSource) { // Use the camera provided timestamp ptsUsec = mPts + (long) (timestampUsec - mFirstFrameTimestampUsec); } else { @@ -344,10 +369,9 @@ public String encode( } else { if (MainActivity.isStable()) { - while (size < 0 && !done) { try { - if (mYuvReader != null) { + if (mYuvReader != null || mIsFakeInput) { size = queueInputBufferEncoder( mYuvReader, mCodec, @@ -456,6 +480,10 @@ public String encode( if (mYuvReader != null) mYuvReader.closeFile(); + if (mFakeGLRenderer != null) { + mFakeGLRenderer.release(); + } + if (mSurfaceTexture != null) { mSurfaceTexture.detachFromGLContext(); mSurfaceTexture.releaseTexImage(); @@ -476,6 +504,14 @@ protected void checkRealtime() { mRealtime = true; } } + + // Fake GL input doesn't need realtime throttling - it's synthetic data + // Let it run as fast as the encoder can handle + if (mIsFakeInput) { + Log.d(TAG, "Fake GL input detected - disabling realtime throttling for max performance"); + mRealtime = false; + } + if (!mRealtime) { if (mOutputMult != null) { Log.d(TAG, "Outputmultiplier will work in non realtime mode"); @@ -497,54 +533,156 @@ private void setupOutputMult(int width, int height) { Log.d(TAG, "Outputmultiplier will work in non realtime mode"); mOutputMult.setRealtime(false); } + } else if (mIsFakeInput && !mTest.getInput().hasShow()) { + // Fake input without UI display doesn't need vsync synchronization + Log.d(TAG, "Fake input without display - disabling vsync synchronization"); + mOutputMult.setRealtime(false); } } } /** - * Fills input buffer for encoder from YUV buffers. + * Fills input buffer for encoder from YUV file or fake GL input. + * OPTIMIZED: GL path has ZERO Java/CPU overhead. * * @return size of enqueued data. */ private int queueInputBufferEncoder( FileReader fileReader, MediaCodec codec, ByteBuffer byteBuffer, int frameCount, int flags, int size) { - byteBuffer.clear(); - int read = 0; - if (mIsFakeInput) { - read = mFakeInputReader.fillBuffer(byteBuffer, size); + + int read; + if (mIsFakeInput && mFakeGLRenderer != null) { + // GL rendering path - ZERO CPU overhead! + long ptsUsec = computePresentationTimeUs(mPts, mInFramesCount, mRefFrameTime); + setRuntimeParameters(mInFramesCount); + mDropNext = dropFrame(mInFramesCount); + mDropNext |= dropFromDynamicFramerate(mInFramesCount); + updateDynamicFramerate(mInFramesCount); + + if (mDropNext) { + mSkipped++; + mDropNext = false; + read = -2; + } else { + mFramesAdded++; + if (mFirstFrameTimestampUsec == -1) { + mFirstFrameTimestampUsec = ptsUsec; + } + + // Direct GL rendering - no bitmap, no copying! + mOutputMult.newGLPatternFrame(mFakeGLRenderer, ptsUsec, frameCount, mStats); + + // NOTE: startEncodingFrame is NOT called here. It will be called AFTER swapBuffers() + // in OutputMultiplier to measure only the encoder time, not GL rendering time. + read = size; // Success + + // Apply realtime throttling if enabled + if (mRealtime) { + sleepUntilNextFrame(); + } + } } else { + // Original bitmap path for YUV from disk + long t0 = 0, t1 = 0, t2 = 0, t3 = 0, t4 = 0, t5 = 0, t6 = 0, t7 = 0, t8 = 0, t9 = 0, t10 = 0, t11 = 0, t12 = 0; + + if (CliSettings.isTracingEnabled()) { + t0 = ClockTimes.currentTimeMs(); + } + byteBuffer.clear(); + + if (CliSettings.isTracingEnabled()) { + t1 = ClockTimes.currentTimeMs(); + } + // Regular file input needs manual YUV->RGBA conversion read = fileReader.fillBuffer(byteBuffer, size); - } - long ptsUsec = computePresentationTimeUs(mPts, mInFramesCount, mRefFrameTime); - setRuntimeParameters(mInFramesCount); - mDropNext = dropFrame(mInFramesCount); - mDropNext |= dropFromDynamicFramerate(mInFramesCount); - updateDynamicFramerate(mInFramesCount); - if (mDropNext) { - mSkipped++; - mDropNext = false; - read = -2; - } else if (read == size) { - mFramesAdded++; - if (!mIsRgbaSource) { - mYuvIn.copyFrom(byteBuffer.array()); - yuvToRgbIntrinsic.setInput(mYuvIn); - yuvToRgbIntrinsic.forEach(mYuvOut); - - mYuvOut.copyTo(mBitmap); - } else { - mBitmap.copyPixelsFromBuffer(byteBuffer); + if (CliSettings.isTracingEnabled()) { + t2 = ClockTimes.currentTimeMs(); + } + if (read == size) { + if (!mIsRgbaSource) { + if (CliSettings.isTracingEnabled()) { + t3 = ClockTimes.currentTimeMs(); + } + mYuvIn.copyFrom(byteBuffer.array()); + if (CliSettings.isTracingEnabled()) { + t4 = ClockTimes.currentTimeMs(); + } + yuvToRgbIntrinsic.setInput(mYuvIn); + if (CliSettings.isTracingEnabled()) { + t5 = ClockTimes.currentTimeMs(); + } + yuvToRgbIntrinsic.forEach(mYuvOut); + if (CliSettings.isTracingEnabled()) { + t6 = ClockTimes.currentTimeMs(); + } + mYuvOut.copyTo(mBitmap); + if (CliSettings.isTracingEnabled()) { + t7 = ClockTimes.currentTimeMs(); + if (frameCount < 10) { + Log.d(TAG, "Frame " + frameCount + " [YUV] fileRead: " + (t2-t1) + "ms, copyFrom: " + (t4-t3) + "ms, setInput: " + (t5-t4) + "ms, forEach: " + (t6-t5) + "ms, copyTo: " + (t7-t6) + "ms, TOTAL: " + (t7-t1) + "ms"); + } + } + } else { + mBitmap.copyPixelsFromBuffer(byteBuffer); + if (CliSettings.isTracingEnabled()) { + t3 = ClockTimes.currentTimeMs(); + if (frameCount < 10) { + Log.d(TAG, "Frame " + frameCount + " [RGBA] fileRead: " + (t2-t1) + "ms, copyPixels: " + (t3-t2) + "ms"); + } + } + } } - if (mFirstFrameTimestampUsec == -1) { - mFirstFrameTimestampUsec = ptsUsec; + if (CliSettings.isTracingEnabled()) { + t8 = ClockTimes.currentTimeMs(); + } + long ptsUsec = computePresentationTimeUs(mPts, mInFramesCount, mRefFrameTime); + setRuntimeParameters(mInFramesCount); + mDropNext = dropFrame(mInFramesCount); + mDropNext |= dropFromDynamicFramerate(mInFramesCount); + updateDynamicFramerate(mInFramesCount); + if (CliSettings.isTracingEnabled()) { + t9 = ClockTimes.currentTimeMs(); + } + + if (mDropNext) { + mSkipped++; + mDropNext = false; + read = -2; + } else if (read == size) { + mFramesAdded++; + + if (mFirstFrameTimestampUsec == -1) { + mFirstFrameTimestampUsec = ptsUsec; + } + + if (CliSettings.isTracingEnabled()) { + t10 = ClockTimes.currentTimeMs(); + } + mOutputMult.newBitmapAvailable(mBitmap, ptsUsec, frameCount, mStats); + if (CliSettings.isTracingEnabled()) { + t11 = ClockTimes.currentTimeMs(); + } + + // NOTE: startEncodingFrame is NOT called here. It will be called AFTER swapBuffers() + // in OutputMultiplier to measure only the encoder time, not YUV processing time. + if (CliSettings.isTracingEnabled()) { + t12 = ClockTimes.currentTimeMs(); + if (frameCount < 10) { + Log.d(TAG, "Frame " + frameCount + " paramCalc: " + (t9-t8) + "ms, newBitmapAvailable: " + (t11-t10) + "ms, TOTAL_QUEUE: " + (t12-t0) + "ms"); + } + } + + // Apply realtime throttling if enabled - AFTER all processing is done + if (mRealtime) { + sleepUntilNextFrame(); + } + } else { + Log.d(TAG, "***************** FAILED READING SURFACE ENCODER ******************"); + return -1; } - mOutputMult.newBitmapAvailable(mBitmap, ptsUsec); - mStats.startEncodingFrame(ptsUsec, frameCount); - } else { - Log.d(TAG, "***************** FAILED READING SURFACE ENCODER ******************"); - return -1; } + mInFramesCount++; return read; } @@ -561,6 +699,10 @@ public void release() { mOutputMult.stopAndRelease(); } + public OutputMultiplier getOutputMultiplier() { + return mOutputMult; + } + @Override public void vsync(long frameTimeNs) { synchronized (mSyncLock) { diff --git a/app/src/main/java/com/facebook/encapp/SurfaceTranscoder.java b/app/src/main/java/com/facebook/encapp/SurfaceTranscoder.java index 2c77ae64..a6198606 100644 --- a/app/src/main/java/com/facebook/encapp/SurfaceTranscoder.java +++ b/app/src/main/java/com/facebook/encapp/SurfaceTranscoder.java @@ -469,9 +469,10 @@ public void readFromBuffer(@NonNull MediaCodec codec, int index, boolean encoder mDropNext = false; codec.releaseOutputBuffer(index, false); } else { - mStats.startEncodingFrame(ptsUsec, mInFramesCount); + // NOTE: startEncodingFrame is NOT called here. It will be called AFTER swapBuffers() + // in OutputMultiplier to measure only the encoder time. mFramesAdded++; - mOutputMult.newFrameAvailableInBuffer(codec, index, info); + mOutputMult.newFrameAvailableInBuffer(codec, index, info, mInFramesCount, mStats); } } else { mCurrentTimeSec = diffUsec/1000000.0f; @@ -493,7 +494,8 @@ public void readFromBuffer(@NonNull MediaCodec codec, int index, boolean encoder if (mDropNext) { codec.releaseOutputBuffer(index, false); } else { - mOutputMult.newFrameAvailableInBuffer(codec, index, info); + // Decode-only mode: no encoder, so pass 0 for frameCount and null for stats + mOutputMult.newFrameAvailableInBuffer(codec, index, info, 0, null); } } } else { diff --git a/app/src/main/java/com/facebook/encapp/utils/CliSettings.java b/app/src/main/java/com/facebook/encapp/utils/CliSettings.java index 3cb216ba..dd91497a 100644 --- a/app/src/main/java/com/facebook/encapp/utils/CliSettings.java +++ b/app/src/main/java/com/facebook/encapp/utils/CliSettings.java @@ -17,8 +17,10 @@ public class CliSettings { public static final String WORKDIR = "workdir"; // Either /sdcard/ or /data/data/com.facebook.encapp public static final String CHECK_WORKDIR = "check_workdir"; + public static final String ENABLE_TRACING = "enable_tracing"; private static String mWorkDir = "/sdcard/"; + private static boolean mEnableTracing = false; public static void setWorkDir(Context context, Bundle mExtraData) { if (mExtraData != null && mExtraData.containsKey(CliSettings.WORKDIR)) { @@ -40,4 +42,13 @@ public static void setWorkDir(Context context, Bundle mExtraData) { public static String getWorkDir() { return mWorkDir; } + + public static void setEnableTracing(boolean enable) { + mEnableTracing = enable; + Log.d(TAG, "Performance tracing: " + (enable ? "enabled" : "disabled")); + } + + public static boolean isTracingEnabled() { + return mEnableTracing; + } } diff --git a/app/src/main/java/com/facebook/encapp/utils/FakeGLRenderer.java b/app/src/main/java/com/facebook/encapp/utils/FakeGLRenderer.java new file mode 100644 index 00000000..9b116a38 --- /dev/null +++ b/app/src/main/java/com/facebook/encapp/utils/FakeGLRenderer.java @@ -0,0 +1,290 @@ +package com.facebook.encapp.utils; + +import android.opengl.GLES20; +import android.util.Log; + +import com.facebook.encapp.utils.grafika.EglCore; +import com.facebook.encapp.utils.grafika.WindowSurface; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; + +/** + * FakeGLRenderer generates synthetic video frames using OpenGL ES shaders. + * This renders directly to GL surfaces with zero CPU overhead - patterns are + * generated entirely on the GPU using fragment shaders. + * + */ +public class FakeGLRenderer { + private static final String TAG = "encapp.fake_gl_renderer"; + + // Vertex shader - simple passthrough for full-screen quad + private static final String VERTEX_SHADER = + "attribute vec4 aPosition;\n" + + "attribute vec2 aTexCoord;\n" + + "varying vec2 vTexCoord;\n" + + "void main() {\n" + + " gl_Position = aPosition;\n" + + " vTexCoord = aTexCoord;\n" + + "}\n"; + + // Fragment shader - generates animated gradient pattern + private static final String FRAGMENT_SHADER_GRADIENT = + "precision mediump float;\n" + + "varying vec2 vTexCoord;\n" + + "uniform float uTime;\n" + + "void main() {\n" + + " // Animated gradient that moves and pulses\n" + + " float wave = sin(uTime * 2.0);\n" + + " float r = 0.5 + 0.3 * vTexCoord.x + 0.2 * sin(uTime + vTexCoord.y * 3.14);\n" + + " float g = 0.5 + 0.3 * vTexCoord.y + 0.2 * cos(uTime + vTexCoord.x * 3.14);\n" + + " float b = 0.5 + 0.2 * (vTexCoord.x + vTexCoord.y) * 0.5 + 0.15 * wave;\n" + + " gl_FragColor = vec4(r, g, b, 1.0);\n" + + "}\n"; + + // Fragment shader - solid gray (for testing - most compressible) + private static final String FRAGMENT_SHADER_SOLID = + "precision mediump float;\n" + + "uniform float uTime;\n" + + "void main() {\n" + + " // Pulsating gray\n" + + " float gray = 0.5 + 0.3 * sin(uTime * 2.0);\n" + + " gl_FragColor = vec4(gray, gray, gray, 1.0);\n" + + "}\n"; + + // Fragment shader - animated checkerboard pattern + private static final String FRAGMENT_SHADER_TEXTURE = + "precision mediump float;\n" + + "varying vec2 vTexCoord;\n" + + "uniform float uTime;\n" + + "void main() {\n" + + " // Animated checkerboard pattern that moves and changes color\n" + + " float scale = 40.0;\n" + + " \n" + + " // Scroll the pattern over time\n" + + " vec2 coord = vTexCoord * scale + vec2(uTime * 2.0, uTime * 1.5);\n" + + " \n" + + " // Create checkerboard\n" + + " float check = mod(floor(coord.x) + floor(coord.y), 2.0);\n" + + " \n" + + " // Animated colors that pulse\n" + + " float pulse = sin(uTime * 2.0) * 0.2;\n" + + " float baseR = 0.4 + check * 0.2 + pulse;\n" + + " float baseG = 0.5 + check * 0.15 + pulse * 0.8;\n" + + " float baseB = 0.45 + check * 0.1 + pulse * 0.6;\n" + + " \n" + + " gl_FragColor = vec4(baseR, baseG, baseB, 1.0);\n" + + "}\n"; + + // Full-screen quad vertices + private static final float[] VERTICES = { + // Position (x, y) TexCoord (s, t) + -1.0f, -1.0f, 0.0f, 0.0f, // bottom-left + 1.0f, -1.0f, 1.0f, 0.0f, // bottom-right + -1.0f, 1.0f, 0.0f, 1.0f, // top-left + 1.0f, 1.0f, 1.0f, 1.0f // top-right + }; + + private static final int COORDS_PER_VERTEX = 2; + private static final int TEXCOORDS_PER_VERTEX = 2; + private static final int VERTEX_STRIDE = (COORDS_PER_VERTEX + TEXCOORDS_PER_VERTEX) * 4; // 4 bytes per float + + private FloatBuffer mVertexBuffer; + private int mProgram; + private int mPositionHandle; + private int mTexCoordHandle; + private int mTimeHandle; + + private long mFrameCount = 0; + private boolean mInitialized = false; + + public enum PatternType { + SOLID, // Solid gray - most compressible + GRADIENT, // Simple gradient + TEXTURE // Textured pattern - similar to real video + } + + private PatternType mPatternType = PatternType.TEXTURE; + + public FakeGLRenderer() { + } + + /** + * Initialize GL resources. Must be called on GL thread. + */ + public void init() { + if (mInitialized) { + Log.w(TAG, "Already initialized"); + return; + } + + // Create vertex buffer + mVertexBuffer = ByteBuffer.allocateDirect(VERTICES.length * 4) + .order(ByteOrder.nativeOrder()) + .asFloatBuffer(); + mVertexBuffer.put(VERTICES).position(0); + + // Create shader program + mProgram = createProgram(VERTEX_SHADER, getFragmentShaderForPattern(mPatternType)); + if (mProgram == 0) { + throw new RuntimeException("Failed to create GL program"); + } + + // Get attribute/uniform locations + mPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition"); + checkGlError("glGetAttribLocation aPosition"); + + mTexCoordHandle = GLES20.glGetAttribLocation(mProgram, "aTexCoord"); + checkGlError("glGetAttribLocation aTexCoord"); + + mTimeHandle = GLES20.glGetUniformLocation(mProgram, "uTime"); + checkGlError("glGetUniformLocation uTime"); + + mInitialized = true; + Log.d(TAG, "FakeGLRenderer initialized with pattern: " + mPatternType); + } + + /** + * Set the pattern type. Must call before init() or call release() + init() to switch. + */ + public void setPatternType(PatternType type) { + mPatternType = type; + if (mInitialized) { + Log.w(TAG, "Pattern type changed after init - need to release() and init() again"); + } + } + + /** + * Render a frame directly to the current GL context. + * Must be called on GL thread with proper context current. + * Handles lazy initialization on first call. + */ + public void renderFrame(long timestampUs) { + // Lazy init on first frame (ensures we're on GL thread with context current) + if (!mInitialized) { + init(); + } + + // Use shader program + GLES20.glUseProgram(mProgram); + checkGlError("glUseProgram"); + + // Set time uniform (for animation) + float timeValue = (float) timestampUs / 1000000.0f; // Convert to seconds + GLES20.glUniform1f(mTimeHandle, timeValue); + checkGlError("glUniform1f uTime"); + + // Enable vertex arrays + GLES20.glEnableVertexAttribArray(mPositionHandle); + GLES20.glEnableVertexAttribArray(mTexCoordHandle); + + // Set vertex data + mVertexBuffer.position(0); + GLES20.glVertexAttribPointer(mPositionHandle, COORDS_PER_VERTEX, + GLES20.GL_FLOAT, false, VERTEX_STRIDE, mVertexBuffer); + checkGlError("glVertexAttribPointer aPosition"); + + mVertexBuffer.position(COORDS_PER_VERTEX); + GLES20.glVertexAttribPointer(mTexCoordHandle, TEXCOORDS_PER_VERTEX, + GLES20.GL_FLOAT, false, VERTEX_STRIDE, mVertexBuffer); + checkGlError("glVertexAttribPointer aTexCoord"); + + // Draw full-screen quad + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + checkGlError("glDrawArrays"); + + // Cleanup + GLES20.glDisableVertexAttribArray(mPositionHandle); + GLES20.glDisableVertexAttribArray(mTexCoordHandle); + + mFrameCount++; + } + + /** + * Release GL resources. Must be called on GL thread. + */ + public void release() { + if (mProgram != 0) { + GLES20.glDeleteProgram(mProgram); + mProgram = 0; + } + mInitialized = false; + Log.d(TAG, "FakeGLRenderer released after " + mFrameCount + " frames"); + } + + private String getFragmentShaderForPattern(PatternType type) { + switch (type) { + case SOLID: + return FRAGMENT_SHADER_SOLID; + case GRADIENT: + return FRAGMENT_SHADER_GRADIENT; + case TEXTURE: + return FRAGMENT_SHADER_TEXTURE; + default: + return FRAGMENT_SHADER_TEXTURE; + } + } + + private int createProgram(String vertexSource, String fragmentSource) { + int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource); + if (vertexShader == 0) { + return 0; + } + + int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource); + if (fragmentShader == 0) { + return 0; + } + + int program = GLES20.glCreateProgram(); + checkGlError("glCreateProgram"); + + GLES20.glAttachShader(program, vertexShader); + checkGlError("glAttachShader vertex"); + + GLES20.glAttachShader(program, fragmentShader); + checkGlError("glAttachShader fragment"); + + GLES20.glLinkProgram(program); + int[] linkStatus = new int[1]; + GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0); + if (linkStatus[0] != GLES20.GL_TRUE) { + Log.e(TAG, "Could not link program: " + GLES20.glGetProgramInfoLog(program)); + GLES20.glDeleteProgram(program); + return 0; + } + + return program; + } + + private int loadShader(int type, String source) { + int shader = GLES20.glCreateShader(type); + checkGlError("glCreateShader type=" + type); + + GLES20.glShaderSource(shader, source); + GLES20.glCompileShader(shader); + + int[] compiled = new int[1]; + GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0); + if (compiled[0] == 0) { + String errorLog = GLES20.glGetShaderInfoLog(shader); + Log.e(TAG, "Could not compile shader " + type + ":"); + Log.e(TAG, "Shader source:\n" + source); + Log.e(TAG, "Error log: " + errorLog); + GLES20.glDeleteShader(shader); + return 0; + } + + return shader; + } + + private void checkGlError(String op) { + int error = GLES20.glGetError(); + if (error != GLES20.GL_NO_ERROR) { + String msg = op + ": glError 0x" + Integer.toHexString(error); + Log.e(TAG, msg); + throw new RuntimeException(msg); + } + } +} diff --git a/app/src/main/java/com/facebook/encapp/utils/FrameInfo.java b/app/src/main/java/com/facebook/encapp/utils/FrameInfo.java index 1619e980..e40fbb16 100644 --- a/app/src/main/java/com/facebook/encapp/utils/FrameInfo.java +++ b/app/src/main/java/com/facebook/encapp/utils/FrameInfo.java @@ -61,15 +61,17 @@ public void setFlags(int flags) { } public void start(){ mStartTime = ClockTimes.currentTimeNs(); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - Trace.beginAsyncSection("Process frame", mUUID); - } + // Trace disabled for performance - adds ~1-2ms overhead per frame + // if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + // Trace.beginAsyncSection("Process frame", mUUID); + // } } public void stop(){ - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - Trace.endAsyncSection("Process frame", mUUID); - } + // Trace disabled for performance - adds ~1-2ms overhead per frame + // if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + // Trace.endAsyncSection("Process frame", mUUID); + // } mStopTime = ClockTimes.currentTimeNs(); if (mStopTime < mStartTime) { mStopTime = -1; diff --git a/app/src/main/java/com/facebook/encapp/utils/OutputMultiplier.java b/app/src/main/java/com/facebook/encapp/utils/OutputMultiplier.java index 43fa3749..9d040bda 100644 --- a/app/src/main/java/com/facebook/encapp/utils/OutputMultiplier.java +++ b/app/src/main/java/com/facebook/encapp/utils/OutputMultiplier.java @@ -17,6 +17,64 @@ import java.util.Vector; import java.util.concurrent.ConcurrentLinkedQueue; +/** + * Buffer objects for queuing render work to renderer thread + */ +abstract class RenderBufferObject { + long mTimestampUs; + int mFrameCount; + Statistics mStats; // For encoding measurement + + RenderBufferObject(long timestampUs, int frameCount, Statistics stats) { + mTimestampUs = timestampUs; + mFrameCount = frameCount; + mStats = stats; + } + + long getTimestampUs() { + return mTimestampUs; + } + + int getFrameCount() { + return mFrameCount; + } + + Statistics getStats() { + return mStats; + } +} + +class RenderFrameBuffer extends RenderBufferObject { + MediaCodec mCodec; + int mBufferId; + MediaCodec.BufferInfo mInfo; + + RenderFrameBuffer(MediaCodec codec, int id, MediaCodec.BufferInfo info, int frameCount, Statistics stats) { + super(info.presentationTimeUs, frameCount, stats); + mCodec = codec; + mBufferId = id; + mInfo = info; + } +} + +class RenderBitmapBuffer extends RenderBufferObject { + Bitmap mBitmap; + + RenderBitmapBuffer(Bitmap bitmap, long timestampUs, int frameCount, Statistics stats) { + super(timestampUs, frameCount, stats); + mBitmap = bitmap; + } +} + +class RenderGLPatternBuffer extends RenderBufferObject { + FakeGLRenderer mGLRenderer; + + RenderGLPatternBuffer(FakeGLRenderer glRenderer, long timestampUs, int frameCount, Statistics stats) { + super(timestampUs, frameCount, stats); + mGLRenderer = glRenderer; + } +} + public class OutputMultiplier { final static int WAIT_TIME_SHORT_MS = 3000; // 3 sec private static final String TAG = "encapp.mult"; @@ -30,8 +88,10 @@ public class OutputMultiplier { private EglCore mEglCore; private SurfaceTexture mInputTexture; private FullFrameRect mFullFrameBlit; + private FullFrameRect mBitmapBlit; // Separate blit for 2D textures (bitmaps) private Surface mInputSurface; private int mTextureId; + private int mBitmapTextureId = -1; // Separate 2D texture for bitmap input private FrameswapControl mMasterSurface = null; private String mName = "OutputMultiplier"; private int mWidth = -1; @@ -118,8 +178,16 @@ public long awaitNewImage() { } - public void newBitmapAvailable(Bitmap bitmap, long timestampUsec) { - mRenderer.newBitmapAvailable(bitmap, timestampUsec); + public void newBitmapAvailable(Bitmap bitmap, long timestampUsec, int frameCount, Statistics stats) { + mRenderer.newBitmapAvailable(bitmap, timestampUsec, frameCount, stats); + } + + /** + * Signal that a new GL-rendered frame is available (for fake input). + * This is used when rendering synthetic patterns directly with GL. + */ + public void newGLPatternFrame(FakeGLRenderer glRenderer, long timestampUsec, int frameCount, Statistics stats) { + mRenderer.newGLPatternFrame(glRenderer, timestampUsec, frameCount, stats); } public void newFrameAvailable() { @@ -152,9 +220,9 @@ public void stopAndRelease() { Log.d(TAG, "Done stop and release"); } - public void newFrameAvailableInBuffer(MediaCodec codec, int bufferId, MediaCodec.BufferInfo info) { + public void newFrameAvailableInBuffer(MediaCodec codec, int bufferId, MediaCodec.BufferInfo info, int frameCount, Statistics stats) { if (mRenderer != null) { // it will be null if no surface is connected - mRenderer.newFrameAvailableInBuffer(codec, bufferId, info); + mRenderer.newFrameAvailableInBuffer(codec, bufferId, info, frameCount, stats); } else { try { codec.releaseOutputBuffer(bufferId, false); @@ -176,7 +244,7 @@ private class Renderer extends Thread implements SurfaceTexture.OnFrameAvailable private final Object mVSynchLock = new Object(); private final Object mSizeLock = new Object(); boolean mDone = false; - ConcurrentLinkedQueue mFrameBuffers = new ConcurrentLinkedQueue<>(); + ConcurrentLinkedQueue mFrameBuffers = new ConcurrentLinkedQueue<>(); private long mLatestTimestampNsec = 0; private long mTimestamp0Ns = -1; private long mCurrentVsyncNs = 0; @@ -207,10 +275,16 @@ public void run() { mSurfaceObject = null; // we do not need it anymore mOutputSurfaces.add(mMasterSurface); mMasterSurface.makeCurrent(); + + // Create shader program for camera input (external OES texture) mFullFrameBlit = new FullFrameRect( new Texture2dProgram(mProgramType)); mTextureId = mFullFrameBlit.createTextureObject(); mInputTexture = new SurfaceTexture(mTextureId); + + // Create shader program for bitmap input (2D texture) + mBitmapBlit = new FullFrameRect( + new Texture2dProgram(Texture2dProgram.ProgramType.TEXTURE_2D)); // We need to know how big the texture should be synchronized (mSizeLock) { @@ -323,7 +397,7 @@ public void drawBufferSwap() { } try { synchronized (mVSynchLock) { - BufferObject buffer = mFrameBuffers.poll(); + RenderBufferObject buffer = mFrameBuffers.poll(); if (buffer == null) { return; } @@ -340,7 +414,7 @@ public void drawBufferSwap() { // Drop frame if we have frame in the buffert and we are more than one frame late if((diff - (mCurrentVsyncNs - mVsync0) < -2L * LATE_LIMIT_NS) && mFrameBuffers.size() > 0) { - FrameBuffer fb = (FrameBuffer)buffer; + RenderFrameBuffer fb = (RenderFrameBuffer)buffer; Log.d(TAG, "Drop late frame " + (diff - (mCurrentVsyncNs - mVsync0)/1000000) + " ms "); fb.mCodec.releaseOutputBuffer(fb.mBufferId, false); synchronized (mFrameDrawnLock) { @@ -358,43 +432,84 @@ public void drawBufferSwap() { } } } + + boolean isGLPattern = false; try { mLatestTimestampNsec = timeNs; - if (buffer instanceof FrameBuffer) { - // Draw texture - FrameBuffer fb = (FrameBuffer)buffer; + if (buffer instanceof RenderFrameBuffer) { + // Draw texture from MediaCodec + RenderFrameBuffer fb = (RenderFrameBuffer)buffer; fb.mCodec.releaseOutputBuffer(fb.mBufferId, true); mMasterSurface.makeCurrent(); mInputTexture.updateTexImage(); mInputTexture.getTransformMatrix(mTmpMatrix); + } else if (buffer instanceof RenderGLPatternBuffer) { + // Render GL pattern directly - FAST PATH! + // This renders directly to surfaces, no texture blit needed + RenderGLPatternBuffer glBuffer = (RenderGLPatternBuffer)buffer; + renderGLPattern(glBuffer.mGLRenderer, glBuffer.mTimestampUs); + isGLPattern = true; } else { // Draw bitmap - drawBitmap(((BitmapBuffer)buffer).mBitmap); + drawBitmap(((RenderBitmapBuffer)buffer).mBitmap); } } catch (IllegalStateException ise) { // not important } - } - mMasterSurface.setPresentationTime(mLatestTimestampNsec); - - synchronized (mLock) { - for (FrameswapControl surface : mOutputSurfaces) { - if (surface.keepFrame()) { - surface.makeCurrent(); - int width = surface.getWidth(); - int height = surface.getHeight(); - GLES20.glViewport(0, 0, width, height); - mFullFrameBlit.drawFrame(mTextureId, mTmpMatrix); - surface.setPresentationTime(mLatestTimestampNsec); - surface.swapBuffers(); + + // For GL patterns, we've already rendered directly to surfaces + // For texture/bitmap, we need to blit to all surfaces + if (!isGLPattern) { + mMasterSurface.setPresentationTime(mLatestTimestampNsec); + + synchronized (mLock) { + // Use the appropriate blitter and texture based on input type + FullFrameRect blitter = (mBitmapTextureId != -1) ? mBitmapBlit : mFullFrameBlit; + int textureToUse = (mBitmapTextureId != -1) ? mBitmapTextureId : mTextureId; + + for (FrameswapControl surface : mOutputSurfaces) { + if (surface.keepFrame()) { + surface.makeCurrent(); + int width = surface.getWidth(); + int height = surface.getHeight(); + GLES20.glViewport(0, 0, width, height); + blitter.drawFrame(textureToUse, mTmpMatrix); + surface.setPresentationTime(mLatestTimestampNsec); + surface.swapBuffers(); + } + } + + // NOW start encoding measurement - frame has been submitted to encoder! + // Called ONCE per frame after all surfaces are swapped, not once per surface. + // This measures only encoder time, not preparation/rendering time. + if (buffer.getStats() != null) { + buffer.getStats().startEncodingFrame(buffer.getTimestampUs(), buffer.getFrameCount()); + } + } + } else { + // GL pattern already rendered, just set timestamps and swap + synchronized (mLock) { + for (FrameswapControl surface : mOutputSurfaces) { + if (surface.keepFrame()) { + surface.setPresentationTime(mLatestTimestampNsec); + surface.swapBuffers(); + } + } + + // NOW start encoding measurement - frame has been submitted to encoder! + // Called ONCE per frame after all surfaces are swapped, not once per surface. + // This measures only encoder time, not GL rendering or queuing time. + if (buffer.getStats() != null) { + buffer.getStats().startEncodingFrame(buffer.getTimestampUs(), buffer.getFrameCount()); } } } - synchronized (mFrameDrawnLock) { - frameAvailable = (frameAvailable > 0) ? frameAvailable - 1 : 0; - mFrameDrawnLock.notifyAll(); + synchronized (mFrameDrawnLock) { + frameAvailable = (frameAvailable > 0) ? frameAvailable - 1 : 0; + mFrameDrawnLock.notifyAll(); + } } } catch (Exception ex) { Log.e(TAG, "Exception: " + ex.getMessage()); @@ -446,19 +561,72 @@ public void drawBitmap(Bitmap bitmap) { Log.d(TAG, "Skipping drawFrame after shutdown"); return; } + if (mMasterSurface == null) { + Log.e(TAG, "Master surface is null, cannot draw bitmap!"); + return; + } mMasterSurface.makeCurrent(); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureId); - GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, - GLES20.GL_LINEAR); - GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, - GLES20.GL_LINEAR); - GlUtil.checkGlError("loadImageTexture"); + + // Create a separate 2D texture for bitmap input (only once) + if (mBitmapTextureId == -1) { + int[] textures = new int[1]; + GLES20.glGenTextures(1, textures, 0); + mBitmapTextureId = textures[0]; + Log.d(TAG, "Created 2D texture for bitmap input: " + mBitmapTextureId); + + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mBitmapTextureId); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); + GlUtil.checkGlError("create 2D texture"); + } else { + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mBitmapTextureId); + } + + // Load bitmap into the 2D texture GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, bitmap, 0); - GlUtil.checkGlError("loadImageTexture"); - mInputTexture.getTransformMatrix(mTmpMatrix); + GlUtil.checkGlError("GLUtils.texImage2D"); + + // Set up transform matrix for bitmap (no transform from SurfaceTexture) + Matrix.setIdentityM(mTmpMatrix, 0); Matrix.rotateM(mTmpMatrix, 0, 180, 1f, 0, 0); } catch (Exception ex) { - Log.e(TAG, "Exception: " + ex.getMessage()); + Log.e(TAG, "Exception in drawBitmap: " + ex.getMessage()); + ex.printStackTrace(); + } + } + + /** + * Render GL pattern directly to all surfaces - FAST PATH! + * No bitmap, no texture upload, just pure GL rendering. + */ + public void renderGLPattern(FakeGLRenderer glRenderer, long timestampUs) { + try { + if (mEglCore == null) { + Log.d(TAG, "Skipping GL render after shutdown"); + return; + } + + // Render pattern to all surfaces + synchronized (mLock) { + for (FrameswapControl surface : mOutputSurfaces) { + if (surface.keepFrame()) { + surface.makeCurrent(); + int width = surface.getWidth(); + int height = surface.getHeight(); + GLES20.glViewport(0, 0, width, height); + + // Render GL pattern directly - ZERO CPU overhead! + glRenderer.renderFrame(timestampUs); + + // No need to set transform matrix - pattern fills viewport + } + } + } + } catch (Exception ex) { + Log.e(TAG, "Exception in renderGLPattern: " + ex.getMessage()); + ex.printStackTrace(); } } @@ -470,9 +638,9 @@ public void onFrameAvailable(SurfaceTexture surfaceTexture) { } } - public void newFrameAvailableInBuffer(MediaCodec codec, int id, MediaCodec.BufferInfo info) { + public void newFrameAvailableInBuffer(MediaCodec codec, int id, MediaCodec.BufferInfo info, int frameCount, Statistics stats) { synchronized (mInputFrameLock) { - mFrameBuffers.offer(new FrameBuffer(codec, id, info)); + mFrameBuffers.offer(new RenderFrameBuffer(codec, id, info, frameCount, stats)); frameAvailable += 1; mInputFrameLock.notifyAll(); } @@ -485,9 +653,9 @@ public void newFrameAvailable() { } } - public void newBitmapAvailable(Bitmap bitmap, long timestampUsec) { + public void newBitmapAvailable(Bitmap bitmap, long timestampUsec, int frameCount, Statistics stats) { synchronized (mInputFrameLock) { - mFrameBuffers.offer(new BitmapBuffer(bitmap.copy(bitmap.getConfig(), true), timestampUsec)); + mFrameBuffers.offer(new RenderBitmapBuffer(bitmap.copy(bitmap.getConfig(), true), timestampUsec, frameCount, stats)); frameAvailable += 1; mInputFrameLock.notifyAll(); } @@ -501,6 +669,26 @@ public void newBitmapAvailable(Bitmap bitmap, long timestampUsec) { } } } + + /** + * Render a GL pattern frame - queues the work to be done on GL thread. + * This is the fast path for fake input - no bitmap overhead! + * OPTIMIZED: No blocking wait - just queue and return immediately. + */ + public void newGLPatternFrame(FakeGLRenderer glRenderer, long timestampUsec, int frameCount, Statistics stats) { + synchronized (mInputFrameLock) { + // Queue a GL render request (will be processed on GL thread in main loop) + mFrameBuffers.offer(new RenderGLPatternBuffer(glRenderer, timestampUsec, frameCount, stats)); + frameAvailable += 1; + mInputFrameLock.notifyAll(); + } + + // REMOVED BLOCKING WAIT - GL rendering is async, no need to wait for frame drawn + // The bitmap path needs to wait because it copies memory, but GL just queues work + + // NOTE: startEncodingFrame will be called AFTER swapBuffers() in drawBufferSwap() + // to measure only the encoding time, not the GL rendering + queuing time. + } public void quit() { mDone = true; diff --git a/app/src/main/java/com/facebook/encapp/utils/Statistics.java b/app/src/main/java/com/facebook/encapp/utils/Statistics.java index 8229e8ff..74834625 100644 --- a/app/src/main/java/com/facebook/encapp/utils/Statistics.java +++ b/app/src/main/java/com/facebook/encapp/utils/Statistics.java @@ -221,7 +221,12 @@ private FrameInfo getClosestMatch(long pts) { for (int i = mEncodingFrames.size() - 1; i >= 0; i--) { FrameInfo info = mEncodingFrames.get(i); if (info == null) { - Log.e(TAG, "Failed to lookip object at " + i); + Log.e(TAG, "Failed to lookup object at " + i); + continue; + } + // Skip frames that have already been stopped + if (info.getStopTime() != 0) { + continue; } long dist = Math.abs(pts - info.getPts()); if (dist <= minDist) { diff --git a/scripts/encapp.py b/scripts/encapp.py index f67e8c3a..4eea0bbd 100755 --- a/scripts/encapp.py +++ b/scripts/encapp.py @@ -294,7 +294,9 @@ def collect_results( if test.common and test.common.output_filename: filename = test.common.output_filename output_files += re.findall(f"{filename}.*", stdout, re.MULTILINE) - + for subtest in test.parallel.test: + filename = subtest.common.output_filename + output_files += re.findall(f"{filename}.*", stdout, re.MULTILINE) output_files += re.findall( encapp_tool.adb_cmds.ENCAPP_OUTPUT_FILE_NAME_RE, stdout, re.MULTILINE ) diff --git a/scripts/encapp_plot_stats_csv.py b/scripts/encapp_plot_stats_csv.py index 6ce858c0..67411f67 100755 --- a/scripts/encapp_plot_stats_csv.py +++ b/scripts/encapp_plot_stats_csv.py @@ -200,10 +200,33 @@ def plotLatency(data, options): # drop na from the column that will be plotted data = data.dropna(subset=[proctime]) print(f"{data['proctime'][:3]=}, {data['source'][:3]}") - average_lat_msec = round(data["proctime"].mean() / 1e6, 2) - p50_msec = int(round(data["proctime"].quantile(0.5) / 1e6, 0)) - p95_msec = int(round(data["proctime"].quantile(0.95) / 1e6, 0)) - p99_msec = int(round(data["proctime"].quantile(0.99) / 1e6, 0)) + + # Get the number of unique datasets + unique_datasets = data[hue].unique() + num_datasets = len(unique_datasets) + + # Calculate statistics - either combined or per-dataset + if num_datasets <= 3: + # Show per-dataset statistics + title_lines = [f"{options.label}"] + for item in unique_datasets: + itemdata = data.loc[data[hue] == item] + average_lat_msec = round(itemdata["proctime"].mean() / 1e6, 2) + p50_msec = int(round(itemdata["proctime"].quantile(0.5) / 1e6, 0)) + p95_msec = int(round(itemdata["proctime"].quantile(0.95) / 1e6, 0)) + p99_msec = int(round(itemdata["proctime"].quantile(0.99) / 1e6, 0)) + title_lines.append( + f"{item}: mean: {average_lat_msec}, p50,p95,p99: {p50_msec}, {p95_msec}, {p99_msec}" + ) + title_text = "\n".join(title_lines) + else: + # Show combined statistics for more than 3 datasets + average_lat_msec = round(data["proctime"].mean() / 1e6, 2) + p50_msec = int(round(data["proctime"].quantile(0.5) / 1e6, 0)) + p95_msec = int(round(data["proctime"].quantile(0.95) / 1e6, 0)) + p99_msec = int(round(data["proctime"].quantile(0.99) / 1e6, 0)) + title_text = f"{options.label}\nLatency, mean: {average_lat_msec} msec, p50,p95,p99: {p50_msec}, {p95_msec}, {p99_msec}" + p = sns.lineplot( # noqa: F841 x=data["rel_start_quant"] / 1e3, y=data[proctime] / 1e6, @@ -213,9 +236,34 @@ def plotLatency(data, options): ) axs = p.axes # p.set_ylim(0, 90) - axs.set_title( - f"{options.label}\nLatency, mean: {average_lat_msec} msec, p50,p95,p99: {p50_msec}, {p95_msec}, {p99_msec}" - ) + axs.set_title(title_text) + + # Add horizontal lines for max latency per dataset + colors = plt.rcParams["axes.prop_cycle"].by_key()["color"] + for idx, item in enumerate(unique_datasets): + itemdata = data.loc[data[hue] == item] + max_lat_msec = round(itemdata["proctime"].max() / 1e6, 1) + color = colors[idx % len(colors)] + + # Draw horizontal line + axs.axhline(y=max_lat_msec, color=color, linestyle="--", linewidth=1, alpha=0.7) + + # Add text label inside the graph area + xlim = axs.get_xlim() + x_pos = xlim[0] + (xlim[1] - xlim[0]) * 0.02 + axs.text( + x_pos, + max_lat_msec, + f" {max_lat_msec}", + color=color, + va="bottom", + ha="left", + fontsize=8, + bbox=dict( + boxstyle="round,pad=0.3", facecolor="white", alpha=0.7, edgecolor=color + ), + ) + axs.legend(loc="best", fancybox=True, framealpha=0.5) axs.get_yaxis().set_minor_locator(mlp.ticker.AutoMinorLocator()) axs.set(xlabel="Time (sec)", ylabel="Latency (msec)") @@ -231,21 +279,45 @@ def plotLatency(data, options): for item in data[hue].unique(): itemdata = data.loc[data[hue] == item] average_lat_msec = round(itemdata["proctime"].mean() / 1e6, 2) - p50_msec = round(itemdata["proctime"].quantile(0.5) / 1e6, 1) - p95_msec = round(itemdata["proctime"].quantile(0.95) / 1e6, 1) - p99_msec = round(itemdata["proctime"].quantile(0.99) / 1e6, 1) + p50_msec = round(itemdata["proctime"].quantile(0.5) / 1e6, 2) + p95_msec = round(itemdata["proctime"].quantile(0.95) / 1e6, 2) + p99_msec = round(itemdata["proctime"].quantile(0.99) / 1e6, 2) tmp.append([item, average_lat_msec, p50_msec, p95_msec, p99_msec]) meandata = pd.DataFrame(tmp, columns=[hue, "average", "p50", "p90", "p99"]) meandata.sort_values(["p50"], inplace=True) meandata["index"] = np.arange(1, len(meandata) + 1) meanmelt = pd.melt(meandata, ["index", hue]) - p = sns.lineplot(x="variable", y="value", hue=hue, data=meanmelt) + p = sns.lineplot(x="variable", y="value", hue=hue, data=meanmelt, marker="o") ymax = meanmelt["value"].max() xmax = meanmelt["index"].max() for num in meanmelt["index"].unique(): item = meanmelt.loc[meanmelt["index"] == num].iloc[0][hue] text += f"{num}: {item}\n" axs = p.axes + + # Add value labels at each point in the graph + stat_types = ["average", "p50", "p90", "p99"] + for idx, dataset_name in enumerate(meandata[hue]): + row = meandata[meandata[hue] == dataset_name].iloc[0] + for stat_idx, stat_type in enumerate(stat_types): + value = row[stat_type] + # Add text annotation slightly above each point + axs.text( + stat_idx, + value, + f"{value:.2f}", + ha="center", + va="bottom", + fontsize=8, + bbox=dict( + boxstyle="round,pad=0.2", + facecolor="white", + alpha=0.7, + edgecolor="gray", + linewidth=0.5, + ), + ) + axs.set_title("Averaged values") axs.set(ylabel="Latency (msec)") @@ -529,7 +601,7 @@ def plot_named_timestamps(data, enc_dec_data, options): try: print(f"{enc_dec_data.loc[enc_dec_data['frame'] == 0]}") print( - f"frame 0: {enc_dec_data.loc[(enc_dec_data['source'] == source) ]['proctime']}" + f"frame 0: {enc_dec_data.loc[(enc_dec_data['source'] == source)]['proctime']}" ) proctime = enc_dec_data.loc[(enc_dec_data["source"] == source)][ "proctime" @@ -576,6 +648,16 @@ def plot_named_timestamps(data, enc_dec_data, options): plt.xticks(rotation=70) plt.suptitle(f"{options.label} - Timestamp times in ms") axs = p.axes + + # Add value labels on top of each bar + for container in axs.containers: + # Add labels with rounded integer values (no decimals) + labels = [ + f"{int(round(v.get_height()))}" if v.get_height() > 0 else "" + for v in container + ] + axs.bar_label(container, labels=labels, fontsize=8, padding=3) + axs.legend(loc="best", fancybox=True, framealpha=0.5) axs.get_yaxis().set_minor_locator(mlp.ticker.AutoMinorLocator()) axs.grid(visible=True, which="both")