| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 |
| 11 | package org.webrtc; |
| 12 |
| 13 | import android.graphics.Bitmap; |
| 14 | import android.graphics.Matrix; |
| 15 | import android.graphics.SurfaceTexture; |
| 16 | import android.opengl.GLES20; |
| 17 | import android.os.Handler; |
| 18 | import android.os.HandlerThread; |
| 19 | import android.os.Looper; |
| 20 | import android.os.Message; |
| 21 | import androidx.annotation.Nullable; |
| 22 | import android.view.Surface; |
| 23 |
| 24 | import org.telegram.messenger.FileLog; |
| 25 |
| 26 | import java.nio.ByteBuffer; |
| 27 | import java.util.ArrayList; |
| 28 | import java.util.Iterator; |
| 29 | import java.util.concurrent.CountDownLatch; |
| 30 | import java.util.concurrent.TimeUnit; |
| 31 |
| 32 | /** |
| 33 | * Implements VideoSink by displaying the video stream on an EGL Surface. This class is intended to |
| 34 | * be used as a helper class for rendering on SurfaceViews and TextureViews. |
| 35 | */ |
| 36 | public class EglRenderer implements VideoSink { |
| 37 | private static final String TAG = "EglRenderer"; |
| 38 | private static final long LOG_INTERVAL_SEC = 4; |
| 39 |
| 40 | public boolean firstFrameRendered; |
| 41 |
| 42 | public interface FrameListener { void onFrame(Bitmap frame); } |
| 43 |
| 44 | /** Callback for clients to be notified about errors encountered during rendering. */ |
| 45 | public interface ErrorCallback { |
| 46 | /** Called if GLES20.GL_OUT_OF_MEMORY is encountered during rendering. */ |
| 47 | void onGlOutOfMemory(); |
| 48 | } |
| 49 |
| 50 | private static class FrameListenerAndParams { |
| 51 | public final FrameListener listener; |
| 52 | public final float scale; |
| 53 | public final RendererCommon.GlDrawer drawer; |
| 54 | public final boolean applyFpsReduction; |
| 55 |
| 56 | public FrameListenerAndParams(FrameListener listener, float scale, |
| 57 | RendererCommon.GlDrawer drawer, boolean applyFpsReduction) { |
| 58 | this.listener = listener; |
| 59 | this.scale = scale; |
| 60 | this.drawer = drawer; |
| 61 | this.applyFpsReduction = applyFpsReduction; |
| 62 | } |
| 63 | } |
| 64 |
| 65 | private class EglSurfaceCreation implements Runnable { |
| 66 | private Object surface; |
| 67 | private final boolean background; |
| 68 |
| 69 | public EglSurfaceCreation(boolean background) { |
| 70 | this.background = background; |
| 71 | } |
| 72 |
| 73 | // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. |
| 74 | @SuppressWarnings("NoSynchronizedMethodCheck") |
| 75 | public synchronized void setSurface(Object surface) { |
| 76 | this.surface = surface; |
| 77 | } |
| 78 |
| 79 | @Override |
| 80 | // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression. |
| 81 | @SuppressWarnings("NoSynchronizedMethodCheck") |
| 82 | public synchronized void run() { |
| 83 | if (surface != null && eglBase != null && (background ? !eglBase.hasBackgroundSurface() : !eglBase.hasSurface())) { |
| 84 | if (surface instanceof Surface) { |
| 85 | eglBase.createSurface((Surface) surface); |
| 86 | } else if (surface instanceof SurfaceTexture) { |
| 87 | if (background) { |
| 88 | eglBase.createBackgroundSurface((SurfaceTexture) surface); |
| 89 | } else { |
| 90 | eglBase.createSurface((SurfaceTexture) surface); |
| 91 | } |
| 92 |
| 93 | } else { |
| 94 | throw new IllegalStateException("Invalid surface: " + surface); |
| 95 | } |
| 96 | if (!background) { |
| 97 | eglBase.makeCurrent(); |
| 98 | // Necessary for YUV frames with odd width. |
| 99 | GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); |
| 100 | } else { |
| 101 | eglBase.makeBackgroundCurrent(); |
| 102 | // Necessary for YUV frames with odd width. |
| 103 | GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); |
| 104 | if (eglBase.hasSurface()) { |
| 105 | eglBase.makeCurrent(); |
| 106 | } |
| 107 | } |
| 108 | } |
| 109 | } |
| 110 | } |
| 111 |
| 112 | /** |
| 113 | * Handler that triggers a callback when an uncaught exception happens when handling a message. |
| 114 | */ |
| 115 | private static class HandlerWithExceptionCallback extends Handler { |
| 116 | private final Runnable exceptionCallback; |
| 117 |
| 118 | public HandlerWithExceptionCallback(Looper looper, Runnable exceptionCallback) { |
| 119 | super(looper); |
| 120 | this.exceptionCallback = exceptionCallback; |
| 121 | } |
| 122 |
| 123 | @Override |
| 124 | public void dispatchMessage(Message msg) { |
| 125 | try { |
| 126 | super.dispatchMessage(msg); |
| 127 | } catch (Exception e) { |
| 128 | Logging.e(TAG, "Exception on EglRenderer thread", e); |
| 129 | exceptionCallback.run(); |
| 130 | throw e; |
| 131 | } |
| 132 | } |
| 133 | } |
| 134 |
| 135 | protected final String name; |
| 136 |
| 137 | // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized |
| 138 | // on |handlerLock|. |
| 139 | private final Object handlerLock = new Object(); |
| 140 | @Nullable private Handler renderThreadHandler; |
| 141 |
| 142 | private final ArrayList<FrameListenerAndParams> frameListeners = new ArrayList<>(); |
| 143 |
| 144 | private volatile ErrorCallback errorCallback; |
| 145 |
| 146 | // Variables for fps reduction. |
| 147 | private final Object fpsReductionLock = new Object(); |
| 148 | // Time for when next frame should be rendered. |
| 149 | private long nextFrameTimeNs; |
| 150 | // Minimum duration between frames when fps reduction is active, or -1 if video is completely |
| 151 | // paused. |
| 152 | private long minRenderPeriodNs; |
| 153 |
| 154 | // EGL and GL resources for drawing YUV/OES textures. After initialization, these are only |
| 155 | // accessed from the render thread. |
| 156 | @Nullable private EglBase eglBase; |
| 157 | private final VideoFrameDrawer frameDrawer; |
| 158 | @Nullable private RendererCommon.GlDrawer drawer; |
| 159 | private boolean usePresentationTimeStamp; |
| 160 | private final Matrix drawMatrix = new Matrix(); |
| 161 |
| 162 | // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|. |
| 163 | private final Object frameLock = new Object(); |
| 164 | @Nullable private VideoFrame pendingFrame; |
| 165 |
| 166 | // These variables are synchronized on |layoutLock|. |
| 167 | private final Object layoutLock = new Object(); |
| 168 | private float layoutAspectRatio; |
| 169 | // If true, mirrors the video stream horizontally. |
| 170 | private boolean mirrorHorizontally; |
| 171 | // If true, mirrors the video stream vertically. |
| 172 | private boolean mirrorVertically; |
| 173 |
| 174 | private int rotation; |
| 175 |
| 176 | // Total number of video frames received in renderFrame() call. |
| 177 | private int framesReceived; |
| 178 | // Number of video frames dropped by renderFrame() because previous frame has not been rendered |
| 179 | // yet. |
| 180 | private int framesDropped; |
| 181 | // Number of rendered video frames. |
| 182 | private int framesRendered; |
| 183 | // Time in ns spent in renderFrameOnRenderThread() function. |
| 184 | private long renderTimeNs; |
| 185 | // Time in ns spent by the render thread in the swapBuffers() function. |
| 186 | private long renderSwapBufferTimeNs; |
| 187 |
| 188 | // Used for bitmap capturing. |
| 189 | private final GlTextureFrameBuffer bitmapTextureFramebuffer = |
| 190 | new GlTextureFrameBuffer(GLES20.GL_RGBA); |
| 191 |
| 192 | private final EglSurfaceCreation eglSurfaceCreationRunnable = new EglSurfaceCreation(false); |
| 193 | private final EglSurfaceCreation eglSurfaceBackgroundCreationRunnable = new EglSurfaceCreation(true); |
| 194 |
| 195 | /** |
| 196 | * Standard constructor. The name will be used for the render thread name and included when |
| 197 | * logging. In order to render something, you must first call init() and createEglSurface. |
| 198 | */ |
| 199 | public EglRenderer(String name) { |
| 200 | this(name, new VideoFrameDrawer()); |
| 201 | } |
| 202 |
| 203 | public EglRenderer(String name, VideoFrameDrawer videoFrameDrawer) { |
| 204 | this.name = name; |
| 205 | this.frameDrawer = videoFrameDrawer; |
| 206 | } |
| 207 |
| 208 | /** |
| 209 | * Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used |
| 210 | * for drawing frames on the EGLSurface. This class is responsible for calling release() on |
| 211 | * |drawer|. It is allowed to call init() to reinitialize the renderer after a previous |
| 212 | * init()/release() cycle. If usePresentationTimeStamp is true, eglPresentationTimeANDROID will be |
| 213 | * set with the frame timestamps, which specifies desired presentation time and might be useful |
| 214 | * for e.g. syncing audio and video. |
| 215 | */ |
| 216 | public void init(@Nullable final EglBase.Context sharedContext, final int[] configAttributes, |
| 217 | RendererCommon.GlDrawer drawer, boolean usePresentationTimeStamp) { |
| 218 | synchronized (handlerLock) { |
| 219 | if (renderThreadHandler != null) { |
| 220 | throw new IllegalStateException(name + "Already initialized"); |
| 221 | } |
| 222 | logD("Initializing EglRenderer"); |
| 223 | this.drawer = drawer; |
| 224 | this.usePresentationTimeStamp = usePresentationTimeStamp; |
| 225 | firstFrameRendered = false; |
| 226 |
| 227 | final HandlerThread renderThread = new HandlerThread(name + "EglRenderer"); |
| 228 | renderThread.start(); |
| 229 | renderThreadHandler = |
| 230 | new HandlerWithExceptionCallback(renderThread.getLooper(), new Runnable() { |
| 231 | @Override |
| 232 | public void run() { |
| 233 | synchronized (handlerLock) { |
| 234 | renderThreadHandler = null; |
| 235 | } |
| 236 | } |
| 237 | }); |
| 238 | // Create EGL context on the newly created render thread. It should be possibly to create the |
| 239 | // context on this thread and make it current on the render thread, but this causes failure on |
| 240 | // some Marvel based JB devices. https://bugs.chromium.org/p/webrtc/issues/detail?id=6350. |
| 241 | renderThreadHandler.post(() -> { |
| 242 | // If sharedContext is null, then texture frames are disabled. This is typically for old |
| 243 | // devices that might not be fully spec compliant, so force EGL 1.0 since EGL 1.4 has |
| 244 | // caused trouble on some weird devices. |
| 245 | if (sharedContext == null) { |
| 246 | logD("EglBase10.create context"); |
| 247 | eglBase = EglBase.createEgl10(configAttributes); |
| 248 | } else { |
| 249 | logD("EglBase.create shared context"); |
| 250 | eglBase = EglBase.create(sharedContext, configAttributes); |
| 251 | } |
| 252 | }); |
| 253 | renderThreadHandler.post(eglSurfaceCreationRunnable); |
| 254 | } |
| 255 | } |
| 256 |
| 257 | /** |
| 258 | * Same as above with usePresentationTimeStamp set to false. |
| 259 | * |
| 260 | * @see #init(EglBase.Context, int[], RendererCommon.GlDrawer, boolean) |
| 261 | */ |
| 262 | public void init(@Nullable final EglBase.Context sharedContext, final int[] configAttributes, |
| 263 | RendererCommon.GlDrawer drawer) { |
| 264 | init(sharedContext, configAttributes, drawer, /* usePresentationTimeStamp= */ false); |
| 265 | } |
| 266 |
| 267 | public void createEglSurface(Surface surface) { |
| 268 | createEglSurfaceInternal(surface, false); |
| 269 | } |
| 270 |
| 271 | public void createEglSurface(SurfaceTexture surfaceTexture) { |
| 272 | createEglSurfaceInternal(surfaceTexture, false); |
| 273 | } |
| 274 |
| 275 | public void createBackgroundSurface(SurfaceTexture surface) { |
| 276 | createEglSurfaceInternal(surface, true); |
| 277 | } |
| 278 |
| 279 | private void createEglSurfaceInternal(Object surface, boolean background) { |
| 280 | if (background) { |
| 281 | eglSurfaceBackgroundCreationRunnable.setSurface(surface); |
| 282 | synchronized (handlerLock) { |
| 283 | if (renderThreadHandler != null) { |
| 284 | renderThreadHandler.post(eglSurfaceBackgroundCreationRunnable); |
| 285 | } else { |
| 286 | FileLog.d("can't create background surface. render thread is null"); |
| 287 | } |
| 288 | } |
| 289 | // postToRenderThread(eglSurfaceBackgroundCreationRunnable); |
| 290 | } else { |
| 291 | eglSurfaceCreationRunnable.setSurface(surface); |
| 292 | postToRenderThread(eglSurfaceCreationRunnable); |
| 293 | } |
| 294 | } |
| 295 |
| 296 | /** |
| 297 | * Block until any pending frame is returned and all GL resources released, even if an interrupt |
| 298 | * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function |
| 299 | * should be called before the Activity is destroyed and the EGLContext is still valid. If you |
| 300 | * don't call this function, the GL resources might leak. |
| 301 | */ |
| 302 | public void release() { |
| 303 | logD("Releasing."); |
| 304 | final CountDownLatch eglCleanupBarrier = new CountDownLatch(1); |
| 305 | synchronized (handlerLock) { |
| 306 | if (renderThreadHandler == null) { |
| 307 | logD("Already released"); |
| 308 | return; |
| 309 | } |
| 310 | // Release EGL and GL resources on render thread. |
| 311 | renderThreadHandler.postAtFrontOfQueue(() -> { |
| 312 | // Detach current shader program. |
| 313 | synchronized (EglBase.lock) { |
| 314 | GLES20.glUseProgram(/* program= */ 0); |
| 315 | } |
| 316 | if (drawer != null) { |
| 317 | drawer.release(); |
| 318 | drawer = null; |
| 319 | } |
| 320 | frameDrawer.release(); |
| 321 | bitmapTextureFramebuffer.release(); |
| 322 | if (eglBase != null) { |
| 323 | logD("eglBase detach and release."); |
| 324 | eglBase.detachCurrent(); |
| 325 | eglBase.release(); |
| 326 | eglBase = null; |
| 327 | } |
| 328 | frameListeners.clear(); |
| 329 | eglCleanupBarrier.countDown(); |
| 330 | }); |
| 331 | final Looper renderLooper = renderThreadHandler.getLooper(); |
| 332 | // TODO(magjed): Replace this post() with renderLooper.quitSafely() when API support >= 18. |
| 333 | renderThreadHandler.post(() -> { |
| 334 | logD("Quitting render thread."); |
| 335 | renderLooper.quit(); |
| 336 | }); |
| 337 | // Don't accept any more frames or messages to the render thread. |
| 338 | renderThreadHandler = null; |
| 339 | } |
| 340 | // Make sure the EGL/GL cleanup posted above is executed. |
| 341 | ThreadUtils.awaitUninterruptibly(eglCleanupBarrier); |
| 342 | synchronized (frameLock) { |
| 343 | if (pendingFrame != null) { |
| 344 | pendingFrame.release(); |
| 345 | pendingFrame = null; |
| 346 | } |
| 347 | } |
| 348 | logD("Releasing done."); |
| 349 | } |
| 350 |
| 351 | public void printStackTrace() { |
| 352 | synchronized (handlerLock) { |
| 353 | final Thread renderThread = |
| 354 | (renderThreadHandler == null) ? null : renderThreadHandler.getLooper().getThread(); |
| 355 | if (renderThread != null) { |
| 356 | final StackTraceElement[] renderStackTrace = renderThread.getStackTrace(); |
| 357 | if (renderStackTrace.length > 0) { |
| 358 | logW("EglRenderer stack trace:"); |
| 359 | for (StackTraceElement traceElem : renderStackTrace) { |
| 360 | logW(traceElem.toString()); |
| 361 | } |
| 362 | } |
| 363 | } |
| 364 | } |
| 365 | } |
| 366 |
| 367 | /** |
| 368 | * Set if the video stream should be mirrored horizontally or not. |
| 369 | */ |
| 370 | public void setMirror(final boolean mirror) { |
| 371 | logD("setMirrorHorizontally: " + mirror); |
| 372 | synchronized (layoutLock) { |
| 373 | this.mirrorHorizontally = mirror; |
| 374 | } |
| 375 | } |
| 376 |
| 377 | /** |
| 378 | * Set if the video stream should be mirrored vertically or not. |
| 379 | */ |
| 380 | public void setMirrorVertically(final boolean mirrorVertically) { |
| 381 | logD("setMirrorVertically: " + mirrorVertically); |
| 382 | synchronized (layoutLock) { |
| 383 | this.mirrorVertically = mirrorVertically; |
| 384 | } |
| 385 | } |
| 386 |
| 387 | /** |
| 388 | * Set layout aspect ratio. This is used to crop frames when rendering to avoid stretched video. |
| 389 | * Set this to 0 to disable cropping. |
| 390 | */ |
| 391 | public void setLayoutAspectRatio(float layoutAspectRatio) { |
| 392 | if (this.layoutAspectRatio != layoutAspectRatio) { |
| 393 | synchronized (layoutLock) { |
| 394 | this.layoutAspectRatio = layoutAspectRatio; |
| 395 | } |
| 396 | } |
| 397 | } |
| 398 |
| 399 | /** |
| 400 | * Limit render framerate. |
| 401 | * |
| 402 | * @param fps Limit render framerate to this value, or use Float.POSITIVE_INFINITY to disable fps |
| 403 | * reduction. |
| 404 | */ |
| 405 | public void setFpsReduction(float fps) { |
| 406 | logD("setFpsReduction: " + fps); |
| 407 | synchronized (fpsReductionLock) { |
| 408 | final long previousRenderPeriodNs = minRenderPeriodNs; |
| 409 | if (fps <= 0) { |
| 410 | minRenderPeriodNs = Long.MAX_VALUE; |
| 411 | } else { |
| 412 | minRenderPeriodNs = (long) (TimeUnit.SECONDS.toNanos(1) / fps); |
| 413 | } |
| 414 | if (minRenderPeriodNs != previousRenderPeriodNs) { |
| 415 | // Fps reduction changed - reset frame time. |
| 416 | nextFrameTimeNs = System.nanoTime(); |
| 417 | } |
| 418 | } |
| 419 | } |
| 420 |
| 421 | public void disableFpsReduction() { |
| 422 | setFpsReduction(Float.POSITIVE_INFINITY /* fps */); |
| 423 | } |
| 424 |
| 425 | public void pauseVideo() { |
| 426 | setFpsReduction(0 /* fps */); |
| 427 | } |
| 428 |
| 429 | /** |
| 430 | * Register a callback to be invoked when a new video frame has been received. This version uses |
| 431 | * the drawer of the EglRenderer that was passed in init. |
| 432 | * |
| 433 | * @param listener The callback to be invoked. The callback will be invoked on the render thread. |
| 434 | * It should be lightweight and must not call removeFrameListener. |
| 435 | * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is |
| 436 | * required. |
| 437 | */ |
| 438 | public void addFrameListener(final FrameListener listener, final float scale) { |
| 439 | addFrameListener(listener, scale, null, false /* applyFpsReduction */); |
| 440 | } |
| 441 |
| 442 | /** |
| 443 | * Register a callback to be invoked when a new video frame has been received. |
| 444 | * |
| 445 | * @param listener The callback to be invoked. The callback will be invoked on the render thread. |
| 446 | * It should be lightweight and must not call removeFrameListener. |
| 447 | * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is |
| 448 | * required. |
| 449 | * @param drawer Custom drawer to use for this frame listener or null to use the default one. |
| 450 | */ |
| 451 | public void addFrameListener( |
| 452 | final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawerParam) { |
| 453 | addFrameListener(listener, scale, drawerParam, false /* applyFpsReduction */); |
| 454 | } |
| 455 |
| 456 | /** |
| 457 | * Register a callback to be invoked when a new video frame has been received. |
| 458 | * |
| 459 | * @param listener The callback to be invoked. The callback will be invoked on the render thread. |
| 460 | * It should be lightweight and must not call removeFrameListener. |
| 461 | * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is |
| 462 | * required. |
| 463 | * @param drawer Custom drawer to use for this frame listener or null to use the default one. |
| 464 | * @param applyFpsReduction This callback will not be called for frames that have been dropped by |
| 465 | * FPS reduction. |
| 466 | */ |
| 467 | public void addFrameListener(final FrameListener listener, final float scale, |
| 468 | @Nullable final RendererCommon.GlDrawer drawerParam, final boolean applyFpsReduction) { |
| 469 | postToRenderThread(() -> { |
| 470 | final RendererCommon.GlDrawer listenerDrawer = drawerParam == null ? drawer : drawerParam; |
| 471 | frameListeners.add( |
| 472 | new FrameListenerAndParams(listener, scale, listenerDrawer, applyFpsReduction)); |
| 473 | }); |
| 474 | } |
| 475 |
| 476 | /** |
| 477 | * Remove any pending callback that was added with addFrameListener. If the callback is not in |
| 478 | * the queue, nothing happens. It is ensured that callback won't be called after this method |
| 479 | * returns. |
| 480 | * |
| 481 | * @param runnable The callback to remove. |
| 482 | */ |
| 483 | public void removeFrameListener(final FrameListener listener) { |
| 484 | final CountDownLatch latch = new CountDownLatch(1); |
| 485 | synchronized (handlerLock) { |
| 486 | if (renderThreadHandler == null) { |
| 487 | return; |
| 488 | } |
| 489 | if (Thread.currentThread() == renderThreadHandler.getLooper().getThread()) { |
| 490 | throw new RuntimeException("removeFrameListener must not be called on the render thread."); |
| 491 | } |
| 492 | postToRenderThread(() -> { |
| 493 | latch.countDown(); |
| 494 | final Iterator<FrameListenerAndParams> iter = frameListeners.iterator(); |
| 495 | while (iter.hasNext()) { |
| 496 | if (iter.next().listener == listener) { |
| 497 | iter.remove(); |
| 498 | } |
| 499 | } |
| 500 | }); |
| 501 | } |
| 502 | ThreadUtils.awaitUninterruptibly(latch); |
| 503 | } |
| 504 |
| 505 | /** Can be set in order to be notified about errors encountered during rendering. */ |
| 506 | public void setErrorCallback(ErrorCallback errorCallback) { |
| 507 | this.errorCallback = errorCallback; |
| 508 | } |
| 509 |
| 510 | // VideoSink interface. |
| 511 | @Override |
| 512 | public void onFrame(VideoFrame frame) { |
| 513 | final boolean dropOldFrame; |
| 514 | synchronized (handlerLock) { |
| 515 | if (renderThreadHandler == null) { |
| 516 | logD("Dropping frame - Not initialized or already released."); |
| 517 | return; |
| 518 | } |
| 519 | synchronized (frameLock) { |
| 520 | dropOldFrame = (pendingFrame != null); |
| 521 | if (dropOldFrame) { |
| 522 | pendingFrame.release(); |
| 523 | } |
| 524 | pendingFrame = frame; |
| 525 | pendingFrame.retain(); |
| 526 | renderThreadHandler.post(this ::renderFrameOnRenderThread); |
| 527 | } |
| 528 | } |
| 529 | } |
| 530 |
| 531 | public void setRotation(int value) { |
| 532 | synchronized (layoutLock) { |
| 533 | rotation = value; |
| 534 | } |
| 535 | } |
| 536 |
| 537 | /** |
| 538 | * Release EGL surface. This function will block until the EGL surface is released. |
| 539 | */ |
| 540 | public void releaseEglSurface(final Runnable completionCallback, boolean background) { |
| 541 | // Ensure that the render thread is no longer touching the Surface before returning from this |
| 542 | // function. |
| 543 | eglSurfaceCreationRunnable.setSurface(null /* surface */); |
| 544 | synchronized (handlerLock) { |
| 545 | if (renderThreadHandler != null) { |
| 546 | renderThreadHandler.removeCallbacks(eglSurfaceCreationRunnable); |
| 547 | renderThreadHandler.postAtFrontOfQueue(() -> { |
| 548 | if (eglBase != null) { |
| 549 | eglBase.detachCurrent(); |
| 550 | eglBase.releaseSurface(background); |
| 551 | } |
| 552 | if (completionCallback != null) { |
| 553 | completionCallback.run(); |
| 554 | } |
| 555 | }); |
| 556 | return; |
| 557 | } |
| 558 | } |
| 559 | if (completionCallback != null) { |
| 560 | completionCallback.run(); |
| 561 | } |
| 562 | } |
| 563 |
| 564 | /** |
| 565 | * Private helper function to post tasks safely. |
| 566 | */ |
| 567 | private void postToRenderThread(Runnable runnable) { |
| 568 | synchronized (handlerLock) { |
| 569 | if (renderThreadHandler != null) { |
| 570 | renderThreadHandler.post(runnable); |
| 571 | } |
| 572 | } |
| 573 | } |
| 574 |
| 575 | private void clearSurfaceOnRenderThread(float r, float g, float b, float a) { |
| 576 | if (eglBase != null && eglBase.hasSurface()) { |
| 577 | logD("clearSurface"); |
| 578 | GLES20.glClearColor(r, g, b, a); |
| 579 | GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); |
| 580 | eglBase.swapBuffers(false); |
| 581 | } |
| 582 | if (eglBase != null && eglBase.hasBackgroundSurface()) { |
| 583 | eglBase.makeBackgroundCurrent(); |
| 584 | logD("clearSurface in background"); |
| 585 | GLES20.glClearColor(r, g, b, a); |
| 586 | GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); |
| 587 | eglBase.swapBuffers(true); |
| 588 | eglBase.makeCurrent(); |
| 589 | } |
| 590 | } |
| 591 |
| 592 | /** |
| 593 | * Post a task to clear the surface to a transparent uniform color. |
| 594 | */ |
| 595 | public void clearImage() { |
| 596 | clearImage(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); |
| 597 | firstFrameRendered = false; |
| 598 | } |
| 599 |
| 600 | /** |
| 601 | * Post a task to clear the surface to a specific color. |
| 602 | */ |
| 603 | public void clearImage(final float r, final float g, final float b, final float a) { |
| 604 | synchronized (handlerLock) { |
| 605 | if (renderThreadHandler == null) { |
| 606 | return; |
| 607 | } |
| 608 | renderThreadHandler.postAtFrontOfQueue(() -> clearSurfaceOnRenderThread(r, g, b, a)); |
| 609 | } |
| 610 | } |
| 611 |
| 612 | public void getTexture(GlGenericDrawer.TextureCallback callback) { |
| 613 | synchronized (handlerLock) { |
| 614 | try { |
| 615 | if (renderThreadHandler != null) { |
| 616 | renderThreadHandler.post(() -> frameDrawer.getRenderBufferBitmap(drawer, rotation, callback)); |
| 617 | } |
| 618 | } catch (Exception e) { |
| 619 | FileLog.e(e); |
| 620 | } |
| 621 | } |
| 622 | } |
| 623 |
| 624 | /** |
| 625 | * Renders and releases |pendingFrame|. |
| 626 | */ |
| 627 | private void renderFrameOnRenderThread() { |
| 628 | // Fetch and render |pendingFrame|. |
| 629 | final VideoFrame frame; |
| 630 | synchronized (frameLock) { |
| 631 | if (pendingFrame == null) { |
| 632 | return; |
| 633 | } |
| 634 | frame = pendingFrame; |
| 635 | pendingFrame = null; |
| 636 | } |
| 637 | if (eglBase == null || !eglBase.hasSurface()) { |
| 638 | logD("Dropping frame - No surface"); |
| 639 | frame.release(); |
| 640 | return; |
| 641 | } |
| 642 | // Check if fps reduction is active. |
| 643 | final boolean shouldRenderFrame; |
| 644 | synchronized (fpsReductionLock) { |
| 645 | if (minRenderPeriodNs == Long.MAX_VALUE) { |
| 646 | // Rendering is paused. |
| 647 | shouldRenderFrame = false; |
| 648 | } else if (minRenderPeriodNs <= 0) { |
| 649 | // FPS reduction is disabled. |
| 650 | shouldRenderFrame = true; |
| 651 | } else { |
| 652 | final long currentTimeNs = System.nanoTime(); |
| 653 | if (currentTimeNs < nextFrameTimeNs) { |
| 654 | shouldRenderFrame = false; |
| 655 | } else { |
| 656 | nextFrameTimeNs += minRenderPeriodNs; |
| 657 | // The time for the next frame should always be in the future. |
| 658 | nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs); |
| 659 | shouldRenderFrame = true; |
| 660 | } |
| 661 | } |
| 662 | } |
| 663 |
| 664 | final long startTimeNs = System.nanoTime(); |
| 665 |
| 666 | boolean rotate = Math.abs(rotation) == 90 || Math.abs(rotation) == 270; |
| 667 | final float frameAspectRatio = (rotate ? frame.getRotatedHeight() : frame.getRotatedWidth()) / (float) (rotate ? frame.getRotatedWidth() : frame.getRotatedHeight()); |
| 668 | final float drawnAspectRatio; |
| 669 | synchronized (layoutLock) { |
| 670 | drawnAspectRatio = layoutAspectRatio != 0f ? layoutAspectRatio : frameAspectRatio; |
| 671 | } |
| 672 |
| 673 | final float scaleX; |
| 674 | final float scaleY; |
| 675 |
| 676 | if (frameAspectRatio > drawnAspectRatio) { |
| 677 | scaleX = drawnAspectRatio / frameAspectRatio; |
| 678 | scaleY = 1f; |
| 679 | } else { |
| 680 | scaleX = 1f; |
| 681 | scaleY = frameAspectRatio / drawnAspectRatio; |
| 682 | } |
| 683 |
| 684 | drawMatrix.reset(); |
| 685 | drawMatrix.preTranslate(0.5f, 0.5f); |
| 686 | drawMatrix.preRotate(rotation); |
| 687 | drawMatrix.preScale(mirrorHorizontally ? -1f : 1f, mirrorVertically ? -1f : 1f); |
| 688 | drawMatrix.preScale(scaleX, scaleY); |
| 689 | drawMatrix.preTranslate(-0.5f, -0.5f); |
| 690 |
| 691 | try { |
| 692 | if (shouldRenderFrame) { |
| 693 | frameDrawer.drawFrame(frame, drawer, drawMatrix, 0 /* viewportX */, 0 /* viewportY */, |
| 694 | eglBase.surfaceWidth(), eglBase.surfaceHeight(), rotate, false); |
| 695 |
| 696 | if (eglBase.hasBackgroundSurface()) { |
| 697 | eglBase.makeBackgroundCurrent(); |
| 698 |
| 699 | frameDrawer.drawFrame(frame, drawer, drawMatrix, 0, 0, |
| 700 | eglBase.surfaceWidth(), eglBase.surfaceHeight(), rotate, true); |
| 701 |
| 702 | if (usePresentationTimeStamp) { |
| 703 | eglBase.swapBuffers(frame.getTimestampNs(), true); |
| 704 | } else { |
| 705 | eglBase.swapBuffers(true); |
| 706 | } |
| 707 | eglBase.makeCurrent(); |
| 708 | } |
| 709 |
| 710 | final long swapBuffersStartTimeNs = System.nanoTime(); |
| 711 | if (usePresentationTimeStamp) { |
| 712 | eglBase.swapBuffers(frame.getTimestampNs(), false); |
| 713 | } else { |
| 714 | eglBase.swapBuffers(false); |
| 715 | } |
| 716 |
| 717 |
| 718 | if (!firstFrameRendered) { |
| 719 | firstFrameRendered = true; |
| 720 | onFirstFrameRendered(); |
| 721 | } |
| 722 | } |
| 723 |
| 724 | notifyCallbacks(frame, shouldRenderFrame); |
| 725 | } catch (GlUtil.GlOutOfMemoryException e) { |
| 726 | logE("Error while drawing frame", e); |
| 727 | final ErrorCallback errorCallback = this.errorCallback; |
| 728 | if (errorCallback != null) { |
| 729 | errorCallback.onGlOutOfMemory(); |
| 730 | } |
| 731 | // Attempt to free up some resources. |
| 732 | drawer.release(); |
| 733 | frameDrawer.release(); |
| 734 | bitmapTextureFramebuffer.release(); |
| 735 | // Continue here on purpose and retry again for next frame. In worst case, this is a continous |
| 736 | // problem and no more frames will be drawn. |
| 737 | } finally { |
| 738 | frame.release(); |
| 739 | } |
| 740 | } |
| 741 |
| 742 | protected void onFirstFrameRendered() { |
| 743 |
| 744 | } |
| 745 |
| 746 | private void notifyCallbacks(VideoFrame frame, boolean wasRendered) { |
| 747 | if (frameListeners.isEmpty()) |
| 748 | return; |
| 749 |
| 750 | drawMatrix.reset(); |
| 751 | drawMatrix.preTranslate(0.5f, 0.5f); |
| 752 | drawMatrix.preRotate(rotation); |
| 753 | drawMatrix.preScale(mirrorHorizontally ? -1f : 1f, mirrorVertically ? -1f : 1f); |
| 754 | drawMatrix.preScale(1f, -1f); // We want the output to be upside down for Bitmap. |
| 755 | drawMatrix.preTranslate(-0.5f, -0.5f); |
| 756 |
| 757 | Iterator<FrameListenerAndParams> it = frameListeners.iterator(); |
| 758 | while (it.hasNext()) { |
| 759 | FrameListenerAndParams listenerAndParams = it.next(); |
| 760 | if (!wasRendered && listenerAndParams.applyFpsReduction) { |
| 761 | continue; |
| 762 | } |
| 763 | it.remove(); |
| 764 |
| 765 | final int scaledWidth = (int) (listenerAndParams.scale * frame.getRotatedWidth()); |
| 766 | final int scaledHeight = (int) (listenerAndParams.scale * frame.getRotatedHeight()); |
| 767 |
| 768 | if (scaledWidth == 0 || scaledHeight == 0) { |
| 769 | listenerAndParams.listener.onFrame(null); |
| 770 | continue; |
| 771 | } |
| 772 |
| 773 | bitmapTextureFramebuffer.setSize(scaledWidth, scaledHeight); |
| 774 |
| 775 | GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapTextureFramebuffer.getFrameBufferId()); |
| 776 | GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, |
| 777 | GLES20.GL_TEXTURE_2D, bitmapTextureFramebuffer.getTextureId(), 0); |
| 778 |
| 779 | GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */); |
| 780 | GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); |
| 781 | frameDrawer.drawFrame(frame, listenerAndParams.drawer, drawMatrix, 0 /* viewportX */, |
| 782 | 0 /* viewportY */, scaledWidth, scaledHeight, false, false); |
| 783 |
| 784 | final ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(scaledWidth * scaledHeight * 4); |
| 785 | GLES20.glViewport(0, 0, scaledWidth, scaledHeight); |
| 786 | GLES20.glReadPixels( |
| 787 | 0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer); |
| 788 |
| 789 | GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); |
| 790 | GlUtil.checkNoGLES2Error("EglRenderer.notifyCallbacks"); |
| 791 |
| 792 | final Bitmap bitmap = Bitmap.createBitmap(scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888); |
| 793 | bitmap.copyPixelsFromBuffer(bitmapBuffer); |
| 794 | listenerAndParams.listener.onFrame(bitmap); |
| 795 | } |
| 796 | } |
| 797 |
| 798 | private void logE(String string, Throwable e) { |
| 799 | Logging.e(TAG, name + string, e); |
| 800 | } |
| 801 |
| 802 | private void logD(String string) { |
| 803 | Logging.d(TAG, name + string); |
| 804 | } |
| 805 |
| 806 | private void logW(String string) { |
| 807 | Logging.w(TAG, name + string); |
| 808 | } |
| 809 | } |
| 810 |