| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 |
| 11 | package org.webrtc; |
| 12 |
| 13 | import android.media.MediaCodec; |
| 14 | import android.media.MediaCodecInfo.CodecCapabilities; |
| 15 | import android.media.MediaFormat; |
| 16 | import android.os.SystemClock; |
| 17 | import android.view.Surface; |
| 18 | import androidx.annotation.Nullable; |
| 19 | import java.io.IOException; |
| 20 | import java.nio.ByteBuffer; |
| 21 | import java.util.concurrent.BlockingDeque; |
| 22 | import java.util.concurrent.LinkedBlockingDeque; |
| 23 | import java.util.concurrent.TimeUnit; |
| 24 |
| 25 | import org.telegram.messenger.FileLog; |
| 26 | import org.webrtc.ThreadUtils.ThreadChecker; |
| 27 |
| 28 | /** |
| 29 | * Android hardware video decoder. |
| 30 | */ |
| 31 | class AndroidVideoDecoder implements VideoDecoder, VideoSink { |
| 32 | private static final String TAG = "AndroidVideoDecoder"; |
| 33 |
| 34 | // MediaCodec.release() occasionally hangs. Release stops waiting and reports failure after |
| 35 | // this timeout. |
| 36 | private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; |
| 37 |
| 38 | // WebRTC queues input frames quickly in the beginning on the call. Wait for input buffers with a |
| 39 | // long timeout (500 ms) to prevent this from causing the codec to return an error. |
| 40 | private static final int DEQUEUE_INPUT_TIMEOUT_US = 500000; |
| 41 |
| 42 | // Dequeuing an output buffer will block until a buffer is available (up to 100 milliseconds). |
| 43 | // If this timeout is exceeded, the output thread will unblock and check if the decoder is still |
| 44 | // running. If it is, it will block on dequeue again. Otherwise, it will stop and release the |
| 45 | // MediaCodec. |
| 46 | private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000; |
| 47 |
| 48 | private final MediaCodecWrapperFactory mediaCodecWrapperFactory; |
| 49 | private final String codecName; |
| 50 | private final VideoCodecMimeType codecType; |
| 51 |
| 52 | private static class FrameInfo { |
| 53 | final long decodeStartTimeMs; |
| 54 | final int rotation; |
| 55 |
| 56 | FrameInfo(long decodeStartTimeMs, int rotation) { |
| 57 | this.decodeStartTimeMs = decodeStartTimeMs; |
| 58 | this.rotation = rotation; |
| 59 | } |
| 60 | } |
| 61 |
| 62 | private final BlockingDeque<FrameInfo> frameInfos; |
| 63 | private int colorFormat; |
| 64 |
| 65 | // Output thread runs a loop which polls MediaCodec for decoded output buffers. It reformats |
| 66 | // those buffers into VideoFrames and delivers them to the callback. Variable is set on decoder |
| 67 | // thread and is immutable while the codec is running. |
| 68 | @Nullable private Thread outputThread; |
| 69 |
| 70 | // Checker that ensures work is run on the output thread. |
| 71 | private ThreadChecker outputThreadChecker; |
| 72 |
| 73 | // Checker that ensures work is run on the decoder thread. The decoder thread is owned by the |
| 74 | // caller and must be used to call initDecode, decode, and release. |
| 75 | private ThreadChecker decoderThreadChecker; |
| 76 |
| 77 | private volatile boolean running; |
| 78 | @Nullable private volatile Exception shutdownException; |
| 79 |
| 80 | // Dimensions (width, height, stride, and sliceHeight) may be accessed by either the decode thread |
| 81 | // or the output thread. Accesses should be protected with this lock. |
| 82 | private final Object dimensionLock = new Object(); |
| 83 | private int width; |
| 84 | private int height; |
| 85 | private int stride; |
| 86 | private int sliceHeight; |
| 87 |
| 88 | // Whether the decoder has finished the first frame. The codec may not change output dimensions |
| 89 | // after delivering the first frame. Only accessed on the output thread while the decoder is |
| 90 | // running. |
| 91 | private boolean hasDecodedFirstFrame; |
| 92 | // Whether the decoder has seen a key frame. The first frame must be a key frame. Only accessed |
| 93 | // on the decoder thread. |
| 94 | private boolean keyFrameRequired; |
| 95 |
| 96 | private final @Nullable EglBase.Context sharedContext; |
| 97 | // Valid and immutable while the decoder is running. |
| 98 | @Nullable private SurfaceTextureHelper surfaceTextureHelper; |
| 99 | @Nullable private Surface surface; |
| 100 |
| 101 | private static class DecodedTextureMetadata { |
| 102 | final long presentationTimestampUs; |
| 103 | final Integer decodeTimeMs; |
| 104 |
| 105 | DecodedTextureMetadata(long presentationTimestampUs, Integer decodeTimeMs) { |
| 106 | this.presentationTimestampUs = presentationTimestampUs; |
| 107 | this.decodeTimeMs = decodeTimeMs; |
| 108 | } |
| 109 | } |
| 110 |
| 111 | // Metadata for the last frame rendered to the texture. |
| 112 | private final Object renderedTextureMetadataLock = new Object(); |
| 113 | @Nullable private DecodedTextureMetadata renderedTextureMetadata; |
| 114 |
| 115 | // Decoding proceeds asynchronously. This callback returns decoded frames to the caller. Valid |
| 116 | // and immutable while the decoder is running. |
| 117 | @Nullable private Callback callback; |
| 118 |
| 119 | // Valid and immutable while the decoder is running. |
| 120 | @Nullable private MediaCodecWrapper codec; |
| 121 |
| 122 | AndroidVideoDecoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName, |
| 123 | VideoCodecMimeType codecType, int colorFormat, @Nullable EglBase.Context sharedContext) { |
| 124 | if (!isSupportedColorFormat(colorFormat)) { |
| 125 | throw new IllegalArgumentException("Unsupported color format: " + colorFormat); |
| 126 | } |
| 127 | Logging.d(TAG, |
| 128 | "ctor name: " + codecName + " type: " + codecType + " color format: " + colorFormat |
| 129 | + " context: " + sharedContext); |
| 130 | this.mediaCodecWrapperFactory = mediaCodecWrapperFactory; |
| 131 | this.codecName = codecName; |
| 132 | this.codecType = codecType; |
| 133 | this.colorFormat = colorFormat; |
| 134 | this.sharedContext = sharedContext; |
| 135 | this.frameInfos = new LinkedBlockingDeque<>(); |
| 136 | } |
| 137 |
| 138 | @Override |
| 139 | public VideoCodecStatus initDecode(Settings settings, Callback callback) { |
| 140 | this.decoderThreadChecker = new ThreadChecker(); |
| 141 |
| 142 | this.callback = callback; |
| 143 | if (sharedContext != null) { |
| 144 | surfaceTextureHelper = createSurfaceTextureHelper(); |
| 145 | surface = new Surface(surfaceTextureHelper.getSurfaceTexture()); |
| 146 | surfaceTextureHelper.startListening(this); |
| 147 | } |
| 148 | return initDecodeInternal(settings.width, settings.height); |
| 149 | } |
| 150 |
| 151 | // Internal variant is used when restarting the codec due to reconfiguration. |
| 152 | private VideoCodecStatus initDecodeInternal(int width, int height) { |
| 153 | decoderThreadChecker.checkIsOnValidThread(); |
| 154 | Logging.d(TAG, |
| 155 | "initDecodeInternal name: " + codecName + " type: " + codecType + " width: " + width |
| 156 | + " height: " + height + " color format: " + colorFormat); |
| 157 | if (outputThread != null) { |
| 158 | Logging.e(TAG, "initDecodeInternal called while the codec is already running"); |
| 159 | return VideoCodecStatus.FALLBACK_SOFTWARE; |
| 160 | } |
| 161 |
| 162 | // Note: it is not necessary to initialize dimensions under the lock, since the output thread |
| 163 | // is not running. |
| 164 | this.width = width; |
| 165 | this.height = height; |
| 166 |
| 167 | stride = width; |
| 168 | sliceHeight = height; |
| 169 | hasDecodedFirstFrame = false; |
| 170 | keyFrameRequired = true; |
| 171 |
| 172 | try { |
| 173 | codec = mediaCodecWrapperFactory.createByCodecName(codecName); |
| 174 | } catch (IOException | IllegalArgumentException | IllegalStateException e) { |
| 175 | Logging.e(TAG, "Cannot create media decoder " + codecName); |
| 176 | return VideoCodecStatus.FALLBACK_SOFTWARE; |
| 177 | } |
| 178 | try { |
| 179 | MediaFormat format = MediaFormat.createVideoFormat(codecType.mimeType(), width, height); |
| 180 | if (sharedContext == null) { |
| 181 | format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat); |
| 182 | } |
| 183 | codec.configure(format, surface, null, 0); |
| 184 | codec.start(); |
| 185 | } catch (IllegalStateException | IllegalArgumentException e) { |
| 186 | Logging.e(TAG, "initDecode failed", e); |
| 187 | release(); |
| 188 | return VideoCodecStatus.FALLBACK_SOFTWARE; |
| 189 | } |
| 190 | running = true; |
| 191 | outputThread = createOutputThread(); |
| 192 | outputThread.start(); |
| 193 |
| 194 | Logging.d(TAG, "initDecodeInternal done"); |
| 195 | return VideoCodecStatus.OK; |
| 196 | } |
| 197 |
| 198 | @Override |
| 199 | public VideoCodecStatus decode(EncodedImage frame, DecodeInfo info) { |
| 200 | decoderThreadChecker.checkIsOnValidThread(); |
| 201 | if (codec == null || callback == null) { |
| 202 | Logging.d(TAG, "decode uninitalized, codec: " + (codec != null) + ", callback: " + callback); |
| 203 | return VideoCodecStatus.UNINITIALIZED; |
| 204 | } |
| 205 |
| 206 | if (frame.buffer == null) { |
| 207 | Logging.e(TAG, "decode() - no input data"); |
| 208 | return VideoCodecStatus.ERR_PARAMETER; |
| 209 | } |
| 210 |
| 211 | int size = frame.buffer.remaining(); |
| 212 | if (size == 0) { |
| 213 | Logging.e(TAG, "decode() - input buffer empty"); |
| 214 | return VideoCodecStatus.ERR_PARAMETER; |
| 215 | } |
| 216 |
| 217 | // Load dimensions from shared memory under the dimension lock. |
| 218 | final int width; |
| 219 | final int height; |
| 220 | synchronized (dimensionLock) { |
| 221 | width = this.width; |
| 222 | height = this.height; |
| 223 | } |
| 224 |
| 225 | // Check if the resolution changed and reset the codec if necessary. |
| 226 | if (frame.encodedWidth * frame.encodedHeight > 0 |
| 227 | && (frame.encodedWidth != width || frame.encodedHeight != height)) { |
| 228 | VideoCodecStatus status = reinitDecode(frame.encodedWidth, frame.encodedHeight); |
| 229 | if (status != VideoCodecStatus.OK) { |
| 230 | return status; |
| 231 | } |
| 232 | } |
| 233 |
| 234 | if (keyFrameRequired) { |
| 235 | // Need to process a key frame first. |
| 236 | if (frame.frameType != EncodedImage.FrameType.VideoFrameKey) { |
| 237 | Logging.e(TAG, "decode() - key frame required first"); |
| 238 | return VideoCodecStatus.NO_OUTPUT; |
| 239 | } |
| 240 | } |
| 241 |
| 242 | int index; |
| 243 | try { |
| 244 | index = codec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT_US); |
| 245 | } catch (IllegalStateException e) { |
| 246 | Logging.e(TAG, "dequeueInputBuffer failed", e); |
| 247 | return VideoCodecStatus.ERROR; |
| 248 | } |
| 249 | if (index < 0) { |
| 250 | // Decoder is falling behind. No input buffers available. |
| 251 | // The decoder can't simply drop frames; it might lose a key frame. |
| 252 | Logging.e(TAG, "decode() - no HW buffers available; decoder falling behind"); |
| 253 | return VideoCodecStatus.ERROR; |
| 254 | } |
| 255 |
| 256 | ByteBuffer buffer; |
| 257 | try { |
| 258 | buffer = codec.getInputBuffer(index); |
| 259 | } catch (IllegalStateException e) { |
| 260 | Logging.e(TAG, "getInputBuffer with index=" + index + " failed", e); |
| 261 | return VideoCodecStatus.ERROR; |
| 262 | } |
| 263 |
| 264 | if (buffer.capacity() < size) { |
| 265 | Logging.e(TAG, "decode() - HW buffer too small"); |
| 266 | return VideoCodecStatus.ERROR; |
| 267 | } |
| 268 | buffer.put(frame.buffer); |
| 269 |
| 270 | frameInfos.offer(new FrameInfo(SystemClock.elapsedRealtime(), frame.rotation)); |
| 271 | try { |
| 272 | codec.queueInputBuffer(index, 0 /* offset */, size, |
| 273 | TimeUnit.NANOSECONDS.toMicros(frame.captureTimeNs), 0 /* flags */); |
| 274 | } catch (IllegalStateException e) { |
| 275 | Logging.e(TAG, "queueInputBuffer failed", e); |
| 276 | frameInfos.pollLast(); |
| 277 | return VideoCodecStatus.ERROR; |
| 278 | } |
| 279 | if (keyFrameRequired) { |
| 280 | keyFrameRequired = false; |
| 281 | } |
| 282 | return VideoCodecStatus.OK; |
| 283 | } |
| 284 |
| 285 | @Override |
| 286 | public String getImplementationName() { |
| 287 | return codecName; |
| 288 | } |
| 289 |
| 290 | @Override |
| 291 | public VideoCodecStatus release() { |
| 292 | // TODO(sakal): This is not called on the correct thread but is still called synchronously. |
| 293 | // Re-enable the check once this is called on the correct thread. |
| 294 | // decoderThreadChecker.checkIsOnValidThread(); |
| 295 | Logging.d(TAG, "release"); |
| 296 | VideoCodecStatus status = releaseInternal(); |
| 297 | if (surface != null) { |
| 298 | releaseSurface(); |
| 299 | surface = null; |
| 300 | surfaceTextureHelper.stopListening(); |
| 301 | surfaceTextureHelper.dispose(); |
| 302 | surfaceTextureHelper = null; |
| 303 | } |
| 304 | synchronized (renderedTextureMetadataLock) { |
| 305 | renderedTextureMetadata = null; |
| 306 | } |
| 307 | callback = null; |
| 308 | frameInfos.clear(); |
| 309 | return status; |
| 310 | } |
| 311 |
| 312 | // Internal variant is used when restarting the codec due to reconfiguration. |
| 313 | private VideoCodecStatus releaseInternal() { |
| 314 | if (!running) { |
| 315 | Logging.d(TAG, "release: Decoder is not running."); |
| 316 | return VideoCodecStatus.OK; |
| 317 | } |
| 318 | try { |
| 319 | // The outputThread actually stops and releases the codec once running is false. |
| 320 | running = false; |
| 321 | if (!ThreadUtils.joinUninterruptibly(outputThread, MEDIA_CODEC_RELEASE_TIMEOUT_MS)) { |
| 322 | // Log an exception to capture the stack trace and turn it into a TIMEOUT error. |
| 323 | Logging.e(TAG, "Media decoder release timeout", new RuntimeException()); |
| 324 | return VideoCodecStatus.TIMEOUT; |
| 325 | } |
| 326 | if (shutdownException != null) { |
| 327 | // Log the exception and turn it into an error. Wrap the exception in a new exception to |
| 328 | // capture both the output thread's stack trace and this thread's stack trace. |
| 329 | Logging.e(TAG, "Media decoder release error", new RuntimeException(shutdownException)); |
| 330 | shutdownException = null; |
| 331 | return VideoCodecStatus.ERROR; |
| 332 | } |
| 333 | } finally { |
| 334 | codec = null; |
| 335 | outputThread = null; |
| 336 | } |
| 337 | return VideoCodecStatus.OK; |
| 338 | } |
| 339 |
| 340 | private VideoCodecStatus reinitDecode(int newWidth, int newHeight) { |
| 341 | decoderThreadChecker.checkIsOnValidThread(); |
| 342 | VideoCodecStatus status = releaseInternal(); |
| 343 | if (status != VideoCodecStatus.OK) { |
| 344 | return status; |
| 345 | } |
| 346 | return initDecodeInternal(newWidth, newHeight); |
| 347 | } |
| 348 |
| 349 | private Thread createOutputThread() { |
| 350 | return new Thread("AndroidVideoDecoder.outputThread") { |
| 351 | @Override |
| 352 | public void run() { |
| 353 | outputThreadChecker = new ThreadChecker(); |
| 354 | while (running) { |
| 355 | deliverDecodedFrame(); |
| 356 | } |
| 357 | releaseCodecOnOutputThread(); |
| 358 | } |
| 359 | }; |
| 360 | } |
| 361 |
| 362 | // Visible for testing. |
| 363 | protected void deliverDecodedFrame() { |
| 364 | outputThreadChecker.checkIsOnValidThread(); |
| 365 | try { |
| 366 | MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); |
| 367 | // Block until an output buffer is available (up to 100 milliseconds). If the timeout is |
| 368 | // exceeded, deliverDecodedFrame() will be called again on the next iteration of the output |
| 369 | // thread's loop. Blocking here prevents the output thread from busy-waiting while the codec |
| 370 | // is idle. |
| 371 | int index = codec.dequeueOutputBuffer(info, DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US); |
| 372 | if (index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { |
| 373 | reformat(codec.getOutputFormat()); |
| 374 | return; |
| 375 | } |
| 376 |
| 377 | if (index < 0) { |
| 378 | Logging.v(TAG, "dequeueOutputBuffer returned " + index); |
| 379 | return; |
| 380 | } |
| 381 |
| 382 | FrameInfo frameInfo = frameInfos.poll(); |
| 383 | Integer decodeTimeMs = null; |
| 384 | int rotation = 0; |
| 385 | if (frameInfo != null) { |
| 386 | decodeTimeMs = (int) (SystemClock.elapsedRealtime() - frameInfo.decodeStartTimeMs); |
| 387 | rotation = frameInfo.rotation; |
| 388 | } |
| 389 |
| 390 | hasDecodedFirstFrame = true; |
| 391 |
| 392 | if (surfaceTextureHelper != null) { |
| 393 | deliverTextureFrame(index, info, rotation, decodeTimeMs); |
| 394 | } else { |
| 395 | deliverByteFrame(index, info, rotation, decodeTimeMs); |
| 396 | } |
| 397 |
| 398 | } catch (IllegalStateException e) { |
| 399 | Logging.e(TAG, "deliverDecodedFrame failed", e); |
| 400 | } |
| 401 | } |
| 402 |
| 403 | private void deliverTextureFrame(final int index, final MediaCodec.BufferInfo info, |
| 404 | final int rotation, final Integer decodeTimeMs) { |
| 405 | // Load dimensions from shared memory under the dimension lock. |
| 406 | final int width; |
| 407 | final int height; |
| 408 | synchronized (dimensionLock) { |
| 409 | width = this.width; |
| 410 | height = this.height; |
| 411 | } |
| 412 |
| 413 | synchronized (renderedTextureMetadataLock) { |
| 414 | if (renderedTextureMetadata != null) { |
| 415 | codec.releaseOutputBuffer(index, false); |
| 416 | return; // We are still waiting for texture for the previous frame, drop this one. |
| 417 | } |
| 418 | surfaceTextureHelper.setTextureSize(width, height); |
| 419 | surfaceTextureHelper.setFrameRotation(rotation); |
| 420 | renderedTextureMetadata = new DecodedTextureMetadata(info.presentationTimeUs, decodeTimeMs); |
| 421 | codec.releaseOutputBuffer(index, /* render= */ true); |
| 422 | } |
| 423 | } |
| 424 |
| 425 | @Override |
| 426 | public void onFrame(VideoFrame frame) { |
| 427 | final VideoFrame newFrame; |
| 428 | final Integer decodeTimeMs; |
| 429 | final long timestampNs; |
| 430 | synchronized (renderedTextureMetadataLock) { |
| 431 | if (renderedTextureMetadata == null) { |
| 432 | throw new IllegalStateException( |
| 433 | "Rendered texture metadata was null in onTextureFrameAvailable."); |
| 434 | } |
| 435 | timestampNs = renderedTextureMetadata.presentationTimestampUs * 1000; |
| 436 | decodeTimeMs = renderedTextureMetadata.decodeTimeMs; |
| 437 | renderedTextureMetadata = null; |
| 438 | } |
| 439 | // Change timestamp of frame. |
| 440 | final VideoFrame frameWithModifiedTimeStamp = |
| 441 | new VideoFrame(frame.getBuffer(), frame.getRotation(), timestampNs); |
| 442 | callback.onDecodedFrame(frameWithModifiedTimeStamp, decodeTimeMs, null /* qp */); |
| 443 | } |
| 444 |
| 445 | private void deliverByteFrame( |
| 446 | int index, MediaCodec.BufferInfo info, int rotation, Integer decodeTimeMs) { |
| 447 | // Load dimensions from shared memory under the dimension lock. |
| 448 | int width; |
| 449 | int height; |
| 450 | int stride; |
| 451 | int sliceHeight; |
| 452 | synchronized (dimensionLock) { |
| 453 | width = this.width; |
| 454 | height = this.height; |
| 455 | stride = this.stride; |
| 456 | sliceHeight = this.sliceHeight; |
| 457 | } |
| 458 |
| 459 | // Output must be at least width * height bytes for Y channel, plus (width / 2) * (height / 2) |
| 460 | // bytes for each of the U and V channels. |
| 461 | if (info.size < width * height * 3 / 2) { |
| 462 | Logging.e(TAG, "Insufficient output buffer size: " + info.size); |
| 463 | return; |
| 464 | } |
| 465 |
| 466 | if (info.size < stride * height * 3 / 2 && sliceHeight == height && stride > width) { |
| 467 | // Some codecs (Exynos) report an incorrect stride. Correct it here. |
| 468 | // Expected size == stride * height * 3 / 2. A bit of algebra gives the correct stride as |
| 469 | // 2 * size / (3 * height). |
| 470 | stride = info.size * 2 / (height * 3); |
| 471 | } |
| 472 |
| 473 | ByteBuffer buffer = codec.getOutputBuffer(index); |
| 474 | buffer.position(info.offset); |
| 475 | buffer.limit(info.offset + info.size); |
| 476 | buffer = buffer.slice(); |
| 477 |
| 478 | final VideoFrame.Buffer frameBuffer; |
| 479 | if (colorFormat == CodecCapabilities.COLOR_FormatYUV420Planar) { |
| 480 | frameBuffer = copyI420Buffer(buffer, stride, sliceHeight, width, height); |
| 481 | } else { |
| 482 | // All other supported color formats are NV12. |
| 483 | frameBuffer = copyNV12ToI420Buffer(buffer, stride, sliceHeight, width, height); |
| 484 | } |
| 485 | codec.releaseOutputBuffer(index, /* render= */ false); |
| 486 |
| 487 | long presentationTimeNs = info.presentationTimeUs * 1000; |
| 488 | VideoFrame frame = new VideoFrame(frameBuffer, rotation, presentationTimeNs); |
| 489 |
| 490 | // Note that qp is parsed on the C++ side. |
| 491 | callback.onDecodedFrame(frame, decodeTimeMs, null /* qp */); |
| 492 | frame.release(); |
| 493 | } |
| 494 |
| 495 | private VideoFrame.Buffer copyNV12ToI420Buffer( |
| 496 | ByteBuffer buffer, int stride, int sliceHeight, int width, int height) { |
| 497 | // toI420 copies the buffer. |
| 498 | return new NV12Buffer(width, height, stride, sliceHeight, buffer, null /* releaseCallback */) |
| 499 | .toI420(); |
| 500 | } |
| 501 |
| 502 | private VideoFrame.Buffer copyI420Buffer( |
| 503 | ByteBuffer buffer, int stride, int sliceHeight, int width, int height) { |
| 504 | if (stride % 2 != 0) { |
| 505 | throw new AssertionError("Stride is not divisible by two: " + stride); |
| 506 | } |
| 507 |
| 508 | // Note that the case with odd `sliceHeight` is handled in a special way. |
| 509 | // The chroma height contained in the payload is rounded down instead of |
| 510 | // up, making it one row less than what we expect in WebRTC. Therefore, we |
| 511 | // have to duplicate the last chroma rows for this case. Also, the offset |
| 512 | // between the Y plane and the U plane is unintuitive for this case. See |
| 513 | // http://bugs.webrtc.org/6651 for more info. |
| 514 | final int chromaWidth = (width + 1) / 2; |
| 515 | final int chromaHeight = (sliceHeight % 2 == 0) ? (height + 1) / 2 : height / 2; |
| 516 |
| 517 | final int uvStride = stride / 2; |
| 518 |
| 519 | final int yPos = 0; |
| 520 | final int yEnd = yPos + stride * height; |
| 521 | final int uPos = yPos + stride * sliceHeight; |
| 522 | final int uEnd = uPos + uvStride * chromaHeight; |
| 523 | final int vPos = uPos + uvStride * sliceHeight / 2; |
| 524 | final int vEnd = vPos + uvStride * chromaHeight; |
| 525 |
| 526 | VideoFrame.I420Buffer frameBuffer = allocateI420Buffer(width, height); |
| 527 | try { //don't crash |
| 528 | buffer.limit(yEnd); |
| 529 | buffer.position(yPos); |
| 530 | copyPlane( |
| 531 | buffer.slice(), stride, frameBuffer.getDataY(), frameBuffer.getStrideY(), width, height); |
| 532 |
| 533 | buffer.limit(uEnd); |
| 534 | buffer.position(uPos); |
| 535 | copyPlane(buffer.slice(), uvStride, frameBuffer.getDataU(), frameBuffer.getStrideU(), |
| 536 | chromaWidth, chromaHeight); |
| 537 | if (sliceHeight % 2 == 1) { |
| 538 | buffer.position(uPos + uvStride * (chromaHeight - 1)); // Seek to beginning of last full row. |
| 539 |
| 540 | ByteBuffer dataU = frameBuffer.getDataU(); |
| 541 | dataU.position(frameBuffer.getStrideU() * chromaHeight); // Seek to beginning of last row. |
| 542 | dataU.put(buffer); // Copy the last row. |
| 543 | } |
| 544 |
| 545 | buffer.limit(vEnd); |
| 546 | buffer.position(vPos); |
| 547 | copyPlane(buffer.slice(), uvStride, frameBuffer.getDataV(), frameBuffer.getStrideV(), |
| 548 | chromaWidth, chromaHeight); |
| 549 | if (sliceHeight % 2 == 1) { |
| 550 | buffer.position(vPos + uvStride * (chromaHeight - 1)); // Seek to beginning of last full row. |
| 551 |
| 552 | ByteBuffer dataV = frameBuffer.getDataV(); |
| 553 | dataV.position(frameBuffer.getStrideV() * chromaHeight); // Seek to beginning of last row. |
| 554 | dataV.put(buffer); // Copy the last row. |
| 555 | } |
| 556 | } catch (Throwable e) { |
| 557 | FileLog.e(e); |
| 558 | } |
| 559 |
| 560 | return frameBuffer; |
| 561 | } |
| 562 |
| 563 | private void reformat(MediaFormat format) { |
| 564 | outputThreadChecker.checkIsOnValidThread(); |
| 565 | Logging.d(TAG, "Decoder format changed: " + format); |
| 566 | final int newWidth; |
| 567 | final int newHeight; |
| 568 | if (format.containsKey(MediaFormat.KEY_CROP_LEFT) |
| 569 | && format.containsKey(MediaFormat.KEY_CROP_RIGHT) |
| 570 | && format.containsKey(MediaFormat.KEY_CROP_BOTTOM) |
| 571 | && format.containsKey(MediaFormat.KEY_CROP_TOP)) { |
| 572 | newWidth = 1 + format.getInteger(MediaFormat.KEY_CROP_RIGHT) |
| 573 | - format.getInteger(MediaFormat.KEY_CROP_LEFT); |
| 574 | newHeight = 1 + format.getInteger(MediaFormat.KEY_CROP_BOTTOM) |
| 575 | - format.getInteger(MediaFormat.KEY_CROP_TOP); |
| 576 | } else { |
| 577 | newWidth = format.getInteger(MediaFormat.KEY_WIDTH); |
| 578 | newHeight = format.getInteger(MediaFormat.KEY_HEIGHT); |
| 579 | } |
| 580 | // Compare to existing width, height, and save values under the dimension lock. |
| 581 | synchronized (dimensionLock) { |
| 582 | if (newWidth != width || newHeight != height) { |
| 583 | if (hasDecodedFirstFrame) { |
| 584 | stopOnOutputThread(new RuntimeException("Unexpected size change. " |
| 585 | + "Configured " + width + "*" + height + ". " |
| 586 | + "New " + newWidth + "*" + newHeight)); |
| 587 | return; |
| 588 | } else if (newWidth <= 0 || newHeight <= 0) { |
| 589 | Logging.w(TAG, |
| 590 | "Unexpected format dimensions. Configured " + width + "*" + height + ". " |
| 591 | + "New " + newWidth + "*" + newHeight + ". Skip it"); |
| 592 | return; |
| 593 | } |
| 594 | width = newWidth; |
| 595 | height = newHeight; |
| 596 | } |
| 597 | } |
| 598 |
| 599 | // Note: texture mode ignores colorFormat. Hence, if the texture helper is non-null, skip |
| 600 | // color format updates. |
| 601 | if (surfaceTextureHelper == null && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { |
| 602 | colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); |
| 603 | Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); |
| 604 | if (!isSupportedColorFormat(colorFormat)) { |
| 605 | stopOnOutputThread(new IllegalStateException("Unsupported color format: " + colorFormat)); |
| 606 | return; |
| 607 | } |
| 608 | } |
| 609 |
| 610 | // Save stride and sliceHeight under the dimension lock. |
| 611 | synchronized (dimensionLock) { |
| 612 | if (format.containsKey(MediaFormat.KEY_STRIDE)) { |
| 613 | stride = format.getInteger(MediaFormat.KEY_STRIDE); |
| 614 | } |
| 615 | if (format.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) { |
| 616 | sliceHeight = format.getInteger(MediaFormat.KEY_SLICE_HEIGHT); |
| 617 | } |
| 618 | Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sliceHeight); |
| 619 | stride = Math.max(width, stride); |
| 620 | sliceHeight = Math.max(height, sliceHeight); |
| 621 | } |
| 622 | } |
| 623 |
| 624 | private void releaseCodecOnOutputThread() { |
| 625 | outputThreadChecker.checkIsOnValidThread(); |
| 626 | Logging.d(TAG, "Releasing MediaCodec on output thread"); |
| 627 | try { |
| 628 | codec.stop(); |
| 629 | } catch (Exception e) { |
| 630 | Logging.e(TAG, "Media decoder stop failed", e); |
| 631 | } |
| 632 | try { |
| 633 | codec.release(); |
| 634 | } catch (Exception e) { |
| 635 | Logging.e(TAG, "Media decoder release failed", e); |
| 636 | // Propagate exceptions caught during release back to the main thread. |
| 637 | shutdownException = e; |
| 638 | } |
| 639 | Logging.d(TAG, "Release on output thread done"); |
| 640 | } |
| 641 |
| 642 | private void stopOnOutputThread(Exception e) { |
| 643 | outputThreadChecker.checkIsOnValidThread(); |
| 644 | running = false; |
| 645 | shutdownException = e; |
| 646 | } |
| 647 |
| 648 | private boolean isSupportedColorFormat(int colorFormat) { |
| 649 | for (int supported : MediaCodecUtils.DECODER_COLOR_FORMATS) { |
| 650 | if (supported == colorFormat) { |
| 651 | return true; |
| 652 | } |
| 653 | } |
| 654 | return false; |
| 655 | } |
| 656 |
| 657 | // Visible for testing. |
| 658 | protected SurfaceTextureHelper createSurfaceTextureHelper() { |
| 659 | return SurfaceTextureHelper.create("decoder-texture-thread", sharedContext); |
| 660 | } |
| 661 |
| 662 | // Visible for testing. |
| 663 | // TODO(sakal): Remove once Robolectric commit fa991a0 has been rolled to WebRTC. |
| 664 | protected void releaseSurface() { |
| 665 | surface.release(); |
| 666 | } |
| 667 |
| 668 | // Visible for testing. |
| 669 | protected VideoFrame.I420Buffer allocateI420Buffer(int width, int height) { |
| 670 | return JavaI420Buffer.allocate(width, height); |
| 671 | } |
| 672 |
| 673 | // Visible for testing. |
| 674 | protected void copyPlane( |
| 675 | ByteBuffer src, int srcStride, ByteBuffer dst, int dstStride, int width, int height) { |
| 676 | YuvHelper.copyPlane(src, srcStride, dst, dstStride, width, height); |
| 677 | } |
| 678 | } |
| 679 |