| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 |
| 11 | package org.webrtc; |
| 12 |
| 13 | import android.graphics.Matrix; |
| 14 | import android.opengl.GLES11Ext; |
| 15 | import android.opengl.GLES20; |
| 16 | import androidx.annotation.Nullable; |
| 17 | import java.nio.ByteBuffer; |
| 18 |
| 19 | /** |
| 20 | * Java version of webrtc::VideoFrame and webrtc::VideoFrameBuffer. A difference from the C++ |
| 21 | * version is that no explicit tag is used, and clients are expected to use 'instanceof' to find the |
| 22 | * right subclass of the buffer. This allows clients to create custom VideoFrame.Buffer in |
| 23 | * arbitrary format in their custom VideoSources, and then cast it back to the correct subclass in |
| 24 | * their custom VideoSinks. All implementations must also implement the toI420() function, |
| 25 | * converting from the underlying representation if necessary. I420 is the most widely accepted |
| 26 | * format and serves as a fallback for video sinks that can only handle I420, e.g. the internal |
| 27 | * WebRTC software encoders. |
| 28 | */ |
| 29 | public class VideoFrame implements RefCounted { |
| 30 | /** |
| 31 | * Implements image storage medium. Might be for example an OpenGL texture or a memory region |
| 32 | * containing I420-data. |
| 33 | * |
| 34 | * <p>Reference counting is needed since a video buffer can be shared between multiple VideoSinks, |
| 35 | * and the buffer needs to be returned to the VideoSource as soon as all references are gone. |
| 36 | */ |
| 37 | public interface Buffer extends RefCounted { |
| 38 | /** |
| 39 | * Representation of the underlying buffer. Currently, only NATIVE and I420 are supported. |
| 40 | */ |
| 41 | default int getBufferType() { |
| 42 | return 0; |
| 43 | } |
| 44 |
| 45 | /** |
| 46 | * Resolution of the buffer in pixels. |
| 47 | */ |
| 48 | @CalledByNative("Buffer") int getWidth(); |
| 49 | @CalledByNative("Buffer") int getHeight(); |
| 50 |
| 51 | /** |
| 52 | * Returns a memory-backed frame in I420 format. If the pixel data is in another format, a |
| 53 | * conversion will take place. All implementations must provide a fallback to I420 for |
| 54 | * compatibility with e.g. the internal WebRTC software encoders. |
| 55 | * |
| 56 | * <p> Conversion may fail, for example if reading the pixel data from a texture fails. If the |
| 57 | * conversion fails, null is returned. |
| 58 | */ |
| 59 | @Nullable @CalledByNative("Buffer") I420Buffer toI420(); |
| 60 |
| 61 | @Override @CalledByNative("Buffer") void retain(); |
| 62 | @Override @CalledByNative("Buffer") void release(); |
| 63 |
| 64 | /** |
| 65 | * Crops a region defined by `cropx`, `cropY`, `cropWidth` and `cropHeight`. Scales it to size |
| 66 | * `scaleWidth` x `scaleHeight`. |
| 67 | */ |
| 68 | @CalledByNative("Buffer") |
| 69 | Buffer cropAndScale( |
| 70 | int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight); |
| 71 | } |
| 72 |
| 73 | /** |
| 74 | * Interface for I420 buffers. |
| 75 | */ |
| 76 | public interface I420Buffer extends Buffer { |
| 77 | @Override |
| 78 | default int getBufferType() { |
| 79 | return 1; |
| 80 | } |
| 81 |
| 82 | /** |
| 83 | * Returns a direct ByteBuffer containing Y-plane data. The buffer capacity is at least |
| 84 | * getStrideY() * getHeight() bytes. The position of the returned buffer is ignored and must |
| 85 | * be 0. Callers may mutate the ByteBuffer (eg. through relative-read operations), so |
| 86 | * implementations must return a new ByteBuffer or slice for each call. |
| 87 | */ |
| 88 | @CalledByNative("I420Buffer") ByteBuffer getDataY(); |
| 89 | /** |
| 90 | * Returns a direct ByteBuffer containing U-plane data. The buffer capacity is at least |
| 91 | * getStrideU() * ((getHeight() + 1) / 2) bytes. The position of the returned buffer is ignored |
| 92 | * and must be 0. Callers may mutate the ByteBuffer (eg. through relative-read operations), so |
| 93 | * implementations must return a new ByteBuffer or slice for each call. |
| 94 | */ |
| 95 | @CalledByNative("I420Buffer") ByteBuffer getDataU(); |
| 96 | /** |
| 97 | * Returns a direct ByteBuffer containing V-plane data. The buffer capacity is at least |
| 98 | * getStrideV() * ((getHeight() + 1) / 2) bytes. The position of the returned buffer is ignored |
| 99 | * and must be 0. Callers may mutate the ByteBuffer (eg. through relative-read operations), so |
| 100 | * implementations must return a new ByteBuffer or slice for each call. |
| 101 | */ |
| 102 | @CalledByNative("I420Buffer") ByteBuffer getDataV(); |
| 103 |
| 104 | @CalledByNative("I420Buffer") int getStrideY(); |
| 105 | @CalledByNative("I420Buffer") int getStrideU(); |
| 106 | @CalledByNative("I420Buffer") int getStrideV(); |
| 107 | } |
| 108 |
| 109 | /** |
| 110 | * Interface for buffers that are stored as a single texture, either in OES or RGB format. |
| 111 | */ |
| 112 | public interface TextureBuffer extends Buffer { |
| 113 | enum Type { |
| 114 | OES(GLES11Ext.GL_TEXTURE_EXTERNAL_OES), |
| 115 | RGB(GLES20.GL_TEXTURE_2D); |
| 116 |
| 117 | private final int glTarget; |
| 118 |
| 119 | private Type(final int glTarget) { |
| 120 | this.glTarget = glTarget; |
| 121 | } |
| 122 |
| 123 | public int getGlTarget() { |
| 124 | return glTarget; |
| 125 | } |
| 126 | } |
| 127 |
| 128 | Type getType(); |
| 129 | int getTextureId(); |
| 130 |
| 131 | /** |
| 132 | * Retrieve the transform matrix associated with the frame. This transform matrix maps 2D |
| 133 | * homogeneous coordinates of the form (s, t, 1) with s and t in the inclusive range [0, 1] to |
| 134 | * the coordinate that should be used to sample that location from the buffer. |
| 135 | */ |
| 136 | Matrix getTransformMatrix(); |
| 137 |
| 138 | /** |
| 139 | * Create a new TextureBufferImpl with an applied transform matrix and a new size. The existing |
| 140 | * buffer is unchanged. The given transform matrix is applied first when texture coordinates are |
| 141 | * still in the unmodified [0, 1] range. |
| 142 | */ |
| 143 | default TextureBuffer applyTransformMatrix( |
| 144 | Matrix transformMatrix, int newWidth, int newHeight) { |
| 145 | throw new UnsupportedOperationException("Not implemented"); |
| 146 | } |
| 147 |
| 148 | /** |
| 149 | * Returns the width of the texture in memory. This should only be used for downscaling, and you |
| 150 | * should still respect the width from getWidth(). |
| 151 | */ |
| 152 | default public int getUnscaledWidth() { |
| 153 | return getWidth(); |
| 154 | } |
| 155 |
| 156 | /** |
| 157 | * Returns the height of the texture in memory. This should only be used for downscaling, and |
| 158 | * you should still respect the height from getHeight(). |
| 159 | */ |
| 160 | default public int getUnscaledHeight() { |
| 161 | return getHeight(); |
| 162 | } |
| 163 | } |
| 164 |
| 165 | private final Buffer buffer; |
| 166 | private final int rotation; |
| 167 | private final long timestampNs; |
| 168 |
| 169 | /** |
| 170 | * Constructs a new VideoFrame backed by the given {@code buffer}. |
| 171 | * |
| 172 | * @note Ownership of the buffer object is tranferred to the new VideoFrame. |
| 173 | */ |
| 174 | @CalledByNative |
| 175 | public VideoFrame(Buffer buffer, int rotation, long timestampNs) { |
| 176 | if (buffer == null) { |
| 177 | throw new IllegalArgumentException("buffer not allowed to be null"); |
| 178 | } |
| 179 | if (rotation % 90 != 0) { |
| 180 | throw new IllegalArgumentException("rotation must be a multiple of 90"); |
| 181 | } |
| 182 | this.buffer = buffer; |
| 183 | this.rotation = rotation; |
| 184 | this.timestampNs = timestampNs; |
| 185 | } |
| 186 |
| 187 | @CalledByNative |
| 188 | public Buffer getBuffer() { |
| 189 | return buffer; |
| 190 | } |
| 191 |
| 192 | /** |
| 193 | * Rotation of the frame in degrees. |
| 194 | */ |
| 195 | @CalledByNative |
| 196 | public int getRotation() { |
| 197 | return rotation; |
| 198 | } |
| 199 |
| 200 | /** |
| 201 | * Timestamp of the frame in nano seconds. |
| 202 | */ |
| 203 | @CalledByNative |
| 204 | public long getTimestampNs() { |
| 205 | return timestampNs; |
| 206 | } |
| 207 |
| 208 | public int getRotatedWidth() { |
| 209 | if (rotation % 180 == 0) { |
| 210 | return buffer.getWidth(); |
| 211 | } |
| 212 | return buffer.getHeight(); |
| 213 | } |
| 214 |
| 215 | public int getRotatedHeight() { |
| 216 | if (rotation % 180 == 0) { |
| 217 | return buffer.getHeight(); |
| 218 | } |
| 219 | return buffer.getWidth(); |
| 220 | } |
| 221 |
| 222 | @Override |
| 223 | public void retain() { |
| 224 | buffer.retain(); |
| 225 | } |
| 226 |
| 227 | @Override |
| 228 | @CalledByNative |
| 229 | public void release() { |
| 230 | buffer.release(); |
| 231 | } |
| 232 | } |
| 233 |