All Downloads are FREE. Search and download functionalities are using the official Maven repository.

src.android.media.ImageReader Maven / Gradle / Ivy

Go to download

A library jar that provides APIs for Applications written for the Google Android Platform.

There is a newer version: 15-robolectric-12650502
Show newest version
/*
 * Copyright (C) 2013 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.media;

import android.annotation.CallbackExecutor;
import android.annotation.IntRange;
import android.annotation.NonNull;
import android.graphics.GraphicBuffer;
import android.graphics.ImageFormat;
import android.graphics.ImageFormat.Format;
import android.graphics.Rect;
import android.hardware.HardwareBuffer;
import android.hardware.HardwareBuffer.Usage;
import android.hardware.camera2.MultiResolutionImageReader;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
import android.view.Surface;

import dalvik.system.VMRuntime;

import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.NioUtils;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * 

The ImageReader class allows direct application access to image data * rendered into a {@link android.view.Surface}

* *

Several Android media API classes accept Surface objects as targets to * render to, including {@link MediaPlayer}, {@link MediaCodec}, * {@link android.hardware.camera2.CameraDevice}, {@link ImageWriter} and * {@link android.renderscript.Allocation RenderScript Allocations}. The image * sizes and formats that can be used with each source vary, and should be * checked in the documentation for the specific API.

* *

The image data is encapsulated in {@link Image} objects, and multiple such * objects can be accessed at the same time, up to the number specified by the * {@code maxImages} constructor parameter. New images sent to an ImageReader * through its {@link Surface} are queued until accessed through the {@link #acquireLatestImage} * or {@link #acquireNextImage} call. Due to memory limits, an image source will * eventually stall or drop Images in trying to render to the Surface if the * ImageReader does not obtain and release Images at a rate equal to the * production rate.

*/ public class ImageReader implements AutoCloseable { /** * Returned by nativeImageSetup when acquiring the image was successful. */ private static final int ACQUIRE_SUCCESS = 0; /** * Returned by nativeImageSetup when we couldn't acquire the buffer, * because there were no buffers available to acquire. */ private static final int ACQUIRE_NO_BUFS = 1; /** * Returned by nativeImageSetup when we couldn't acquire the buffer * because the consumer has already acquired {@maxImages} and cannot * acquire more than that. */ private static final int ACQUIRE_MAX_IMAGES = 2; /** *

* Create a new reader for images of the desired size and format. *

*

* The {@code maxImages} parameter determines the maximum number of * {@link Image} objects that can be be acquired from the * {@code ImageReader} simultaneously. Requesting more buffers will use up * more memory, so it is important to use only the minimum number necessary * for the use case. *

*

* The valid sizes and formats depend on the source of the image data. *

*

* If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created * {@link ImageReader} will produce images that are not directly accessible * by the application. The application can still acquire images from this * {@link ImageReader}, and send them to the * {@link android.hardware.camera2.CameraDevice camera} for reprocessing via * {@link ImageWriter} interface. However, the {@link Image#getPlanes() * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE * PRIVATE} format images. The application can check if an existing reader's * format by calling {@link #getImageFormat()}. *

*

* {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader * ImageReaders} are more efficient to use when application access to image * data is not necessary, compared to ImageReaders using other format such * as {@link ImageFormat#YUV_420_888 YUV_420_888}. *

* * @param width The default width in pixels of the Images that this reader * will produce. * @param height The default height in pixels of the Images that this reader * will produce. * @param format The format of the Image that this reader will produce. This * must be one of the {@link android.graphics.ImageFormat} or * {@link android.graphics.PixelFormat} constants. Note that not * all formats are supported, like ImageFormat.NV21. * @param maxImages The maximum number of images the user will want to * access simultaneously. This should be as small as possible to * limit memory use. Once maxImages Images are obtained by the * user, one of them has to be released before a new Image will * become available for access through * {@link #acquireLatestImage()} or {@link #acquireNextImage()}. * Must be greater than 0. * @see Image */ public static @NonNull ImageReader newInstance( @IntRange(from = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages) { // If the format is private don't default to USAGE_CPU_READ_OFTEN since it may not // work, and is inscrutable anyway return new ImageReader(width, height, format, maxImages, format == ImageFormat.PRIVATE ? 0 : HardwareBuffer.USAGE_CPU_READ_OFTEN, /*parent*/ null); } /** *

* Create a new reader for images of the desired size, format and consumer usage flag. *

*

* The {@code maxImages} parameter determines the maximum number of {@link Image} objects that * can be be acquired from the {@code ImageReader} simultaneously. Requesting more buffers will * use up more memory, so it is important to use only the minimum number necessary for the use * case. *

*

* The valid sizes and formats depend on the source of the image data. *

*

* The format and usage flag combination describes how the buffer will be used by * consumer end-points. For example, if the application intends to send the images to * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder} for hardware video * encoding, the format and usage flag combination needs to be * {@link ImageFormat#PRIVATE PRIVATE} and {@link HardwareBuffer#USAGE_VIDEO_ENCODE}. When an * {@link ImageReader} object is created with a valid size and such format/usage flag * combination, the application can send the {@link Image images} to an {@link ImageWriter} that * is created with the input {@link android.view.Surface} provided by the * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder}. *

*

* If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created {@link ImageReader} * will produce images that are not directly accessible by the application. The application can * still acquire images from this {@link ImageReader}, and send them to the * {@link android.hardware.camera2.CameraDevice camera} for reprocessing, or to the * {@link android.media.MediaCodec} / {@link android.media.MediaRecorder} for hardware video * encoding via {@link ImageWriter} interface. However, the {@link Image#getPlanes() * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE PRIVATE} format * images. The application can check if an existing reader's format by calling * {@link #getImageFormat()}. *

*

* {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader ImageReaders} are more * efficient to use when application access to image data is not necessary, compared to * ImageReaders using other format such as {@link ImageFormat#YUV_420_888 YUV_420_888}. *

*

* Note that not all format and usage flag combinations are supported by the * {@link ImageReader}. Below are the supported combinations by the {@link ImageReader} * (assuming the consumer end-points support the such image consumption, e.g., hardware video * encoding). *

* * * * * * * * * * * * *
FormatCompatible usage flags
non-{@link android.graphics.ImageFormat#PRIVATE PRIVATE} formats defined by * {@link android.graphics.ImageFormat ImageFormat} or * {@link android.graphics.PixelFormat PixelFormat}{@link HardwareBuffer#USAGE_CPU_READ_RARELY} or * {@link HardwareBuffer#USAGE_CPU_READ_OFTEN}
{@link android.graphics.ImageFormat#PRIVATE}{@link HardwareBuffer#USAGE_VIDEO_ENCODE} or * {@link HardwareBuffer#USAGE_GPU_SAMPLED_IMAGE}, or combined
* Using other combinations may result in {@link IllegalArgumentException}. Additionally, * specifying {@link HardwareBuffer#USAGE_CPU_WRITE_RARELY} or * {@link HardwareBuffer#USAGE_CPU_WRITE_OFTEN} and writing to the ImageReader's buffers * might break assumptions made by some producers, and should be used with caution. *

*

* If the {@link ImageReader} is used as an output target for a {@link * android.hardware.camera2.CameraDevice}, and if the usage flag contains * {@link HardwareBuffer#USAGE_VIDEO_ENCODE}, the timestamps of the * {@link Image images} produced by the {@link ImageReader} won't be in the same timebase as * {@link android.os.SystemClock#elapsedRealtimeNanos}, even if * {@link android.hardware.camera2.CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE} is * {@link android.hardware.camera2.CameraCharacteristics#SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME}. * Instead, the timestamps will be roughly in the same timebase as in * {@link android.os.SystemClock#uptimeMillis}, so that A/V synchronization could work for * video recording. In this case, the timestamps from the {@link ImageReader} with * {@link HardwareBuffer#USAGE_VIDEO_ENCODE} usage flag may not be directly comparable with * timestamps of other streams or capture result metadata. *

* @param width The default width in pixels of the Images that this reader will produce. * @param height The default height in pixels of the Images that this reader will produce. * @param format The format of the Image that this reader will produce. This must be one of the * {@link android.graphics.ImageFormat} or {@link android.graphics.PixelFormat} * constants. Note that not all formats are supported, like ImageFormat.NV21. * @param maxImages The maximum number of images the user will want to access simultaneously. * This should be as small as possible to limit memory use. Once maxImages Images are * obtained by the user, one of them has to be released before a new Image will * become available for access through {@link #acquireLatestImage()} or * {@link #acquireNextImage()}. Must be greater than 0. * @param usage The intended usage of the images produced by this ImageReader. See the usages * on {@link HardwareBuffer} for a list of valid usage bits. See also * {@link HardwareBuffer#isSupported(int, int, int, int, long)} for checking * if a combination is supported. If it's not supported this will throw * an {@link IllegalArgumentException}. * @see Image * @see HardwareBuffer */ public static @NonNull ImageReader newInstance( @IntRange(from = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages, @Usage long usage) { // TODO: Check this - can't do it just yet because format support is different // Unify formats! The only reliable way to validate usage is to just try it and see. // if (!HardwareBuffer.isSupported(width, height, format, 1, usage)) { // throw new IllegalArgumentException("The given format=" + Integer.toHexString(format) // + " & usage=" + Long.toHexString(usage) + " is not supported"); // } return new ImageReader(width, height, format, maxImages, usage, /*parent*/ null); } /** * @hide */ public static @NonNull ImageReader newInstance( @IntRange(from = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages, @NonNull MultiResolutionImageReader parent) { // If the format is private don't default to USAGE_CPU_READ_OFTEN since it may not // work, and is inscrutable anyway return new ImageReader(width, height, format, maxImages, format == ImageFormat.PRIVATE ? 0 : HardwareBuffer.USAGE_CPU_READ_OFTEN, parent); } /** * @hide */ protected ImageReader(int width, int height, int format, int maxImages, long usage, MultiResolutionImageReader parent) { mWidth = width; mHeight = height; mFormat = format; mUsage = usage; mMaxImages = maxImages; mParent = parent; if (width < 1 || height < 1) { throw new IllegalArgumentException( "The image dimensions must be positive"); } if (mMaxImages < 1) { throw new IllegalArgumentException( "Maximum outstanding image count must be at least 1"); } if (format == ImageFormat.NV21) { throw new IllegalArgumentException( "NV21 format is not supported"); } mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat); nativeInit(new WeakReference<>(this), width, height, format, maxImages, usage); mSurface = nativeGetSurface(); mIsReaderValid = true; // Estimate the native buffer allocation size and register it so it gets accounted for // during GC. Note that this doesn't include the buffers required by the buffer queue // itself and the buffers requested by the producer. // Only include memory for 1 buffer, since actually accounting for the memory used is // complex, and 1 buffer is enough for the VM to treat the ImageReader as being of some // size. mEstimatedNativeAllocBytes = ImageUtils.getEstimatedNativeAllocBytes( width, height, format, /*buffer count*/ 1); VMRuntime.getRuntime().registerNativeAllocation(mEstimatedNativeAllocBytes); } /** * The default width of {@link Image Images}, in pixels. * *

The width may be overridden by the producer sending buffers to this * ImageReader's Surface. If so, the actual width of the images can be * found using {@link Image#getWidth}.

* * @return the expected width of an Image */ public int getWidth() { return mWidth; } /** * The default height of {@link Image Images}, in pixels. * *

The height may be overridden by the producer sending buffers to this * ImageReader's Surface. If so, the actual height of the images can be * found using {@link Image#getHeight}.

* * @return the expected height of an Image */ public int getHeight() { return mHeight; } /** * The default {@link ImageFormat image format} of {@link Image Images}. * *

Some color formats may be overridden by the producer sending buffers to * this ImageReader's Surface if the default color format allows. ImageReader * guarantees that all {@link Image Images} acquired from ImageReader * (for example, with {@link #acquireNextImage}) will have a "compatible" * format to what was specified in {@link #newInstance}. * As of now, each format is only compatible to itself. * The actual format of the images can be found using {@link Image#getFormat}.

* * @return the expected format of an Image * * @see ImageFormat */ public int getImageFormat() { return mFormat; } /** * Maximum number of images that can be acquired from the ImageReader by any time (for example, * with {@link #acquireNextImage}). * *

An image is considered acquired after it's returned by a function from ImageReader, and * until the Image is {@link Image#close closed} to release the image back to the ImageReader. *

* *

Attempting to acquire more than {@code maxImages} concurrently will result in the * acquire function throwing a {@link IllegalStateException}. Furthermore, * while the max number of images have been acquired by the ImageReader user, the producer * enqueueing additional images may stall until at least one image has been released.

* * @return Maximum number of images for this ImageReader. * * @see Image#close */ public int getMaxImages() { return mMaxImages; } /** *

Get a {@link Surface} that can be used to produce {@link Image Images} for this * {@code ImageReader}.

* *

Until valid image data is rendered into this {@link Surface}, the * {@link #acquireNextImage} method will return {@code null}. Only one source * can be producing data into this Surface at the same time, although the * same {@link Surface} can be reused with a different API once the first source is * disconnected from the {@link Surface}.

* *

Please note that holding on to the Surface object returned by this method is not enough * to keep its parent ImageReader from being reclaimed. In that sense, a Surface acts like a * {@link java.lang.ref.WeakReference weak reference} to the ImageReader that provides it.

* * @return A {@link Surface} to use for a drawing target for various APIs. */ public Surface getSurface() { return mSurface; } /** *

* Acquire the latest {@link Image} from the ImageReader's queue, dropping older * {@link Image images}. Returns {@code null} if no new image is available. *

*

* This operation will acquire all the images possible from the ImageReader, * but {@link #close} all images that aren't the latest. This function is * recommended to use over {@link #acquireNextImage} for most use-cases, as it's * more suited for real-time processing. *

*

* Note that {@link #getMaxImages maxImages} should be at least 2 for * {@link #acquireLatestImage} to be any different than {@link #acquireNextImage} - * discarding all-but-the-newest {@link Image} requires temporarily acquiring two * {@link Image Images} at once. Or more generally, calling {@link #acquireLatestImage} * with less than two images of margin, that is * {@code (maxImages - currentAcquiredImages < 2)} will not discard as expected. *

*

* This operation will fail by throwing an {@link IllegalStateException} if * {@code maxImages} have been acquired with {@link #acquireLatestImage} or * {@link #acquireNextImage}. In particular a sequence of {@link #acquireLatestImage} * calls greater than {@link #getMaxImages} without calling {@link Image#close} in-between * will exhaust the underlying queue. At such a time, {@link IllegalStateException} * will be thrown until more images are * released with {@link Image#close}. *

* * @return latest frame of image data, or {@code null} if no image data is available. * @throws IllegalStateException if too many images are currently acquired */ public Image acquireLatestImage() { Image image = acquireNextImage(); if (image == null) { return null; } try { for (;;) { Image next = acquireNextImageNoThrowISE(); if (next == null) { Image result = image; image = null; return result; } image.close(); image = next; } } finally { if (image != null) { image.close(); } if (mParent != null) { mParent.flushOther(this); } } } /** * Don't throw IllegalStateException if there are too many images acquired. * * @return Image if acquiring succeeded, or null otherwise. * * @hide */ public Image acquireNextImageNoThrowISE() { SurfaceImage si = new SurfaceImage(mFormat); return acquireNextSurfaceImage(si) == ACQUIRE_SUCCESS ? si : null; } /** * Attempts to acquire the next image from the underlying native implementation. * *

* Note that unexpected failures will throw at the JNI level. *

* * @param si A blank SurfaceImage. * @return One of the {@code ACQUIRE_*} codes that determine success or failure. * * @see #ACQUIRE_MAX_IMAGES * @see #ACQUIRE_NO_BUFS * @see #ACQUIRE_SUCCESS */ private int acquireNextSurfaceImage(SurfaceImage si) { synchronized (mCloseLock) { // A null image will eventually be returned if ImageReader is already closed. int status = ACQUIRE_NO_BUFS; if (mIsReaderValid) { status = nativeImageSetup(si); } switch (status) { case ACQUIRE_SUCCESS: si.mIsImageValid = true; case ACQUIRE_NO_BUFS: case ACQUIRE_MAX_IMAGES: break; default: throw new AssertionError("Unknown nativeImageSetup return code " + status); } // Only keep track the successfully acquired image, as the native buffer is only mapped // for such case. if (status == ACQUIRE_SUCCESS) { mAcquiredImages.add(si); } return status; } } /** *

* Acquire the next Image from the ImageReader's queue. Returns {@code null} if * no new image is available. *

* *

Warning: Consider using {@link #acquireLatestImage()} instead, as it will * automatically release older images, and allow slower-running processing routines to catch * up to the newest frame. Usage of {@link #acquireNextImage} is recommended for * batch/background processing. Incorrectly using this function can cause images to appear * with an ever-increasing delay, followed by a complete stall where no new images seem to * appear. *

* *

* This operation will fail by throwing an {@link IllegalStateException} if * {@code maxImages} have been acquired with {@link #acquireNextImage} or * {@link #acquireLatestImage}. In particular a sequence of {@link #acquireNextImage} or * {@link #acquireLatestImage} calls greater than {@link #getMaxImages maxImages} without * calling {@link Image#close} in-between will exhaust the underlying queue. At such a time, * {@link IllegalStateException} will be thrown until more images are released with * {@link Image#close}. *

* * @return a new frame of image data, or {@code null} if no image data is available. * @throws IllegalStateException if {@code maxImages} images are currently acquired * @see #acquireLatestImage */ public Image acquireNextImage() { // Initialize with reader format, but can be overwritten by native if the image // format is different from the reader format. SurfaceImage si = new SurfaceImage(mFormat); int status = acquireNextSurfaceImage(si); switch (status) { case ACQUIRE_SUCCESS: return si; case ACQUIRE_NO_BUFS: return null; case ACQUIRE_MAX_IMAGES: throw new IllegalStateException( String.format( "maxImages (%d) has already been acquired, " + "call #close before acquiring more.", mMaxImages)); default: throw new AssertionError("Unknown nativeImageSetup return code " + status); } } /** *

Return the frame to the ImageReader for reuse.

*/ private void releaseImage(Image i) { if (! (i instanceof SurfaceImage) ) { throw new IllegalArgumentException( "This image was not produced by an ImageReader"); } SurfaceImage si = (SurfaceImage) i; if (si.mIsImageValid == false) { return; } if (si.getReader() != this || !mAcquiredImages.contains(i)) { throw new IllegalArgumentException( "This image was not produced by this ImageReader"); } si.clearSurfacePlanes(); nativeReleaseImage(i); si.mIsImageValid = false; mAcquiredImages.remove(i); } /** * Register a listener to be invoked when a new image becomes available * from the ImageReader. * * @param listener * The listener that will be run. * @param handler * The handler on which the listener should be invoked, or null * if the listener should be invoked on the calling thread's looper. * @throws IllegalArgumentException * If no handler specified and the calling thread has no looper. */ public void setOnImageAvailableListener(OnImageAvailableListener listener, Handler handler) { synchronized (mListenerLock) { if (listener != null) { Looper looper = handler != null ? handler.getLooper() : Looper.myLooper(); if (looper == null) { throw new IllegalArgumentException( "handler is null but the current thread is not a looper"); } if (mListenerHandler == null || mListenerHandler.getLooper() != looper) { mListenerHandler = new ListenerHandler(looper); mListenerExecutor = new HandlerExecutor(mListenerHandler); } } else { mListenerHandler = null; mListenerExecutor = null; } mListener = listener; } } /** * Register a listener to be invoked when a new image becomes available * from the ImageReader. * * @param listener * The listener that will be run. * @param executor * The executor which will be used to invoke the listener. * @throws IllegalArgumentException * If no handler specified and the calling thread has no looper. * * @hide */ public void setOnImageAvailableListenerWithExecutor(@NonNull OnImageAvailableListener listener, @NonNull Executor executor) { if (executor == null) { throw new IllegalArgumentException("executor must not be null"); } synchronized (mListenerLock) { mListenerExecutor = executor; mListener = listener; } } /** * Callback interface for being notified that a new image is available. * *

* The onImageAvailable is called per image basis, that is, callback fires for every new frame * available from ImageReader. *

*/ public interface OnImageAvailableListener { /** * Callback that is called when a new image is available from ImageReader. * * @param reader the ImageReader the callback is associated with. * @see ImageReader * @see Image */ void onImageAvailable(ImageReader reader); } /** * Free up all the resources associated with this ImageReader. * *

* After calling this method, this ImageReader can not be used. Calling * any methods on this ImageReader and Images previously provided by * {@link #acquireNextImage} or {@link #acquireLatestImage} * will result in an {@link IllegalStateException}, and attempting to read from * {@link ByteBuffer ByteBuffers} returned by an earlier * {@link Image.Plane#getBuffer Plane#getBuffer} call will * have undefined behavior. *

*/ @Override public void close() { setOnImageAvailableListener(null, null); if (mSurface != null) mSurface.release(); /** * Close all outstanding acquired images before closing the ImageReader. It is a good * practice to close all the images as soon as it is not used to reduce system instantaneous * memory pressure. CopyOnWrite list will use a copy of current list content. For the images * being closed by other thread (e.g., GC thread), doubling the close call is harmless. For * the image being acquired by other threads, mCloseLock is used to synchronize close and * acquire operations. */ synchronized (mCloseLock) { mIsReaderValid = false; for (Image image : mAcquiredImages) { image.close(); } mAcquiredImages.clear(); nativeClose(); if (mEstimatedNativeAllocBytes > 0) { VMRuntime.getRuntime().registerNativeFree(mEstimatedNativeAllocBytes); mEstimatedNativeAllocBytes = 0; } } } /** * Discard any free buffers owned by this ImageReader. * *

* Generally, the ImageReader caches buffers for reuse once they have been * allocated, for best performance. However, sometimes it may be important to * release all the cached, unused buffers to save on memory. *

*

* Calling this method will discard all free cached buffers. This does not include any buffers * associated with Images acquired from the ImageReader, any filled buffers waiting to be * acquired, and any buffers currently in use by the source rendering buffers into the * ImageReader's Surface. *

* The ImageReader continues to be usable after this call, but may need to reallocate buffers * when more buffers are needed for rendering. *

*/ public void discardFreeBuffers() { synchronized (mCloseLock) { nativeDiscardFreeBuffers(); } } @Override protected void finalize() throws Throwable { try { close(); } finally { super.finalize(); } } /** *

* Remove the ownership of this image from the ImageReader. *

*

* After this call, the ImageReader no longer owns this image, and the image * ownership can be transfered to another entity like {@link ImageWriter} * via {@link ImageWriter#queueInputImage}. It's up to the new owner to * release the resources held by this image. For example, if the ownership * of this image is transfered to an {@link ImageWriter}, the image will be * freed by the ImageWriter after the image data consumption is done. *

*

* This method can be used to achieve zero buffer copy for use cases like * {@link android.hardware.camera2.CameraDevice Camera2 API} PRIVATE and YUV * reprocessing, where the application can select an output image from * {@link ImageReader} and transfer this image directly to * {@link ImageWriter}, where this image can be consumed by camera directly. * For PRIVATE reprocessing, this is the only way to send input buffers to * the {@link android.hardware.camera2.CameraDevice camera} for * reprocessing. *

*

* This is a package private method that is only used internally. *

* * @param image The image to be detached from this ImageReader. * @throws IllegalStateException If the ImageReader or image have been * closed, or the has been detached, or has not yet been * acquired. * @hide */ public void detachImage(Image image) { if (image == null) { throw new IllegalArgumentException("input image must not be null"); } if (!isImageOwnedbyMe(image)) { throw new IllegalArgumentException("Trying to detach an image that is not owned by" + " this ImageReader"); } SurfaceImage si = (SurfaceImage) image; si.throwISEIfImageIsInvalid(); if (si.isAttachable()) { throw new IllegalStateException("Image was already detached from this ImageReader"); } nativeDetachImage(image); si.clearSurfacePlanes(); si.mPlanes = null; si.setDetached(true); } private boolean isImageOwnedbyMe(Image image) { if (!(image instanceof SurfaceImage)) { return false; } SurfaceImage si = (SurfaceImage) image; return si.getReader() == this; } /** * Called from Native code when an Event happens. * * This may be called from an arbitrary Binder thread, so access to the ImageReader must be * synchronized appropriately. */ private static void postEventFromNative(Object selfRef) { @SuppressWarnings("unchecked") WeakReference weakSelf = (WeakReference)selfRef; final ImageReader ir = weakSelf.get(); if (ir == null) { return; } final Executor executor; final OnImageAvailableListener listener; synchronized (ir.mListenerLock) { executor = ir.mListenerExecutor; listener = ir.mListener; } final boolean isReaderValid; synchronized (ir.mCloseLock) { isReaderValid = ir.mIsReaderValid; } // It's dangerous to fire onImageAvailable() callback when the ImageReader // is being closed, as application could acquire next image in the // onImageAvailable() callback. if (executor != null && listener != null && isReaderValid) { executor.execute(new Runnable() { @Override public void run() { listener.onImageAvailable(ir); } }); } } private final int mWidth; private final int mHeight; private final int mFormat; private final long mUsage; private final int mMaxImages; private final int mNumPlanes; private final Surface mSurface; private int mEstimatedNativeAllocBytes; private final Object mListenerLock = new Object(); private final Object mCloseLock = new Object(); private boolean mIsReaderValid = false; private OnImageAvailableListener mListener; private Executor mListenerExecutor; private ListenerHandler mListenerHandler; // Keep track of the successfully acquired Images. This need to be thread safe as the images // could be closed by different threads (e.g., application thread and GC thread). private List mAcquiredImages = new CopyOnWriteArrayList<>(); // Applicable if this isn't a standalone ImageReader, but belongs to a // MultiResolutionImageReader. private final MultiResolutionImageReader mParent; /** * This field is used by native code, do not access or modify. */ private long mNativeContext; /** * This custom handler runs asynchronously so callbacks don't get queued behind UI messages. */ private final class ListenerHandler extends Handler { public ListenerHandler(Looper looper) { super(looper, null, true /*async*/); } } /** * An adapter {@link Executor} that posts all executed tasks onto the * given {@link Handler}. **/ private final class HandlerExecutor implements Executor { private final Handler mHandler; public HandlerExecutor(@NonNull Handler handler) { mHandler = Objects.requireNonNull(handler); } @Override public void execute(Runnable command) { mHandler.post(command); } } private class SurfaceImage extends android.media.Image { public SurfaceImage(int format) { mFormat = format; } @Override public void close() { ImageReader.this.releaseImage(this); } public ImageReader getReader() { return ImageReader.this; } @Override public int getFormat() { throwISEIfImageIsInvalid(); int readerFormat = ImageReader.this.getImageFormat(); // Assume opaque reader always produce opaque images. mFormat = (readerFormat == ImageFormat.PRIVATE) ? readerFormat : nativeGetFormat(readerFormat); return mFormat; } @Override public int getWidth() { throwISEIfImageIsInvalid(); int width; switch(getFormat()) { case ImageFormat.JPEG: case ImageFormat.DEPTH_POINT_CLOUD: case ImageFormat.RAW_PRIVATE: case ImageFormat.DEPTH_JPEG: case ImageFormat.HEIC: width = ImageReader.this.getWidth(); break; default: width = nativeGetWidth(); } return width; } @Override public int getHeight() { throwISEIfImageIsInvalid(); int height; switch(getFormat()) { case ImageFormat.JPEG: case ImageFormat.DEPTH_POINT_CLOUD: case ImageFormat.RAW_PRIVATE: case ImageFormat.DEPTH_JPEG: case ImageFormat.HEIC: height = ImageReader.this.getHeight(); break; default: height = nativeGetHeight(); } return height; } @Override public long getTimestamp() { throwISEIfImageIsInvalid(); return mTimestamp; } @Override public int getTransform() { throwISEIfImageIsInvalid(); return mTransform; } @Override public int getScalingMode() { throwISEIfImageIsInvalid(); return mScalingMode; } @Override public int getPlaneCount() { throwISEIfImageIsInvalid(); return ImageReader.this.mNumPlanes; } @Override public int getFenceFd() { throwISEIfImageIsInvalid(); return nativeGetFenceFd(); } @Override public HardwareBuffer getHardwareBuffer() { throwISEIfImageIsInvalid(); return nativeGetHardwareBuffer(); } @Override public void setTimestamp(long timestampNs) { throwISEIfImageIsInvalid(); mTimestamp = timestampNs; } @Override public Plane[] getPlanes() { throwISEIfImageIsInvalid(); if (mPlanes == null) { mPlanes = nativeCreatePlanes(ImageReader.this.mNumPlanes, ImageReader.this.mFormat, ImageReader.this.mUsage); } // Shallow copy is fine. return mPlanes.clone(); } @Override protected final void finalize() throws Throwable { try { close(); } finally { super.finalize(); } } @Override public boolean isAttachable() { throwISEIfImageIsInvalid(); return mIsDetached.get(); } @Override ImageReader getOwner() { throwISEIfImageIsInvalid(); return ImageReader.this; } @Override long getNativeContext() { throwISEIfImageIsInvalid(); return mNativeBuffer; } private void setDetached(boolean detached) { throwISEIfImageIsInvalid(); mIsDetached.getAndSet(detached); } private void clearSurfacePlanes() { // Image#getPlanes may not be called before the image is closed. if (mIsImageValid && mPlanes != null) { for (int i = 0; i < mPlanes.length; i++) { if (mPlanes[i] != null) { mPlanes[i].clearBuffer(); mPlanes[i] = null; } } } } private class SurfacePlane extends android.media.Image.Plane { // SurfacePlane instance is created by native code when SurfaceImage#getPlanes() is // called private SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer) { mRowStride = rowStride; mPixelStride = pixelStride; mBuffer = buffer; /** * Set the byteBuffer order according to host endianness (native * order), otherwise, the byteBuffer order defaults to * ByteOrder.BIG_ENDIAN. */ mBuffer.order(ByteOrder.nativeOrder()); } @Override public ByteBuffer getBuffer() { throwISEIfImageIsInvalid(); return mBuffer; } @Override public int getPixelStride() { SurfaceImage.this.throwISEIfImageIsInvalid(); if (ImageReader.this.mFormat == ImageFormat.RAW_PRIVATE) { throw new UnsupportedOperationException( "getPixelStride is not supported for RAW_PRIVATE plane"); } return mPixelStride; } @Override public int getRowStride() { SurfaceImage.this.throwISEIfImageIsInvalid(); if (ImageReader.this.mFormat == ImageFormat.RAW_PRIVATE) { throw new UnsupportedOperationException( "getRowStride is not supported for RAW_PRIVATE plane"); } return mRowStride; } private void clearBuffer() { // Need null check first, as the getBuffer() may not be called before an image // is closed. if (mBuffer == null) { return; } if (mBuffer.isDirect()) { NioUtils.freeDirectBuffer(mBuffer); } mBuffer = null; } final private int mPixelStride; final private int mRowStride; private ByteBuffer mBuffer; } /** * This field is used to keep track of native object and used by native code only. * Don't modify. */ private long mNativeBuffer; /** * These fields are set by native code during nativeImageSetup(). */ private long mTimestamp; private int mTransform; private int mScalingMode; private SurfacePlane[] mPlanes; private int mFormat = ImageFormat.UNKNOWN; // If this image is detached from the ImageReader. private AtomicBoolean mIsDetached = new AtomicBoolean(false); private synchronized native SurfacePlane[] nativeCreatePlanes(int numPlanes, int readerFormat, long readerUsage); private synchronized native int nativeGetWidth(); private synchronized native int nativeGetHeight(); private synchronized native int nativeGetFormat(int readerFormat); private synchronized native int nativeGetFenceFd(); private synchronized native HardwareBuffer nativeGetHardwareBuffer(); } private synchronized native void nativeInit(Object weakSelf, int w, int h, int fmt, int maxImgs, long consumerUsage); private synchronized native void nativeClose(); private synchronized native void nativeReleaseImage(Image i); private synchronized native Surface nativeGetSurface(); private synchronized native int nativeDetachImage(Image i); private synchronized native void nativeDiscardFreeBuffers(); /** * @return A return code {@code ACQUIRE_*} * * @see #ACQUIRE_SUCCESS * @see #ACQUIRE_NO_BUFS * @see #ACQUIRE_MAX_IMAGES */ private synchronized native int nativeImageSetup(Image i); /** * @hide */ public static class ImagePlane extends android.media.Image.Plane { private ImagePlane(int rowStride, int pixelStride, ByteBuffer buffer) { mRowStride = rowStride; mPixelStride = pixelStride; mBuffer = buffer; /** * Set the byteBuffer order according to host endianness (native * order), otherwise, the byteBuffer order defaults to * ByteOrder.BIG_ENDIAN. */ mBuffer.order(ByteOrder.nativeOrder()); } @Override public ByteBuffer getBuffer() { return mBuffer; } @Override public int getPixelStride() { return mPixelStride; } @Override public int getRowStride() { return mRowStride; } final private int mPixelStride; final private int mRowStride; private ByteBuffer mBuffer; } /** * @hide */ public static ImagePlane[] initializeImagePlanes(int numPlanes, GraphicBuffer buffer, int fenceFd, int format, long timestamp, int transform, int scalingMode, Rect crop) { return nativeCreateImagePlanes(numPlanes, buffer, fenceFd, format, crop.left, crop.top, crop.right, crop.bottom); } private synchronized static native ImagePlane[] nativeCreateImagePlanes(int numPlanes, GraphicBuffer buffer, int fenceFd, int format, int cropLeft, int cropTop, int cropRight, int cropBottom); /** * @hide */ public static void unlockGraphicBuffer(GraphicBuffer buffer) { nativeUnlockGraphicBuffer(buffer); } private synchronized static native void nativeUnlockGraphicBuffer(GraphicBuffer buffer); /** * We use a class initializer to allow the native code to cache some * field offsets. */ private static native void nativeClassInit(); static { System.loadLibrary("media_jni"); nativeClassInit(); } }




© 2015 - 2024 Weber Informatics LLC | Privacy Policy