Merge "Use java Jpeg encoding instead of native codes / fix some issues." into main am: a0df61244f am: 112a2cec91 am: 0ca00b82c1 am: 9d5ec75a29

Original change: https://android-review.googlesource.com/c/platform/frameworks/ex/+/2462898

Change-Id: Idbf2cfcf5e55799d5c7085eb22fbab3a8ae8be27
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

6 files changed, 152 insertions, 124 deletions

diff --git a/camera2/extensions/advancedSample/Android.bp b/camera2/extensions/advancedSample/Android.bp
index 9f0df8e4..8efae6ff 100644
--- a/camera2/extensions/advancedSample/Android.bp
+++ b/camera2/extensions/advancedSample/Android.bp
@@ -20,7 +20,8 @@ java_library {
     name: "androidx.camera.extensions.impl.advanced",
     installable: true,
     static_libs: [
-        "androidx.annotation_annotation"
+        "androidx.annotation_annotation",
+        "androidx.exifinterface_exifinterface-nodeps"
     ],
     exclude_kotlinc_generated_files: true,
     srcs: ["src/**/*.java"],
diff --git a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BaseAdvancedExtenderImpl.java b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BaseAdvancedExtenderImpl.java
index 00b57058..4a77d6ae 100644
--- a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BaseAdvancedExtenderImpl.java
+++ b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BaseAdvancedExtenderImpl.java
@@ -16,9 +16,7 @@
 
 package androidx.camera.extensions.impl.advanced;
 
-import androidx.camera.extensions.impl.advanced.JpegEncoder;
-
-import static  androidx.camera.extensions.impl.advanced.JpegEncoder.JPEG_DEFAULT_QUALITY;
+import static androidx.camera.extensions.impl.advanced.JpegEncoder.JPEG_DEFAULT_QUALITY;
 import static androidx.camera.extensions.impl.advanced.JpegEncoder.JPEG_DEFAULT_ROTATION;
 
 import android.annotation.SuppressLint;
@@ -32,41 +30,30 @@ import android.hardware.camera2.CaptureResult;
 import android.hardware.camera2.TotalCaptureResult;
 import android.hardware.camera2.params.StreamConfigurationMap;
 import android.media.Image;
-import android.media.Image.Plane;
 import android.media.ImageWriter;
-import android.os.Build;
 import android.util.Log;
 import android.util.Pair;
 import android.util.Range;
 import android.util.Size;
-import android.view.Surface;
 
 import androidx.annotation.GuardedBy;
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
+import androidx.camera.extensions.impl.advanced.JpegEncoder;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.Executor;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 @SuppressLint("UnknownNullness")
 public abstract class BaseAdvancedExtenderImpl implements AdvancedExtenderImpl {
-
-    static {
-        try {
-            System.loadLibrary("encoderjpeg_jni");
-        } catch (UnsatisfiedLinkError e) {
-            Log.e("BaseAdvancedExtenderImpl", "libencoderjpeg_jni not loaded");
-        }
-    }
-
+    private final static int JPEG_APP_SEGMENT_SIZE = 64 * 1024;
     protected CameraCharacteristics mCameraCharacteristics;
 
     public BaseAdvancedExtenderImpl() {
@@ -232,6 +219,10 @@ public abstract class BaseAdvancedExtenderImpl implements AdvancedExtenderImpl {
             // default empty implementation
         }
 
+        protected void addRepeatingRequestParameters(RequestBuilder builder) {
+            // default empty implementation
+        }
+
         @Override
         public void deInitSession() {
             synchronized (mLockCaptureSurfaceImageWriter) {
@@ -282,6 +273,7 @@ public abstract class BaseAdvancedExtenderImpl implements AdvancedExtenderImpl {
             RequestBuilder builder = new RequestBuilder(mPreviewOutputConfig.getId(),
                     CameraDevice.TEMPLATE_PREVIEW, 0);
             addTriggerRequestKeys(builder, triggers);
+            addRepeatingRequestParameters(builder);
 
             final int seqId = mNextCaptureSequenceId.getAndIncrement();
 
@@ -348,8 +340,9 @@ public abstract class BaseAdvancedExtenderImpl implements AdvancedExtenderImpl {
                                 .setMaxImages(MAX_NUM_IMAGES)
                                 // For JPEG format, width x height should be set to (w*h) x 1
                                 // since the JPEG image is returned as a 1D byte array
-                                .setWidthAndHeight(mCaptureOutputSurfaceConfig.getSize().getWidth()
-                                        * mCaptureOutputSurfaceConfig.getSize().getHeight(), 1)
+                                .setWidthAndHeight((mCaptureOutputSurfaceConfig.getSize().getWidth()
+                                        * mCaptureOutputSurfaceConfig.getSize().getHeight() * 3)/2
+                                        + JPEG_APP_SEGMENT_SIZE, 1)
                                 .build();
                     } else {
                         mCaptureSurfaceImageWriter = new ImageWriter
@@ -376,6 +369,7 @@ public abstract class BaseAdvancedExtenderImpl implements AdvancedExtenderImpl {
             RequestBuilder builder = new RequestBuilder(mPreviewOutputConfig.getId(),
                     CameraDevice.TEMPLATE_PREVIEW, 0);
             applyParameters(builder);
+            addRepeatingRequestParameters(builder);
             final int seqId = mNextCaptureSequenceId.getAndIncrement();
 
             RequestProcessorImpl.Callback callback = new RequestProcessorImpl.Callback() {
@@ -394,9 +388,8 @@ public abstract class BaseAdvancedExtenderImpl implements AdvancedExtenderImpl {
                 @Override
                 public void onCaptureCompleted(RequestProcessorImpl.Request request,
                         TotalCaptureResult totalCaptureResult) {
-                    addCaptureResultKeys(seqId, totalCaptureResult, captureCallback);
-
                     captureCallback.onCaptureProcessStarted(seqId);
+                    addCaptureResultKeys(seqId, totalCaptureResult, captureCallback);
                 }
 
                 @Override
diff --git a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BeautyAdvancedExtenderImpl.java b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BeautyAdvancedExtenderImpl.java
index d8ff59f9..0223c4ba 100644
--- a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BeautyAdvancedExtenderImpl.java
+++ b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BeautyAdvancedExtenderImpl.java
@@ -69,16 +69,16 @@ public class BeautyAdvancedExtenderImpl extends BaseAdvancedExtenderImpl {
         }
 
         @Override
-        protected void addSessionParameter(Camera2SessionConfigImplBuilder builder) {
-            builder.addSessionParameter(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_TWILIGHT);
+        protected void addRepeatingRequestParameters(RequestBuilder builder) {
+            builder.setParameters(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_TWILIGHT);
         }
 
         @Override
         protected void addCaptureRequestParameters(List<RequestProcessorImpl.Request> requestList) {
             RequestBuilder build = new RequestBuilder(mCaptureOutputConfig.getId(),
                     CameraDevice.TEMPLATE_STILL_CAPTURE, DEFAULT_CAPTURE_ID);
-            build.setParameters(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_TWILIGHT);
             applyParameters(build);
+            build.setParameters(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_TWILIGHT);
 
             requestList.add(build.build());
         }
diff --git a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BokehAdvancedExtenderImpl.java b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BokehAdvancedExtenderImpl.java
index 01569911..7002a1c8 100644
--- a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BokehAdvancedExtenderImpl.java
+++ b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/BokehAdvancedExtenderImpl.java
@@ -75,17 +75,16 @@ public class BokehAdvancedExtenderImpl extends BaseAdvancedExtenderImpl {
         }
 
         @Override
-        protected void addSessionParameter(Camera2SessionConfigImplBuilder builder) {
-            builder.addSessionParameter(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_SHADE);
+        protected void addRepeatingRequestParameters(RequestBuilder builder) {
+            builder.setParameters(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_SHADE);
         }
 
         @Override
         protected void addCaptureRequestParameters(List<RequestProcessorImpl.Request> requestList) {
             RequestBuilder build = new RequestBuilder(mCaptureOutputConfig.getId(),
                     CameraDevice.TEMPLATE_STILL_CAPTURE, DEFAULT_CAPTURE_ID);
-            build.setParameters(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_SHADE);
             applyParameters(build);
-
+            build.setParameters(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_SHADE);
             requestList.add(build.build());
         }
     }
diff --git a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/JpegEncoder.java b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/JpegEncoder.java
index f3eb7eab..5ea7e94c 100644
--- a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/JpegEncoder.java
+++ b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/JpegEncoder.java
@@ -16,121 +16,159 @@
 
 package androidx.camera.extensions.impl.advanced;
 
-import android.annotation.SuppressLint;
-import android.content.Context;
 import android.graphics.ImageFormat;
+import android.graphics.Rect;
+import android.graphics.YuvImage;
 import android.media.Image;
-import android.media.Image.Plane;
-import android.media.ImageWriter;
 import android.util.Log;
 
+import androidx.annotation.NonNull;
+
+import androidx.exifinterface.media.ExifInterface;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 // Jpeg compress input YUV and queue back in the client target surface.
 public class JpegEncoder {
-
+    private final static String TAG = "JpegEncoder";
     public final static int JPEG_DEFAULT_QUALITY = 100;
     public final static int JPEG_DEFAULT_ROTATION = 0;
     public static final int HAL_PIXEL_FORMAT_BLOB = 0x21;
 
-    /**
-     * Compresses a YCbCr image to jpeg, applying a crop and rotation.
-     * <p>
-     * The input is defined as a set of 3 planes of 8-bit samples, one plane for
-     * each channel of Y, Cb, Cr.<br>
-     * The Y plane is assumed to have the same width and height of the entire
-     * image.<br>
-     * The Cb and Cr planes are assumed to be downsampled by a factor of 2, to
-     * have dimensions (floor(width / 2), floor(height / 2)).<br>
-     * Each plane is specified by a direct java.nio.ByteBuffer, a pixel-stride,
-     * and a row-stride. So, the sample at coordinate (x, y) can be retrieved
-     * from byteBuffer[x * pixel_stride + y * row_stride].
-     * <p>
-     * The pre-compression transformation is applied as follows:
-     * <ol>
-     * <li>The image is cropped to the rectangle from (cropLeft, cropTop) to
-     * (cropRight - 1, cropBottom - 1). So, a cropping-rectangle of (0, 0) -
-     * (width, height) is a no-op.</li>
-     * <li>The rotation is applied counter-clockwise relative to the coordinate
-     * space of the image, so a CCW rotation will appear CW when the image is
-     * rendered in scanline order. Only rotations which are multiples of
-     * 90-degrees are suppored, so the parameter 'rot90' specifies which
-     * multiple of 90 to rotate the image.</li>
-     * </ol>
-     *
-     * @param width          the width of the image to compress
-     * @param height         the height of the image to compress
-     * @param yBuf           the buffer containing the Y component of the image
-     * @param yPStride       the stride between adjacent pixels in the same row in
-     *                       yBuf
-     * @param yRStride       the stride between adjacent rows in yBuf
-     * @param cbBuf          the buffer containing the Cb component of the image
-     * @param cbPStride      the stride between adjacent pixels in the same row in
-     *                       cbBuf
-     * @param cbRStride      the stride between adjacent rows in cbBuf
-     * @param crBuf          the buffer containing the Cr component of the image
-     * @param crPStride      the stride between adjacent pixels in the same row in
-     *                       crBuf
-     * @param crRStride      the stride between adjacent rows in crBuf
-     * @param outBuf         a direct java.nio.ByteBuffer to hold the compressed jpeg.
-     *                       This must have enough capacity to store the result, or an
-     *                       error code will be returned.
-     * @param outBufCapacity the capacity of outBuf
-     * @param quality        the jpeg-quality (1-100) to use
-     * @param cropLeft       left-edge of the bounds of the image to crop to before
-     *                       rotation
-     * @param cropTop        top-edge of the bounds of the image to crop to before
-     *                       rotation
-     * @param cropRight      right-edge of the bounds of the image to crop to before
-     *                       rotation
-     * @param cropBottom     bottom-edge of the bounds of the image to crop to
-     *                       before rotation
-     * @param rot90          the multiple of 90 to rotate the image CCW (after cropping)
-     */
-    public static native int compressJpegFromYUV420pNative(
-            int width, int height,
-            ByteBuffer yBuf, int yPStride, int yRStride,
-            ByteBuffer cbBuf, int cbPStride, int cbRStride,
-            ByteBuffer crBuf, int crPStride, int crRStride,
-            ByteBuffer outBuf, int outBufCapacity,
-            int quality,
-            int cropLeft, int cropTop, int cropRight, int cropBottom,
-            int rot90);
-
-    public static void encodeToJpeg(Image yuvImage, Image jpegImage,
+    public static void encodeToJpeg(Image yuvInputImage, Image jpegImage,
             int jpegOrientation, int jpegQuality) {
 
-        jpegOrientation =  (360 - (jpegOrientation % 360)) / 90;
-        ByteBuffer jpegBuffer = jpegImage.getPlanes()[0].getBuffer();
+        byte[] yuvBytes = yuv_420_888toNv21(yuvInputImage);
+        YuvImage yuvImage = new YuvImage(yuvBytes, ImageFormat.NV21, yuvInputImage.getWidth(),
+                yuvInputImage.getHeight(), null);
+        File file = null;
+        try {
+            // Encode YUV to JPEG and save as a teamp file.
+            file = File.createTempFile("ExtensionsTemp", ".jpg");
+            FileOutputStream fileOutputStream = new FileOutputStream(file);
+            Rect imageRect = new Rect(0, 0, yuvInputImage.getWidth(), yuvInputImage.getHeight());
+            yuvImage.compressToJpeg(imageRect, jpegQuality, fileOutputStream);
+            fileOutputStream.close();
 
-        jpegBuffer.clear();
+            // Update orientation EXIF on this file.
+            writeOrientationExif(file, jpegOrientation);
 
-        int jpegCapacity = jpegImage.getWidth();
+            // Read the JPEG data into JPEG Image byte buffer.
+            ByteBuffer jpegBuf = jpegImage.getPlanes()[0].getBuffer();
+            readFileToByteBuffer(file, jpegBuf);
 
-        Plane lumaPlane = yuvImage.getPlanes()[0];
+            // Set limits on jpeg buffer and rewind
+            jpegBuf.limit(jpegBuf.position());
+            jpegBuf.rewind();
+        } catch (IOException e) {
+            Log.e(TAG, "Failed to encode the YUV data into a JPEG image", e);
+        } finally {
+            if (file != null) {
+                file.delete();
+            }
+        }
+    }
 
-        Plane crPlane = yuvImage.getPlanes()[1];
-        Plane cbPlane = yuvImage.getPlanes()[2];
+    private static void writeOrientationExif(File file, int jpegOrientation) throws IOException {
+        ExifInterface exifInterface = new ExifInterface(file);
+        int orientationEnum = ExifInterface.ORIENTATION_NORMAL;
 
-        JpegEncoder.compressJpegFromYUV420pNative(
-            yuvImage.getWidth(), yuvImage.getHeight(),
-            lumaPlane.getBuffer(), lumaPlane.getPixelStride(), lumaPlane.getRowStride(),
-            crPlane.getBuffer(), crPlane.getPixelStride(), crPlane.getRowStride(),
-            cbPlane.getBuffer(), cbPlane.getPixelStride(), cbPlane.getRowStride(),
-            jpegBuffer, jpegCapacity, jpegQuality,
-            0, 0, yuvImage.getWidth(), yuvImage.getHeight(),
-            jpegOrientation);
+        switch (jpegOrientation) {
+            case 0:
+                orientationEnum = ExifInterface.ORIENTATION_NORMAL;
+                break;
+            case 90:
+                orientationEnum = ExifInterface.ORIENTATION_ROTATE_90;
+                break;
+            case 180:
+                orientationEnum = ExifInterface.ORIENTATION_ROTATE_180;
+                break;
+            case 270:
+                orientationEnum = ExifInterface.ORIENTATION_ROTATE_270;
+                break;
+            default:
+                Log.e(TAG, "Invalid jpeg orientation:" + jpegOrientation);
+                break;
+        }
+        exifInterface.setAttribute(ExifInterface.TAG_ORIENTATION, String.valueOf(orientationEnum));
+        exifInterface.saveAttributes();
     }
 
+    private static void readFileToByteBuffer(File file, ByteBuffer byteBuffer) {
+        try (FileInputStream fis = new FileInputStream(file)) {
+            byte[] buffer = new byte[1024];
+            int bytesRead;
+            while ((bytesRead = fis.read(buffer)) != -1) {
+                byteBuffer.put(buffer, 0, bytesRead);
+            }
+        } catch (IOException e) {
+            Log.e(TAG, "failed to read the file into the byte buffer", e);
+        }
+    }
     public static int imageFormatToPublic(int format) {
         switch (format) {
             case HAL_PIXEL_FORMAT_BLOB:
                 return ImageFormat.JPEG;
-            case ImageFormat.JPEG:
-                throw new IllegalArgumentException(
-                        "ImageFormat.JPEG is an unknown internal format");
             default:
                 return format;
         }
     }
-}
\ No newline at end of file
+
+    @NonNull
+    private static byte[] yuv_420_888toNv21(@NonNull Image image) {
+        Image.Plane yPlane = image.getPlanes()[0];
+        Image.Plane uPlane = image.getPlanes()[1];
+        Image.Plane vPlane = image.getPlanes()[2];
+
+        ByteBuffer yBuffer = yPlane.getBuffer();
+        ByteBuffer uBuffer = uPlane.getBuffer();
+        ByteBuffer vBuffer = vPlane.getBuffer();
+        yBuffer.rewind();
+        uBuffer.rewind();
+        vBuffer.rewind();
+
+        int ySize = yBuffer.remaining();
+
+        int position = 0;
+        // TODO(b/115743986): Pull these bytes from a pool instead of allocating for every image.
+        byte[] nv21 = new byte[ySize + (image.getWidth() * image.getHeight() / 2)];
+
+        // Add the full y buffer to the array. If rowStride > 1, some padding may be skipped.
+        for (int row = 0; row < image.getHeight(); row++) {
+            yBuffer.get(nv21, position, image.getWidth());
+            position += image.getWidth();
+            yBuffer.position(
+                    Math.min(ySize, yBuffer.position() - image.getWidth() + yPlane.getRowStride()));
+        }
+
+        int chromaHeight = image.getHeight() / 2;
+        int chromaWidth = image.getWidth() / 2;
+        int vRowStride = vPlane.getRowStride();
+        int uRowStride = uPlane.getRowStride();
+        int vPixelStride = vPlane.getPixelStride();
+        int uPixelStride = uPlane.getPixelStride();
+
+        // Interleave the u and v frames, filling up the rest of the buffer. Use two line buffers to
+        // perform faster bulk gets from the byte buffers.
+        byte[] vLineBuffer = new byte[vRowStride];
+        byte[] uLineBuffer = new byte[uRowStride];
+        for (int row = 0; row < chromaHeight; row++) {
+            vBuffer.get(vLineBuffer, 0, Math.min(vRowStride, vBuffer.remaining()));
+            uBuffer.get(uLineBuffer, 0, Math.min(uRowStride, uBuffer.remaining()));
+            int vLineBufferPosition = 0;
+            int uLineBufferPosition = 0;
+            for (int col = 0; col < chromaWidth; col++) {
+                nv21[position++] = vLineBuffer[vLineBufferPosition];
+                nv21[position++] = uLineBuffer[uLineBufferPosition];
+                vLineBufferPosition += vPixelStride;
+                uLineBufferPosition += uPixelStride;
+            }
+        }
+
+        return nv21;
+    }
+}
diff --git a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/NightAdvancedExtenderImpl.java b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/NightAdvancedExtenderImpl.java
index 82940635..9a9727b5 100644
--- a/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/NightAdvancedExtenderImpl.java
+++ b/camera2/extensions/advancedSample/src/java/androidx/camera/extensions/impl/advanced/NightAdvancedExtenderImpl.java
@@ -133,11 +133,6 @@ public class NightAdvancedExtenderImpl extends BaseAdvancedExtenderImpl {
         }
 
         @Override
-        protected void addSessionParameter(Camera2SessionConfigImplBuilder builder) {
-            builder.addSessionParameter(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_INCANDESCENT);
-        }
-
-        @Override
         public void deInitSession() {
             super.deInitSession();
 
@@ -208,6 +203,8 @@ public class NightAdvancedExtenderImpl extends BaseAdvancedExtenderImpl {
             RequestBuilder builder = new RequestBuilder(mPreviewOutputConfig.getId(),
                     CameraDevice.TEMPLATE_PREVIEW, 0);
             applyParameters(builder);
+            builder.setParameters(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_INCANDESCENT);
+
             final int seqId = mNextCaptureSequenceId.getAndIncrement();
 
             RequestProcessorImpl.Callback callback = new RequestProcessorImpl.Callback() {
@@ -267,8 +264,8 @@ public class NightAdvancedExtenderImpl extends BaseAdvancedExtenderImpl {
         protected void addCaptureRequestParameters(List<RequestProcessorImpl.Request> requestList) {
             RequestBuilder build = new RequestBuilder(mCaptureOutputConfig.getId(),
                     CameraDevice.TEMPLATE_STILL_CAPTURE, DEFAULT_CAPTURE_ID);
-            build.setParameters(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_INCANDESCENT);
             applyParameters(build);
+            build.setParameters(CaptureRequest.CONTROL_AWB_MODE, AWB_MODE_INCANDESCENT);
 
             requestList.add(build.build());
         }