Fix camera frame conversion and native preview sizing (#5955)

* Fix camera frame conversion and native preview layout

* Add required license headers to camera regression fixtures

Author: shai-almog

CodenameOne/src/com/codename1/ai/vision/VisionCameraView.java

@@ -230,7 +230,7 @@ public void start() {
                         "No camera is available on this device");
             }
             opened = Camera.open(info, new CameraSessionOptions()
-                    .frameFormat(FrameFormat.JPEG)
+                    .frameFormat(FrameFormat.NV21)
                     .frameMaxFps(maxFps)
                     .captureAudio(false));
         } catch (Throwable error) {

CodenameOne/src/com/codename1/camera/CameraFrame.java

@@ -29,20 +29,39 @@
 /// framework may reuse or release the underlying buffers. Clone any data you
 /// need to keep.
 public final class CameraFrame {
-    private final byte[] jpegBytes;
+    private byte[] jpegBytes;
+    private JpegEncoder jpegEncoder;
     private final byte[] rawBytes;
     private final int width;
     private final int height;
     private final int rotationDegrees;
     private final long timestampNanos;
     private final FrameFormat format;
 
+    /// Encodes a frame on demand. Implementations must own the input bytes;
+    /// they must not retain a native image that is closed after frame delivery.
+    public interface JpegEncoder {
+        byte[] encode();
+    }
+
     /// Used by platform implementations.
     public CameraFrame(byte[] jpegBytes, byte[] rawBytes,
                        int width, int height,
                        int rotationDegrees, long timestampNanos,
                        FrameFormat format) {
+        this(jpegBytes, rawBytes, width, height, rotationDegrees,
+                timestampNanos, format, null);
+    }
+
+    /// Used by ports that can supply raw frames without eagerly encoding JPEG.
+    /// The encoder is invoked at most once after a successful encoding, when
+    /// `#getJpegBytes()` is first called. Existing eager callers need no encoder.
+    public CameraFrame(byte[] jpegBytes, byte[] rawBytes,
+                       int width, int height, int rotationDegrees,
+                       long timestampNanos, FrameFormat format,
+                       JpegEncoder jpegEncoder) {
         this.jpegBytes = jpegBytes;
+        this.jpegEncoder = jpegEncoder;
         this.rawBytes = rawBytes;
         this.width = width;
         this.height = height;
@@ -54,8 +73,17 @@ public CameraFrame(byte[] jpegBytes, byte[] rawBytes,
     /// JPEG-encoded bytes for this frame. Always non-null regardless of the
     /// requested `FrameFormat`; use this when encoded image data is needed.
     /// `VisionImage#fromCameraFrame(CameraFrame)` instead selects the raw
-    /// buffer for NV21 and RGBA8888 frames.
-    public byte[] getJpegBytes() {
+    /// buffer for NV21 and RGBA8888 frames. A raw frame may encode its JPEG
+    /// lazily on the first call; subsequent calls return the cached bytes.
+    public synchronized byte[] getJpegBytes() {
+        if (jpegBytes == null && jpegEncoder != null) {
+            byte[] encoded = jpegEncoder.encode();
+            if (encoded == null) {
+                throw new IllegalStateException("Camera frame JPEG encoding failed");
+            }
+            jpegBytes = encoded;
+            jpegEncoder = null;
+        }
         return jpegBytes;
     }
 

Ports/Android/src/com/codename1/impl/android/AndroidCameraFrameConverter.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Codename One designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Codename One through http://www.codenameone.com/ if you
+ * need additional information or have any questions.
+ */
+package com.codename1.impl.android;
+
+import java.nio.ByteBuffer;
+
+/** Packs camera YUV_420_888 planes without assuming a device-specific layout. */
+final class AndroidCameraFrameConverter {
+    private AndroidCameraFrameConverter() {
+    }
+
+    static byte[] toNV21(int width, int height,
+                         ByteBuffer y, int yRowStride, int yPixelStride,
+                         ByteBuffer u, int uRowStride, int uPixelStride,
+                         ByteBuffer v, int vRowStride, int vPixelStride) {
+        long pixels = (long) width * height;
+        if (width <= 0 || height <= 0 || (width & 1) != 0 || (height & 1) != 0
+                || pixels + pixels / 2 > Integer.MAX_VALUE) {
+            throw new IllegalArgumentException("Invalid YUV dimensions");
+        }
+        byte[] result = new byte[(int) (pixels + pixels / 2)];
+        copyPlane(y, yRowStride, yPixelStride, width, height, result, 0, 1);
+        // NV21 interleaves V then U. Planes can be planar, overlapping, padded,
+        // sliced, or read-only; absolute reads preserve their positions.
+        copyPlane(v, vRowStride, vPixelStride, width / 2, height / 2,
+                result, (int) pixels, 2);
+        copyPlane(u, uRowStride, uPixelStride, width / 2, height / 2,
+                result, (int) pixels + 1, 2);
+        return result;
+    }
+
+    private static void copyPlane(ByteBuffer plane, int rowStride, int pixelStride,
+                                  int width, int height, byte[] out,
+                                  int offset, int step) {
+        long rowBytes = (long) (width - 1) * pixelStride + 1;
+        long end = (long) plane.position() + (long) (height - 1) * rowStride + rowBytes;
+        if (rowStride <= 0 || pixelStride <= 0 || rowStride < rowBytes
+                || end > plane.limit()) {
+            throw new IllegalArgumentException("Truncated or invalid YUV plane");
+        }
+        int base = plane.position();
+        for (int row = 0; row < height; row++) {
+            int input = base + row * rowStride;
+            for (int col = 0; col < width; col++) {
+                out[offset] = plane.get(input + col * pixelStride);
+                offset += step;
+            }
+        }
+    }
+}

Ports/Android/src/com/codename1/impl/android/AndroidCameraImpl.java

@@ -120,6 +120,8 @@ public class AndroidCameraImpl extends CameraImpl {
     private final AtomicBoolean listenerBusy = new AtomicBoolean();
     private volatile FrameListener frameListener;
     private volatile FrameFormat frameFormat = FrameFormat.JPEG;
+    private volatile long frameIntervalNanos;
+    private long lastFrameNanos;
     private CameraInfo info;
     private CameraSessionOptions options;
     private ExecutorService cameraExecutor;
@@ -422,8 +424,9 @@ public void stopVideoRecording(AsyncResource<String> result) {
 
     @Override
     public void setFrameListener(FrameListener listener, FrameFormat format, int maxFps) {
-        this.frameListener = listener;
         this.frameFormat = format == null ? FrameFormat.JPEG : format;
+        this.frameIntervalNanos = maxFps > 0 ? 1000000000L / maxFps : 0;
+        this.frameListener = listener;
     }
 
     private void installFrameAnalyzer() {
@@ -459,25 +462,23 @@ private void onImageProxy(Object imageProxy) {
             return;
         }
         try {
-            // Convert ImageProxy -> Image -> JPEG bytes
-            byte[] jpeg = null;
-            int w = 0, h = 0;
-            int rotation = 0;
+            long now = System.nanoTime();
+            if (lastFrameNanos != 0 && now - lastFrameNanos < frameIntervalNanos) {
+                return;
+            }
+            CameraFrame frame;
             try {
                 Image img = (Image) imageProxy.getClass().getMethod("getImage").invoke(imageProxy);
                 Object imageInfo = imageProxy.getClass().getMethod("getImageInfo").invoke(imageProxy);
-                rotation = (Integer) imageInfo.getClass().getMethod("getRotationDegrees").invoke(imageInfo);
-                if (img != null) {
-                    w = img.getWidth();
-                    h = img.getHeight();
-                    jpeg = yuvImageToJpeg(img);
-                }
+                int rotation = (Integer) imageInfo.getClass().getMethod("getRotationDegrees").invoke(imageInfo);
+                frame = img == null ? null : createFrame(img, rotation, now, frameFormat);
             } catch (Throwable t) {
                 Log.w(TAG, "Frame conversion failed: " + t.getMessage());
+                return;
             }
-            if (jpeg != null) {
-                l.onFrame(new CameraFrame(jpeg, null, w, h, rotation,
-                        System.nanoTime(), frameFormat));
+            if (frame != null) {
+                lastFrameNanos = now;
+                l.onFrame(frame);
             }
         } finally {
             listenerBusy.set(false);
@@ -491,31 +492,39 @@ private static void closeImageProxy(Object imageProxy) {
         } catch (Throwable ignored) { }
     }
 
-    private static byte[] yuvImageToJpeg(Image image) {
+    private static CameraFrame createFrame(Image image, int rotation, long timestamp,
+                                           FrameFormat requested) {
+        final int w = image.getWidth();
+        final int h = image.getHeight();
         if (image.getFormat() == ImageFormat.JPEG) {
-            ByteBuffer buf = image.getPlanes()[0].getBuffer();
-            byte[] out = new byte[buf.remaining()];
-            buf.get(out);
-            return out;
+            ByteBuffer buf = image.getPlanes()[0].getBuffer().duplicate();
+            byte[] jpeg = new byte[buf.remaining()];
+            buf.get(jpeg);
+            return new CameraFrame(jpeg, null, w, h, rotation, timestamp, requested);
+        }
+        if (image.getFormat() != ImageFormat.YUV_420_888) {
+            throw new IllegalArgumentException("Unsupported camera image format");
         }
-        // YUV_420_888 -> NV21 -> JPEG via YuvImage
-        int w = image.getWidth();
-        int h = image.getHeight();
         Image.Plane[] planes = image.getPlanes();
-        ByteBuffer y = planes[0].getBuffer();
-        ByteBuffer u = planes[1].getBuffer();
-        ByteBuffer v = planes[2].getBuffer();
-        int ySize = y.remaining();
-        int uSize = u.remaining();
-        int vSize = v.remaining();
-        byte[] nv21 = new byte[ySize + uSize + vSize];
-        y.get(nv21, 0, ySize);
-        v.get(nv21, ySize, vSize);
-        u.get(nv21, ySize + vSize, uSize);
-        YuvImage yuv = new YuvImage(nv21, ImageFormat.NV21, w, h, null);
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        yuv.compressToJpeg(new Rect(0, 0, w, h), 80, baos);
-        return baos.toByteArray();
+        final byte[] nv21 = AndroidCameraFrameConverter.toNV21(w, h,
+                planes[0].getBuffer(), planes[0].getRowStride(), planes[0].getPixelStride(),
+                planes[1].getBuffer(), planes[1].getRowStride(), planes[1].getPixelStride(),
+                planes[2].getBuffer(), planes[2].getRowStride(), planes[2].getPixelStride());
+        CameraFrame.JpegEncoder encoder = new CameraFrame.JpegEncoder() {
+            public byte[] encode() {
+                YuvImage yuv = new YuvImage(nv21, ImageFormat.NV21, w, h, null);
+                ByteArrayOutputStream baos = new ByteArrayOutputStream();
+                if (!yuv.compressToJpeg(new Rect(0, 0, w, h), 80, baos)) {
+                    throw new IllegalStateException("Could not encode camera frame");
+                }
+                return baos.toByteArray();
+            }
+        };
+        if (requested == FrameFormat.NV21) {
+            return new CameraFrame(null, nv21, w, h, rotation, timestamp, requested, encoder);
+        }
+        // JPEG remains the fallback for raw formats this port cannot expose.
+        return new CameraFrame(encoder.encode(), null, w, h, rotation, timestamp, requested);
     }
 
     @Override

Ports/iOSPort/nativeSources/CN1Camera.m

@@ -27,11 +27,42 @@
 #if defined(INCLUDE_CN1_CAMERA) && !TARGET_OS_TV
 #import "java_lang_String.h"
 #import "com_codename1_impl_ios_IOSCameraImpl.h"
+#import <QuartzCore/QuartzCore.h>
 
 // Defined in IOSNative.m; converts an NSData into a Java byte[]. It pulls the
 // thread state via getThreadLocalData() internally, so it takes no thread arg.
 extern JAVA_OBJECT nsDataToByteArr(NSData *data);
 
+// The Java peer resizes its native view after creation and on every layout.
+// CALayer sublayers do not automatically follow that view's bounds on iOS.
+@interface CN1CameraPreviewView : CN1View
+@end
+
+@implementation CN1CameraPreviewView
+- (void)layoutCameraLayer {
+    [CATransaction begin];
+    [CATransaction setDisableActions:YES];
+    for (CALayer *layer in self.layer.sublayers) {
+        if ([layer isKindOfClass:[AVCaptureVideoPreviewLayer class]]) {
+            layer.frame = self.bounds;
+        }
+    }
+    [CATransaction commit];
+}
+
+#if TARGET_OS_OSX
+- (void)layout {
+    [super layout];
+    [self layoutCameraLayer];
+}
+#else
+- (void)layoutSubviews {
+    [super layoutSubviews];
+    [self layoutCameraLayer];
+}
+#endif
+@end
+
 @interface CN1Camera ()
 @property (nonatomic, copy) NSString *pendingPhotoFilePath;
 @property (nonatomic, assign) int pendingPhotoCallbackId;
@@ -196,7 +227,7 @@ - (BOOL)openWithCameraId:(NSString *)cameraId
 
 - (CN1View *)createPreviewView {
     if (self.previewView) return self.previewView;
-    CN1View *v = [[CN1View alloc] initWithFrame:CGRectMake(0, 0, 320, 480)];
+    CN1View *v = [[CN1CameraPreviewView alloc] initWithFrame:CGRectMake(0, 0, 320, 480)];
 #ifndef CN1_USE_ARC
     // Autoreleased so the previewView property below is the only owner. The
     // alloc's +1 was never balanced here either: this method does not begin

maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionCameraViewTest.java

@@ -103,6 +103,22 @@ void theScannerNeverAsksForTheMicrophone() {
         assertFalse(camera.captureAudio);
     }
 
+    @Test
+    void liveAnalysisRequestsRawFramesWithoutForcingJpegEncoding() {
+        view = new VisionCameraView<Barcode[]>(new BarcodeScanner());
+        view.start();
+        assertEquals(com.codename1.camera.FrameFormat.NV21,
+                view.getSession().getOptions().getFrameFormat());
+        final int[] encodings = {0};
+        byte[] raw = new byte[6];
+        camera.frameListener.onFrame(new com.codename1.camera.CameraFrame(
+                null, raw, 2, 2, 90, 1, com.codename1.camera.FrameFormat.NV21,
+                () -> { encodings[0]++; return new byte[] {1}; }));
+        flushSerialCalls();
+        assertEquals(1, vision.analyzeCount);
+        assertEquals(0, encodings[0]);
+    }
+
     @Test
     void facingSelectsTheCamera() {
         view = new VisionCameraView<Barcode[]>(new BarcodeScanner());