android.media.image passing in FirebaseVisionImage.fromMediaImage gives java.nio.BufferUnderflowException

[mihirmodiofficial]

it.setAnalyzer(cameraExecutor, LuminosityAnalyzer { luma, image ->
getCardDetails(image,rotation)
})

private fun getCardDetails(imageProxy: ImageProxy, rotation: Int) {

    val mediaImage = imageProxy.image

    val planes = mediaImage!!.planes
    if (planes.size >= 3) {

        val image = mediaImage?.let { FirebaseVisionImage.fromMediaImage(it, getRotationConstant(rotation)) }
        //val image = mediaImage?.let { FirebaseVisionImage.fromByteArray(dataYUV, metadata) }
        val firebaseVisionTextDetector = FirebaseVision.getInstance().onDeviceTextRecognizer

        if (image != null) {
            firebaseVisionTextDetector.processImage(image)
                    .addOnSuccessListener {
                        val words = it.text.split("\n")
                        Toast.makeText(context, words.get(0), Toast.LENGTH_LONG).show()

                    }
                    .addOnFailureListener {
                        Toast.makeText(context, "Sorry, something went wrong!", Toast.LENGTH_SHORT).show()
                    }

        }
    }

}

private fun getRotationConstant(rotation: Int): Int {
return when (rotation) {
90 -> FirebaseVisionImageMetadata.ROTATION_90
180 -> FirebaseVisionImageMetadata.ROTATION_180
270 -> FirebaseVisionImageMetadata.ROTATION_270
else -> FirebaseVisionImageMetadata.ROTATION_0
}
}

// Error

Process: com.demo, PID: 24383
java.nio.BufferUnderflowException
at java.nio.DirectByteBuffer.get(DirectByteBuffer.java:240)
at com.google.android.gms.internal.firebase_ml.zzsc.zza(com.google.firebase:firebase-ml-vision@@24.0.3:42)
at com.google.firebase.ml.vision.common.FirebaseVisionImage.fromMediaImage(com.google.firebase:firebase-ml-vision@@24.0.3:20)
at com.demo.ui.gift_shop.fragment.ScanPayFragment.getCardDetails(ScanPayFragment.kt:529)
at com.demo.ui.gift_shop.fragment.ScanPayFragment.access$getCardDetails(ScanPayFragment.kt:54)
at com.demo.ui.gift_shop.fragment.ScanPayFragment$bindCameraUseCases$1$$special$$inlined$also$lambda$1.invoke(ScanPayFragment.kt:271)
at com.demo.ui.gift_shop.fragment.ScanPayFragment$bindCameraUseCases$1$$special$$inlined$also$lambda$1.invoke(ScanPayFragment.kt:54)
at com.demo.ui.gift_shop.fragment.ScanPayFragment$LuminosityAnalyzer.analyze(ScanPayFragment.kt:383)
at androidx.camera.core.ImageAnalysisAbstractAnalyzer.lambda$analyzeImage$0$ImageAnalysisAbstractAnalyzer(ImageAnalysisAbstractAnalyzer.java:87)
at androidx.camera.core.-$$Lambda$ImageAnalysisAbstractAnalyzer$V6Gtux2-h8spda8vf4wFnIZntYI.run(Unknown Source:8)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.lang.Thread.run(Thread.java:764)

[google-oss-bot]

I couldn't figure out how to label this issue, so I've labeled it for a human to triage. Hang tight.

[samtstern]

@mihirmodiofficial this sounds like an SDK issue not a sample issue (BufferUnderflowException from within MLKit) so I have transferred it here.

[cs-googler]

Hi, we have added this logic internally for future releases to make sure buffer position is reset to 0.

For now, can you call rewind() for each media.Image.getPlanes[i].getBuffer() to see if the issue can be solved?

[mihirmodiofficial]

Looking for future release but Please reply with sample code.

java.lang.IllegalArgumentException: Bad position 460800/460799
    at java.nio.Buffer.position(Buffer.java:259)
    at com.google.android.gms.internal.firebase_ml.zzsc.zza(com.google.firebase:firebase-ml-vision@@24.0.3:34)
    at com.google.firebase.ml.vision.common.FirebaseVisionImage.fromMediaImage(com.google.firebase:firebase-ml-vision@@24.0.3:20)
    at com.abc.ui.gift_shop.fragment.ScanPayFragment2.getCardDetails(ScanPayFragment2.kt:532)
    at com.abc.ui.gift_shop.fragment.ScanPayFragment2.access$getCardDetails(ScanPayFragment2.kt:60)
    at com.abc.ui.gift_shop.fragment.ScanPayFragment2$bindCameraUseCases$1$$special$$inlined$also$lambda$1.invoke(ScanPayFragment2.kt:305)
    at com.abc.ui.gift_shop.fragment.ScanPayFragment2$bindCameraUseCases$1$$special$$inlined$also$lambda$1.invoke(ScanPayFragment2.kt:60)
    at com.abc.ui.gift_shop.fragment.ScanPayFragment2$LuminosityAnalyzer.analyze(ScanPayFragment2.kt:403)
    at androidx.camera.core.ImageAnalysisAbstractAnalyzer.lambda$analyzeImage$0$ImageAnalysisAbstractAnalyzer(ImageAnalysisAbstractAnalyzer.java:87)
    at androidx.camera.core.-$$Lambda$ImageAnalysisAbstractAnalyzer$V6Gtux2-h8spda8vf4wFnIZntYI.run(Unknown Source:8)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
    at java.lang.Thread.run(Thread.java:764)

[cs-googler]

Try to call rewind() for each media.Image.getPlanes[i].getBuffer() in your code:

private fun getCardDetails(imageProxy: ImageProxy, rotation: Int) {

val mediaImage = imageProxy.image

val planes = mediaImage!!.planes
if (planes.size >= 3) {
   // Reset buffer position for each plane's buffer. 
   for (plane in planes) {
       plane.getBuffer().rewind()
    }
    val image = mediaImage?.let { FirebaseVisionImage.fromMediaImage(it, getRotationConstant(rotation)) }
    val firebaseVisionTextDetector = FirebaseVision.getInstance().onDeviceTextRecognizer

    if (image != null) {
        firebaseVisionTextDetector.processImage(image)
                .addOnSuccessListener {
                    val words = it.text.split("\n")
                    Toast.makeText(context, words.get(0), Toast.LENGTH_LONG).show()

                }
                .addOnFailureListener {
                    Toast.makeText(context, "Sorry, something went wrong!", Toast.LENGTH_SHORT).show()
                }
    }
}

}

[maxwellnewage]

Try to call rewind() for each media.Image.getPlanes[i].getBuffer() in your code:

private fun getCardDetails(imageProxy: ImageProxy, rotation: Int) {

val mediaImage = imageProxy.image

val planes = mediaImage!!.planes
if (planes.size >= 3) {
   // Reset buffer position for each plane's buffer. 
   for (plane in planes) {
       plane.getBuffer().rewind()
    }
    val image = mediaImage?.let { FirebaseVisionImage.fromMediaImage(it, getRotationConstant(rotation)) }
    val firebaseVisionTextDetector = FirebaseVision.getInstance().onDeviceTextRecognizer

    if (image != null) {
        firebaseVisionTextDetector.processImage(image)
                .addOnSuccessListener {
                    val words = it.text.split("\n")
                    Toast.makeText(context, words.get(0), Toast.LENGTH_LONG).show()

                }
                .addOnFailureListener {
                    Toast.makeText(context, "Sorry, something went wrong!", Toast.LENGTH_SHORT).show()
                }
    }
}

}

This works for me, but I don't understand why we should do this, and if is this apply also to the rest of detectors (barcode, ocr).