From b5022b4d41b871ed62f8f66b3b77781037e33e5d Mon Sep 17 00:00:00 2001 From: IanRDavies Date: Mon, 25 Apr 2022 11:05:09 +0100 Subject: [PATCH] android: use opensource boofcv to scan QR codes (#562) * open source qr code scanner * refined error localisation - todo moved * tidy up to make picking back up easier * better errors * scan images one at a time and once per second to avoid multiple requests being sent * more sophisticated avoidance of repeat contact requests * remove format specifier * refinements * update QR scanner to use boofcv * remove dependence on zxing * attribution * Update apps/android/app/src/main/java/chat/simplex/app/views/newchat/QRCodeScanner.kt Co-authored-by: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> --- apps/android/app/build.gradle | 6 +- .../chat/simplex/app/views/newchat/QRCode.kt | 20 +-- .../app/views/newchat/QRCodeScanner.kt | 128 ++++++++---------- 3 files changed, 70 insertions(+), 84 deletions(-) diff --git a/apps/android/app/build.gradle b/apps/android/app/build.gradle index b60f2e6840..62bf846734 100644 --- a/apps/android/app/build.gradle +++ b/apps/android/app/build.gradle @@ -90,9 +90,11 @@ dependencies { implementation "androidx.camera:camera-camera2:${camerax_version}" implementation "androidx.camera:camera-lifecycle:${camerax_version}" implementation "androidx.camera:camera-view:${camerax_version}" + //Barcode - implementation 'com.google.zxing:core:3.4.0' - implementation 'com.google.mlkit:barcode-scanning:17.0.2' + implementation 'org.boofcv:boofcv-android:0.40.1' + implementation 'org.boofcv:boofcv-core:0.40.1' + //Camera Permission implementation "com.google.accompanist:accompanist-permissions:0.23.0" diff --git a/apps/android/app/src/main/java/chat/simplex/app/views/newchat/QRCode.kt b/apps/android/app/src/main/java/chat/simplex/app/views/newchat/QRCode.kt index f55489c7d7..d91e6d9200 100644 --- a/apps/android/app/src/main/java/chat/simplex/app/views/newchat/QRCode.kt +++ b/apps/android/app/src/main/java/chat/simplex/app/views/newchat/QRCode.kt @@ -1,18 +1,17 @@ package chat.simplex.app.views.newchat import android.graphics.Bitmap -import android.graphics.Color import androidx.compose.foundation.Image import androidx.compose.runtime.Composable import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.tooling.preview.Preview +import boofcv.alg.fiducial.qrcode.QrCodeEncoder +import boofcv.alg.fiducial.qrcode.QrCodeGeneratorImage +import boofcv.android.ConvertBitmap import chat.simplex.app.R import chat.simplex.app.ui.theme.SimpleXTheme import chat.simplex.app.views.helpers.generalGetString -import com.google.zxing.BarcodeFormat -import com.google.zxing.EncodeHintType -import com.google.zxing.qrcode.QRCodeWriter @Composable fun QRCode(connReq: String, modifier: Modifier = Modifier) { @@ -24,15 +23,10 @@ fun QRCode(connReq: String, modifier: Modifier = Modifier) { } fun qrCodeBitmap(content: String, size: Int): Bitmap { - val hints = hashMapOf().also { it[EncodeHintType.MARGIN] = 1 } - val bits = QRCodeWriter().encode(content, BarcodeFormat.QR_CODE, size, size, hints) - return Bitmap.createBitmap(size, size, Bitmap.Config.RGB_565).also { - for (x in 0 until size) { - for (y in 0 until size) { - it.setPixel(x, y, if (bits[x, y]) Color.BLACK else Color.WHITE) - } - } - } + val qrCode = QrCodeEncoder().addAutomatic(content).fixate() + val renderer = QrCodeGeneratorImage(5) + renderer.render(qrCode) + return ConvertBitmap.grayToBitmap(renderer.gray, Bitmap.Config.RGB_565) } @Preview diff --git a/apps/android/app/src/main/java/chat/simplex/app/views/newchat/QRCodeScanner.kt b/apps/android/app/src/main/java/chat/simplex/app/views/newchat/QRCodeScanner.kt index f6bee70490..5d594b98b3 100644 --- a/apps/android/app/src/main/java/chat/simplex/app/views/newchat/QRCodeScanner.kt +++ b/apps/android/app/src/main/java/chat/simplex/app/views/newchat/QRCodeScanner.kt @@ -1,5 +1,6 @@ package chat.simplex.app.views.newchat +import android.annotation.SuppressLint import android.util.Log import android.view.ViewGroup import androidx.camera.core.* @@ -10,21 +11,24 @@ import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalLifecycleOwner import androidx.compose.ui.viewinterop.AndroidView import androidx.core.content.ContextCompat +import boofcv.abst.fiducial.QrCodeDetector +import boofcv.alg.color.ColorFormat +import boofcv.android.ConvertCameraImage +import boofcv.factory.fiducial.FactoryFiducial +import boofcv.struct.image.GrayU8 import chat.simplex.app.TAG import com.google.common.util.concurrent.ListenableFuture -import com.google.mlkit.vision.barcode.BarcodeScannerOptions -import com.google.mlkit.vision.barcode.BarcodeScanning -import com.google.mlkit.vision.barcode.common.Barcode -import com.google.mlkit.vision.common.InputImage import java.util.concurrent.* -// Bar code scanner adapted from https://github.com/MakeItEasyDev/Jetpack-Compose-BarCode-Scanner +// Adapted from learntodroid - https://gist.github.com/learntodroid/8f839be0b29d0378f843af70607bd7f5 @Composable fun QRCodeScanner(onBarcode: (String) -> Unit) { val context = LocalContext.current val lifecycleOwner = LocalLifecycleOwner.current var preview by remember { mutableStateOf(null) } + var lastAnalyzedTimeStamp = 0L + var contactLink = "" AndroidView( factory = { AndroidViewContext -> @@ -36,74 +40,60 @@ fun QRCodeScanner(onBarcode: (String) -> Unit) { ) implementationMode = PreviewView.ImplementationMode.COMPATIBLE } - }, -// modifier = Modifier.fillMaxSize(), - update = { previewView -> - val cameraSelector: CameraSelector = CameraSelector.Builder() - .requireLensFacing(CameraSelector.LENS_FACING_BACK) - .build() - val cameraExecutor: ExecutorService = Executors.newSingleThreadExecutor() - val cameraProviderFuture: ListenableFuture = - ProcessCameraProvider.getInstance(context) - - cameraProviderFuture.addListener({ - preview = Preview.Builder().build().also { - it.setSurfaceProvider(previewView.surfaceProvider) - } - val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get() - val barcodeAnalyser = BarCodeAnalyser { barcodes -> - barcodes.firstOrNull()?.rawValue?.let(onBarcode) - } - val imageAnalysis: ImageAnalysis = ImageAnalysis.Builder() - .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) - .build() - .also { it.setAnalyzer(cameraExecutor, barcodeAnalyser) } - - try { - cameraProvider.unbindAll() - cameraProvider.bindToLifecycle(lifecycleOwner, cameraSelector, preview, imageAnalysis) - } catch (e: Exception) { - Log.d(TAG, "CameraPreview: ${e.localizedMessage}") - } - }, ContextCompat.getMainExecutor(context)) } - ) -} + ) { previewView -> + val cameraSelector: CameraSelector = CameraSelector.Builder() + .requireLensFacing(CameraSelector.LENS_FACING_BACK) + .build() + val cameraExecutor: ExecutorService = Executors.newSingleThreadExecutor() + val cameraProviderFuture: ListenableFuture = + ProcessCameraProvider.getInstance(context) -class BarCodeAnalyser( - private val onBarcodeDetected: (barcodes: List) -> Unit, -): ImageAnalysis.Analyzer { - private var lastAnalyzedTimeStamp = 0L - - @ExperimentalGetImage - override fun analyze(image: ImageProxy) { - val currentTimestamp = System.currentTimeMillis() - if (currentTimestamp - lastAnalyzedTimeStamp >= TimeUnit.SECONDS.toMillis(1)) { - image.image?.let { imageToAnalyze -> - val options = BarcodeScannerOptions.Builder() - .setBarcodeFormats(Barcode.FORMAT_ALL_FORMATS) - .build() - val barcodeScanner = BarcodeScanning.getClient(options) - val imageToProcess = InputImage.fromMediaImage(imageToAnalyze, image.imageInfo.rotationDegrees) - - barcodeScanner.process(imageToProcess) - .addOnSuccessListener { barcodes -> - if (barcodes.isNotEmpty()) { - onBarcodeDetected(barcodes) - } else { - Log.d(TAG, "BarcodeAnalyser: No barcode Scanned") + cameraProviderFuture.addListener({ + preview = Preview.Builder().build().also { + it.setSurfaceProvider(previewView.surfaceProvider) + } + val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get() + val detector: QrCodeDetector = FactoryFiducial.qrcode(null, GrayU8::class.java) + fun getQR(imageProxy: ImageProxy) { + val currentTimeStamp = System.currentTimeMillis() + if (currentTimeStamp - lastAnalyzedTimeStamp >= TimeUnit.SECONDS.toMillis(1)) { + detector.process(imageProxyToGrayU8(imageProxy)) + val found = detector.detections + val qr = found.firstOrNull() + if (qr != null) { + if (qr.message != contactLink) { + // Make sure link is new and not a repeat + contactLink = qr.message + onBarcode(contactLink) } } - .addOnFailureListener { exception -> - Log.e(TAG, "BarcodeAnalyser: Something went wrong $exception") - } - .addOnCompleteListener { - image.close() - } + } + imageProxy.close() } - lastAnalyzedTimeStamp = currentTimestamp - } else { - image.close() - } + val imageAnalyzer = ImageAnalysis.Analyzer { proxy -> getQR(proxy) } + val imageAnalysis: ImageAnalysis = ImageAnalysis.Builder() + .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .setImageQueueDepth(1) + .build() + .also { it.setAnalyzer(cameraExecutor, imageAnalyzer) } + try { + cameraProvider.unbindAll() + cameraProvider.bindToLifecycle(lifecycleOwner, cameraSelector, preview, imageAnalysis) + } catch (e: Exception) { + Log.d(TAG, "CameraPreview: ${e.localizedMessage}") + } + }, ContextCompat.getMainExecutor(context)) } } + +@SuppressLint("UnsafeOptInUsageError") +private fun imageProxyToGrayU8(img: ImageProxy) : GrayU8? { + val image = img.image + if (image != null) { + val outImg = GrayU8() + ConvertCameraImage.imageToBoof(image, ColorFormat.GRAY, outImg, null) + return outImg + } + return null +}