I am trying to integrate MediaPipe Poselandmarker in my flutter app using native communication with the help of method channel. The problem is when the camera is getting opened,it is being displayed over top of overlay. But i want overlay to be displayed over top of camera. I am sharing my kotlin code for reference. Please tell me where I am doing wrong.
This is my NativeView.kt:, In this I am setting up fragment container
private fun setupFragmentContainer(context: Context) { Log.d("NativeView", "Setting up FragmentContainerView for CameraFragment") // Create a FragmentContainerView dynamically if not already added. val fragmentContainerView = FragmentContainerView(context).apply { id = View.generateViewId() // Generate a unique ID for the container layoutParams = LayoutParams( LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT ) } this.addView(fragmentContainerView) // Use FragmentManager to add or replace the fragment val fragmentActivity = activity as? FragmentActivity ?: throw ClassCastException("Activity must be a FragmentActivity") val fragmentTransaction = fragmentActivity.supportFragmentManager.beginTransaction() // Check if CameraFragment is already added val existingFragment = fragmentActivity.supportFragmentManager.findFragmentByTag("CameraFragment") if (existingFragment == null) { fragmentTransaction.replace(fragmentContainerView.id, CameraFragment(), "CameraFragment") fragmentTransaction.commit() Log.d("NativeView", "CameraFragment added to FragmentContainerView") } else { Log.d("NativeView", "CameraFragment already exists, no need to add again") } // After fragment is added, bring overlays to the front // val overlayView = findViewById<OverlayView>(R.id.overlay) // Ensure this ID is correct // overlayView.bringToFront() // // val bottomSheetLayout = findViewById<View>(R.id.bottom_sheet_layout) // bottomSheetLayout.bringToFront() // // // Optionally, invalidate the views to make sure they get redrawn // overlayView.invalidate() // bottomSheetLayout.invalidate() } This is my CameraFragmemt.kt
/* * Copyright 2023 The TensorFlow Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package app.peak.med.bibo.fragment import android.annotation.SuppressLint import android.content.res.Configuration import android.os.Bundle import android.util.Log import android.view.LayoutInflater import android.view.View import android.view.ViewGroup import android.widget.AdapterView import android.widget.Toast import androidx.camera.core.Preview import androidx.camera.core.CameraSelector import androidx.camera.core.ImageAnalysis import androidx.camera.core.ImageProxy import androidx.camera.core.Camera import androidx.camera.core.AspectRatio import androidx.camera.lifecycle.ProcessCameraProvider import androidx.core.content.ContextCompat import androidx.fragment.app.Fragment import androidx.fragment.app.activityViewModels import androidx.navigation.Navigation import app.peak.med.bibo.PoseLandmarkerHelper import app.peak.med.bibo.MainViewModel import app.peak.med.bibo.OverlayView import app.peak.med.bibo.R import app.peak.med.bibo.databinding.FragmentCameraBinding import com.google.mediapipe.tasks.vision.core.RunningMode import java.util.Locale import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.TimeUnit class CameraFragment : Fragment(), PoseLandmarkerHelper.LandmarkerListener { companion object { private const val TAG = "CAMERA FRAGMENT" } private var _fragmentCameraBinding: FragmentCameraBinding? = null private val fragmentCameraBinding get() = _fragmentCameraBinding!! private lateinit var poseLandmarkerHelper: PoseLandmarkerHelper private val viewModel: MainViewModel by activityViewModels() private var preview: Preview? = null private var imageAnalyzer: ImageAnalysis? = null private var camera: Camera? = null private var cameraProvider: ProcessCameraProvider? = null private var cameraFacing = CameraSelector.LENS_FACING_FRONT /** Blocking ML operations are performed using this executor */ private lateinit var backgroundExecutor: ExecutorService override fun onResume() { super.onResume() // Make sure that all permissions are still present, since the // user could have removed them while the app was in paused state. if (!PermissionsFragment.hasPermissions(requireContext())) { Navigation.findNavController( requireActivity(), R.id.fragment_container ).navigate(R.id.action_camera_to_permissions) } // Start the PoseLandmarkerHelper again when users come back // to the foreground. backgroundExecutor.execute { if(this::poseLandmarkerHelper.isInitialized) { if (poseLandmarkerHelper.isClose()) { poseLandmarkerHelper.setupPoseLandmarker() } } } } override fun onPause() { super.onPause() if(this::poseLandmarkerHelper.isInitialized) { viewModel.setMinPoseDetectionConfidence(poseLandmarkerHelper.minPoseDetectionConfidence) viewModel.setMinPoseTrackingConfidence(poseLandmarkerHelper.minPoseTrackingConfidence) viewModel.setMinPosePresenceConfidence(poseLandmarkerHelper.minPosePresenceConfidence) viewModel.setDelegate(poseLandmarkerHelper.currentDelegate) // Close the PoseLandmarkerHelper and release resources backgroundExecutor.execute { poseLandmarkerHelper.clearPoseLandmarker() } } } override fun onDestroyView() { _fragmentCameraBinding = null super.onDestroyView() // Shut down our background executor backgroundExecutor.shutdown() backgroundExecutor.awaitTermination( Long.MAX_VALUE, TimeUnit.NANOSECONDS ) } override fun onCreateView( inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle? ): View { _fragmentCameraBinding = FragmentCameraBinding.inflate(inflater, container, false) return fragmentCameraBinding.root } @SuppressLint("MissingPermission") override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) val overlayView = view.findViewById<OverlayView>(R.id.overlay) overlayView.bringToFront() // If needed, ensure bottom sheet layout is on top of both camera and overlay // val bottomSheetLayout = view.findViewById<View>(R.id.bottom_sheet_layout) // bottomSheetLayout.bringToFront() view.requestLayout() // Initialize our background executor backgroundExecutor = Executors.newSingleThreadExecutor() // Wait for the views to be properly laid out fragmentCameraBinding.viewFinder.post { // Set up the camera and its use cases setUpCamera() overlayView.bringToFront() //fragmentCameraBinding.overlay.bringToFront() // Ensure overlay is above camera view. } // Create the PoseLandmarkerHelper that will handle the inference backgroundExecutor.execute { poseLandmarkerHelper = PoseLandmarkerHelper( context = requireContext(), runningMode = RunningMode.LIVE_STREAM, minPoseDetectionConfidence = viewModel.currentMinPoseDetectionConfidence, minPoseTrackingConfidence = viewModel.currentMinPoseTrackingConfidence, minPosePresenceConfidence = viewModel.currentMinPosePresenceConfidence, currentDelegate = viewModel.currentDelegate, poseLandmarkerHelperListener = this ) } // Attach listeners to UI control widgets // initBottomSheetControls() } /*private fun initBottomSheetControls() { // init bottom sheet settings fragmentCameraBinding.bottomSheetLayout.detectionThresholdValue.text = String.format( Locale.US, "%.2f", viewModel.currentMinPoseDetectionConfidence ) fragmentCameraBinding.bottomSheetLayout.trackingThresholdValue.text = String.format( Locale.US, "%.2f", viewModel.currentMinPoseTrackingConfidence ) fragmentCameraBinding.bottomSheetLayout.presenceThresholdValue.text = String.format( Locale.US, "%.2f", viewModel.currentMinPosePresenceConfidence ) // When clicked, lower pose detection score threshold floor fragmentCameraBinding.bottomSheetLayout.detectionThresholdMinus.setOnClickListener { if (poseLandmarkerHelper.minPoseDetectionConfidence >= 0.2) { poseLandmarkerHelper.minPoseDetectionConfidence -= 0.1f updateControlsUi() } } // When clicked, raise pose detection score threshold floor fragmentCameraBinding.bottomSheetLayout.detectionThresholdPlus.setOnClickListener { if (poseLandmarkerHelper.minPoseDetectionConfidence <= 0.8) { poseLandmarkerHelper.minPoseDetectionConfidence += 0.1f updateControlsUi() } } // When clicked, lower pose tracking score threshold floor fragmentCameraBinding.bottomSheetLayout.trackingThresholdMinus.setOnClickListener { if (poseLandmarkerHelper.minPoseTrackingConfidence >= 0.2) { poseLandmarkerHelper.minPoseTrackingConfidence -= 0.1f updateControlsUi() } } // When clicked, raise pose tracking score threshold floor fragmentCameraBinding.bottomSheetLayout.trackingThresholdPlus.setOnClickListener { if (poseLandmarkerHelper.minPoseTrackingConfidence <= 0.8) { poseLandmarkerHelper.minPoseTrackingConfidence += 0.1f updateControlsUi() } } // When clicked, lower pose presence score threshold floor fragmentCameraBinding.bottomSheetLayout.presenceThresholdMinus.setOnClickListener { if (poseLandmarkerHelper.minPosePresenceConfidence >= 0.2) { poseLandmarkerHelper.minPosePresenceConfidence -= 0.1f updateControlsUi() } } // When clicked, raise pose presence score threshold floor fragmentCameraBinding.bottomSheetLayout.presenceThresholdPlus.setOnClickListener { if (poseLandmarkerHelper.minPosePresenceConfidence <= 0.8) { poseLandmarkerHelper.minPosePresenceConfidence += 0.1f updateControlsUi() } } // When clicked, change the underlying hardware used for inference. // Current options are CPU and GPU fragmentCameraBinding.bottomSheetLayout.spinnerDelegate.setSelection( viewModel.currentDelegate, false ) fragmentCameraBinding.bottomSheetLayout.spinnerDelegate.onItemSelectedListener = object : AdapterView.OnItemSelectedListener { override fun onItemSelected( p0: AdapterView<*>?, p1: View?, p2: Int, p3: Long ) { try { poseLandmarkerHelper.currentDelegate = p2 updateControlsUi() } catch(e: UninitializedPropertyAccessException) { Log.e(TAG, "PoseLandmarkerHelper has not been initialized yet.") } } override fun onNothingSelected(p0: AdapterView<*>?) { /* no op */ } } // When clicked, change the underlying model used for object detection fragmentCameraBinding.bottomSheetLayout.spinnerModel.setSelection( viewModel.currentModel, false ) fragmentCameraBinding.bottomSheetLayout.spinnerModel.onItemSelectedListener = object : AdapterView.OnItemSelectedListener { override fun onItemSelected( p0: AdapterView<*>?, p1: View?, p2: Int, p3: Long ) { poseLandmarkerHelper.currentModel = p2 updateControlsUi() } override fun onNothingSelected(p0: AdapterView<*>?) { /* no op */ } } }*/ // Update the values displayed in the bottom sheet. Reset Poselandmarker // helper. /*private fun updateControlsUi() { if(this::poseLandmarkerHelper.isInitialized) { fragmentCameraBinding.bottomSheetLayout.detectionThresholdValue.text = String.format( Locale.US, "%.2f", poseLandmarkerHelper.minPoseDetectionConfidence ) fragmentCameraBinding.bottomSheetLayout.trackingThresholdValue.text = String.format( Locale.US, "%.2f", poseLandmarkerHelper.minPoseTrackingConfidence ) fragmentCameraBinding.bottomSheetLayout.presenceThresholdValue.text = String.format( Locale.US, "%.2f", poseLandmarkerHelper.minPosePresenceConfidence ) // Needs to be cleared instead of reinitialized because the GPU // delegate needs to be initialized on the thread using it when applicable backgroundExecutor.execute { poseLandmarkerHelper.clearPoseLandmarker() poseLandmarkerHelper.setupPoseLandmarker() } fragmentCameraBinding.overlay.clear() } }*/ // Initialize CameraX, and prepare to bind the camera use cases private fun setUpCamera() { val cameraProviderFuture = ProcessCameraProvider.getInstance(requireContext()) cameraProviderFuture.addListener( { // CameraProvider cameraProvider = cameraProviderFuture.get() // Build and bind the camera use cases bindCameraUseCases() }, ContextCompat.getMainExecutor(requireContext()) ) } // Declare and bind preview, capture and analysis use cases @SuppressLint("UnsafeOptInUsageError") private fun bindCameraUseCases() { // CameraProvider val cameraProvider = cameraProvider ?: throw IllegalStateException("Camera initialization failed.") val cameraSelector = CameraSelector.Builder().requireLensFacing(cameraFacing).build() // Preview. Only using the 4:3 ratio because this is the closest to our models preview = Preview.Builder().setTargetAspectRatio(AspectRatio.RATIO_4_3) .setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation) .build() // ImageAnalysis. Using RGBA 8888 to match how our models work imageAnalyzer = ImageAnalysis.Builder().setTargetAspectRatio(AspectRatio.RATIO_4_3) .setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation) .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888) .build() // The analyzer can then be assigned to the instance .also { it.setAnalyzer(backgroundExecutor) { image -> detectPose(image) } } // Must unbind the use-cases before rebinding them cameraProvider.unbindAll() try { // A variable number of use-cases can be passed here - // camera provides access to CameraControl & CameraInfo camera = cameraProvider.bindToLifecycle( this, cameraSelector, preview, imageAnalyzer ) // Attach the viewfinder's surface provider to preview use case preview?.setSurfaceProvider(fragmentCameraBinding.viewFinder.surfaceProvider) // val overlayView: View = view.findViewById(R.id.overlay) // This can be any view like lines or annotations // overlayView.bringToFront() val overlayView: View? = view?.findViewById(R.id.overlay)!! overlayView?.let { // Do something with the overlay view, e.g., bring it to the front it.bringToFront() } } catch (exc: Exception) { Log.e(TAG, "Use case binding failed", exc) } } private fun detectPose(imageProxy: ImageProxy) { if(this::poseLandmarkerHelper.isInitialized) { poseLandmarkerHelper.detectLiveStream( imageProxy = imageProxy, isFrontCamera = cameraFacing == CameraSelector.LENS_FACING_FRONT ) } } override fun onConfigurationChanged(newConfig: Configuration) { super.onConfigurationChanged(newConfig) imageAnalyzer?.targetRotation = fragmentCameraBinding.viewFinder.display.rotation } // Update UI after pose have been detected. Extracts original // image height/width to scale and place the landmarks properly through // OverlayView override fun onResults( resultBundle: PoseLandmarkerHelper.ResultBundle ) { activity?.runOnUiThread { if (_fragmentCameraBinding != null) { // fragmentCameraBinding.bottomSheetLayout.inferenceTimeVal.text = // String.format("%d ms", resultBundle.inferenceTime) //fragmentCameraBinding.overlay.bringToFront() // Pass necessary information to OverlayView for drawing on the canvas fragmentCameraBinding.overlay.setResults( resultBundle.results.first(), resultBundle.inputImageHeight, resultBundle.inputImageWidth, RunningMode.LIVE_STREAM ) // Force a redraw fragmentCameraBinding.overlay.invalidate() } } } override fun onError(error: String, errorCode: Int) { activity?.runOnUiThread { Toast.makeText(requireContext(), error, Toast.LENGTH_SHORT).show() if (errorCode == PoseLandmarkerHelper.GPU_ERROR) { // fragmentCameraBinding.bottomSheetLayout.spinnerDelegate.setSelection( // PoseLandmarkerHelper.DELEGATE_CPU, false //) } } } } In MainActivity.kt, when I am running the below code, overlay is getting built on top of camera
class MainActivity : AppCompatActivity() { private lateinit var activityMainBinding: ActivityMainBinding private val viewModel : MainViewModel by viewModels() override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) activityMainBinding = ActivityMainBinding.inflate(layoutInflater) setContentView(activityMainBinding.root) val navHostFragment = supportFragmentManager.findFragmentById(R.id.fragment_container) as NavHostFragment val navController = navHostFragment.navController activityMainBinding.navigation.setupWithNavController(navController) activityMainBinding.navigation.setOnNavigationItemReselectedListener { // ignore the reselection } } override fun onBackPressed() { finish() } } But when I have changed it to flutterFragmentActivity, above problem is coming
class MainActivity : FlutterFragmentActivity() { //class MainActivity : FlutterActivity() {
companion object { private const val TAG = "MainActivity" } override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) // No need to access flutterEngine here; it will be configured below. } override fun configureFlutterEngine(flutterEngine: FlutterEngine) { super.configureFlutterEngine(flutterEngine) try { flutterEngine.plugins.add(FlutterMediapipePlugin()) } catch (e: Exception) { Log.e(TAG, "Error registering plugin flutter_mediapipe", e) } } }
