How to do real time image processing in Android using OpenCV?

This article teaches how to pass camera preview frame to android-opencv on the fly.

The Android camera have preview data callback function. I can get the data from the function convert to the OpenCV Mat data form.Using Android JNI IF pass the frame data to the OpenCV, let the image processing do in the native OpenCV library.

Step1:I need a camera preview class to handle the Android camera device.The class i created from the article How to use camera in Android?.In the article shows the way to take picture and save to the file.But in this article i only need the frame data in the camera preview period.
In the new CameraPreview.java file, i removed unused callback functions. Leave the only callback function onPreviewFrame(), and create a native function interface.
And create a Runnable() object to do the image processing.

/*
*  CameraPreview.java
*/
public class CameraPreview implements SurfaceHolder.Callback, Camera.PreviewCallback
{
  private Camera mCamera = null;
  private ImageView MyCameraPreview = null;
  private Bitmap bitmap = null;
  private int[] pixels = null;
  private byte[] FrameData = null;
  private int imageFormat;
  private int PreviewSizeWidth;
  private int PreviewSizeHeight;
  private boolean bProcessing = false;
 
  Handler mHandler = new Handler(Looper.getMainLooper());
  
  public CameraPreview(int PreviewlayoutWidth, int PreviewlayoutHeight,
     ImageView CameraPreview)
  {
    PreviewSizeWidth = PreviewlayoutWidth;
    PreviewSizeHeight = PreviewlayoutHeight;
    MyCameraPreview = CameraPreview;
    bitmap = Bitmap.createBitmap(PreviewSizeWidth, PreviewSizeHeight, Bitmap.Config.ARGB_8888);
    pixels = new int[PreviewSizeWidth * PreviewSizeHeight];
  }

  @Override
  public void onPreviewFrame(byte[] arg0, Camera arg1) 
  {
    // At preview mode, the frame data will push to here.
    if (imageFormat == ImageFormat.NV21)
    {
      //We only accept the NV21(YUV420) format.
      if ( !bProcessing )
      {
        FrameData = arg0;
        mHandler.post(DoImageProcessing);
      }
    }
  }
 
  public void onPause()
  {
    mCamera.stopPreview();
  }

  @Override
  public void surfaceChanged(SurfaceHolder arg0, int arg1, int arg2, int arg3) 
  {
    Parameters parameters;
  
    parameters = mCamera.getParameters();
    // Set the camera preview size
    parameters.setPreviewSize(PreviewSizeWidth, PreviewSizeHeight);
  
    imageFormat = parameters.getPreviewFormat();
  
    mCamera.setParameters(parameters);
  
    mCamera.startPreview();
  }

  @Override
  public void surfaceCreated(SurfaceHolder arg0) 
  {
    mCamera = Camera.open();
    try
    {
      // If did not set the SurfaceHolder, the preview area will be black.
      mCamera.setPreviewDisplay(arg0);
      mCamera.setPreviewCallback(this);
    } 
    catch (IOException e)
    {
      mCamera.release();
      mCamera = null;
    }
  }

  @Override
  public void surfaceDestroyed(SurfaceHolder arg0) 
  {
     mCamera.setPreviewCallback(null);
  mCamera.stopPreview();
  mCamera.release();
  mCamera = null;
  }

  //
  // Native JNI 
  //
  public native boolean ImageProcessing(int width, int height, 
      byte[] NV21FrameData, int [] pixels);
  static 
  {
     System.loadLibrary("ImageProcessing");
  }
    
  private Runnable DoImageProcessing = new Runnable() 
  {
    public void run() 
    {
      Log.i("MyRealTimeImageProcessing", "DoImageProcessing():");
      bProcessing = true;
      ImageProcessing(PreviewSizeWidth, PreviewSizeHeight, FrameData, pixels);
      
      bitmap.setPixels(pixels, 0, PreviewSizeWidth, 0, 0, PreviewSizeWidth, PreviewSizeHeight);
      MyCameraPreview.setImageBitmap(bitmap);
      bProcessing = false;
    }
  };
}

Step2:Create a JNI cpp file to do the image processing.

/*
*  ImageProcessing.cpp
*/
#include <jni.h>

#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc_c.h>

using namespace std;
using namespace cv;

Mat * mCanny = NULL;

extern "C"
jboolean
Java_my_project_MyRealTimeImageProcessing_CameraPreview_ImageProcessing(
    JNIEnv* env, jobject thiz,
    jint width, jint height,
    jbyteArray NV21FrameData, jintArray outPixels)
{
  jbyte * pNV21FrameData = env->GetByteArrayElements(NV21FrameData, 0);
  jint * poutPixels = env->GetIntArrayElements(outPixels, 0);

  if ( mCanny == NULL )
  {
    mCanny = new Mat(height, width, CV_8UC1);
  }

  Mat mGray(height, width, CV_8UC1, (unsigned char *)pNV21FrameData);
  Mat mResult(height, width, CV_8UC4, (unsigned char *)poutPixels);
  IplImage srcImg = mGray;
  IplImage CannyImg = *mCanny;
  IplImage ResultImg = mResult;

  cvCanny(&srcImg, &CannyImg, 80, 100, 3);
  cvCvtColor(&CannyImg, &ResultImg, CV_GRAY2BGRA);

  env->ReleaseByteArrayElements(NV21FrameData, pNV21FrameData, 0);
  env->ReleaseIntArrayElements(outPixels, poutPixels, 0);
  return true;
}

Step3:Create a instance in the main class.

public class MyRealTimeImageProcessing extends Activity 
{
  private CameraPreview camPreview;
  private ImageView MyCameraPreview = null;
  private FrameLayout mainLayout;
  private int PreviewSizeWidth = 640;
  private int PreviewSizeHeight= 480;
  
  @Override
  public void onCreate(Bundle savedInstanceState) 
  {
    super.onCreate(savedInstanceState);
    //Set this APK Full screen
    getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN,  
              WindowManager.LayoutParams.FLAG_FULLSCREEN);
    //Set this APK no title
    requestWindowFeature(Window.FEATURE_NO_TITLE);  
    setContentView(R.layout.main);
        
    //
    // Create my camera preview 
    //
    MyCameraPreview = new ImageView(this);

    SurfaceView camView = new SurfaceView(this);
    SurfaceHolder camHolder = camView.getHolder();
    camPreview = new CameraPreview(PreviewSizeWidth, PreviewSizeHeight, MyCameraPreview);
        
    camHolder.addCallback(camPreview);
    camHolder.setType(SurfaceHolder.SURFACE_TYPE_PUSH_BUFFERS);
        
        
    mainLayout = (FrameLayout) findViewById(R.id.frameLayout1);
    mainLayout.addView(camView, new LayoutParams(PreviewSizeWidth, PreviewSizeHeight));
    mainLayout.addView(MyCameraPreview, new LayoutParams(PreviewSizeWidth, PreviewSizeHeight));
  }
  protected void onPause()
  {
    if ( camPreview != null)
      camPreview.onPause();
    super.onPause();
 }
}

Step4:The preview will be the canny image process result.