The main activity of the application.
+ * + *Provides operating system services to the LibGDX platform + * independant code, and handles OpenCV initialization and api calls.
+ */ +public class MainActivity extends AndroidApplication implements AndroidFunctionalityWrapper, ImageProcessor{ + /** + * Tag used for logging. + */ private static final String TAG = "NXTAR_ANDROID_MAIN"; + + /** + * Class name used for logging. + */ private static final String CLASS_NAME = MainActivity.class.getSimpleName(); + /** + * Output stream used to codify images as JPEG using Android's Bitmap class. + */ + private static final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + /** + * Indicates if OpenCV was initialized sucessfully. + */ + private static boolean ocvOn = false; + + /** + * Intrinsic camera matrix. + */ + private static Mat cameraMatrix; + + /** + * Distortion coeffitients matrix. + */ + private static Mat distortionCoeffs; + + /** + * Used to set and release multicast locks. + */ private WifiManager wifiManager; + + /** + * Used to maintain the multicast lock during the service discovery procedure. + */ private MulticastLock multicastLock; + + /** + * Handler used for requesting toast messages from the core LibGDX code. + */ private Handler uiHandler; + + /** + * User interface context used to show the toast messages. + */ private Context uiContext; - private boolean ocvOn; + + /** + * OpenCV asynchronous initializer callback for mobile devices. + */ private BaseLoaderCallback loaderCallback; - private final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - /*static{ - if (!OpenCVLoader.initDebug()){ - Gdx.app.exit(); - } - System.loadLibrary("cvproc"); - }*/ + /** + * Indicates if the current video streaming camera has been calibrated. + */ + private boolean cameraCalibrated; - public native void getMarkerCodesAndLocations(long inMat, long outMat, int[] codes); + /** + *Wrapper for the getAllMarkers native function.
+ * + * @param inMat INPUT. The image to analize. + * @param outMat OUTPUT. The image with the markers highlighted. + * @param codes OUTPUT. The codes for each marker detected. Must be {@link ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS} elements long. + * @param camMat INPUT. The intrinsic camera matrix. + * @param distMat INPUT. The distortion coefficients of the camera. + * @param translations OUTPUT. The markers pose translations. Must be {@link ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS} * 3 elements long. + * @param rotations OUTPUT. The markers pose rotations. Must be {@link ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS} * 9 elements long. + */ + private native void getMarkerCodesAndLocations(long inMat, long outMat, int[] codes, long camMat, long distMat, float[] translations, float[] rotations); + /** + *Wrapper for the findCalibrationPattern native function.
+ * + * @param inMat INPUT. The image to analize. + * @param outMat OUTPUT. The image with the calibration pattern highlighted. + * @param points OUTPUT. The spatial location of the calibration points if found. + * @return True if the calibration pattern was found. False otherwise. + */ + private native boolean findCalibrationPattern(long inMat, long outMat, float[] points); + + /** + *Wrapper around the getCameraParameters native function.
+ * + * @param camMat OUTPUT. The intrinsic camera matrix. + * @param distMat OUTPUT. The distortion coeffitients matrix. + * @param frame INPUT. A sample input image from the camera to calibrate. + * @param calibrationPoints INPUT. The calibration points of all samples. + * @return The calibration error as returned by OpenCV. + */ + private native double calibrateCameraParameters(long camMat, long distMat, long frame, float[] calibrationPoints); + + /** + *Static block. Tries to load OpenCV and the native method implementations + * statically if running on an OUYA device.
+ */ + static{ + if(Ouya.runningOnOuya){ + if(!OpenCVLoader.initDebug()) + ocvOn = false; + + try{ + System.loadLibrary("cvproc"); + ocvOn = true; + }catch(UnsatisfiedLinkError u){ + ocvOn = false; + } + } + } + + /** + *Initializes this activity
+ * + *This method handles the initialization of LibGDX and OpenCV. OpenCV is + * loaded the asynchronous method if the devices is not an OUYA console.
+ * + * @param savedInstanceState The application state if it was saved in a previous run. + */ @Override public void onCreate(Bundle savedInstanceState){ super.onCreate(savedInstanceState); - ocvOn = false; + cameraCalibrated = false; + // Set screen orientation. Portrait on mobile devices, landscape on OUYA. if(!Ouya.runningOnOuya){ setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT); }else{ setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE); } + // Set up the Android related variables. uiHandler = new Handler(); uiContext = this; wifiManager = (WifiManager)getSystemService(Context.WIFI_SERVICE); + // Attempt to initialize OpenCV. + if(!Ouya.runningOnOuya){ + // If running on a moble device, use the asynchronous method aided + // by the OpenCV Manager app. + loaderCallback = new BaseLoaderCallback(this){ + @Override + public void onManagerConnected(int status){ + switch(status){ + case LoaderCallbackInterface.SUCCESS: + // If successfully initialized then load the native method implementations and + // initialize the static matrices. + System.loadLibrary("cvproc"); + ocvOn = true; + cameraMatrix = new Mat(); + distortionCoeffs = new Mat(); + break; + + default: + Toast.makeText(uiContext, R.string.ocv_failed, Toast.LENGTH_LONG).show(); + ocvOn = false; + break; + } + } + }; + + // Launch the asynchronous initializer. + OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION_2_4_7, this, loaderCallback); + + }else{ + // If running on an OUYA device. + if(ocvOn){ + // If OpenCV loaded successfully then initialize the native matrices. + cameraMatrix = new Mat(); + distortionCoeffs = new Mat(); + }else{ + Toast.makeText(uiContext, R.string.ocv_failed, Toast.LENGTH_LONG).show(); + } + } + + // Configure LibGDX. AndroidApplicationConfiguration cfg = new AndroidApplicationConfiguration(); cfg.useGL20 = true; cfg.useAccelerometer = false; cfg.useCompass = false; cfg.useWakelock = true; - loaderCallback = new BaseLoaderCallback(this){ - @Override - public void onManagerConnected(int status){ - switch(status){ - case LoaderCallbackInterface.SUCCESS: - System.loadLibrary("cvproc"); - ocvOn = true; - break; - default: - Toast.makeText(uiContext, R.string.ocv_failed, Toast.LENGTH_LONG).show(); - Gdx.app.exit(); - break; - } - } - }; - - OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION_2_4_7, this, loaderCallback); + // Launch the LibGDX core game class. initialize(new NxtARCore(this), cfg); } - //////////////////////////////// - // Toaster interface methods. // - //////////////////////////////// + //////////////////////////////////////////////// + // OSFunctionalityProvider interface methods. // + //////////////////////////////////////////////// + + /** + *Shows a short message on screen using Android's toast mechanism.
+ * + * @param msg The message to show. + */ @Override public void showShortToast(final String msg){ uiHandler.post(new Runnable(){ @@ -119,6 +259,11 @@ public class MainActivity extends AndroidApplication implements Toaster, Multica }); } + /** + *Shows a long message on screen using Android's toast mechanism.
+ * + * @param msg The message to show. + */ @Override public void showLongToast(final String msg){ uiHandler.post(new Runnable(){ @@ -129,9 +274,9 @@ public class MainActivity extends AndroidApplication implements Toaster, Multica }); } - ///////////////////////////////////////// - // MulticastEnabler interface methods. // - ///////////////////////////////////////// + /** + *Enable the transmision and reception of multicast network messages.
+ */ @Override public void enableMulticast(){ Gdx.app.log(TAG, CLASS_NAME + ".enableMulticast() :: Requesting multicast lock."); @@ -140,6 +285,9 @@ public class MainActivity extends AndroidApplication implements Toaster, Multica multicastLock.acquire(); } + /** + *Disables the transmision and reception of multicast network messages.
+ */ @Override public void disableMulticast(){ Gdx.app.log(TAG, CLASS_NAME + ".disableMulticast() :: Releasing multicast lock."); @@ -149,40 +297,205 @@ public class MainActivity extends AndroidApplication implements Toaster, Multica } } + //////////////////////////////////// + // CVProcessor interface methods. // + //////////////////////////////////// + @Override - public CVData processFrame(byte[] frame, int w, int h) { + public MarkerData findMarkersInFrame(byte[] frame){ if(ocvOn){ - int codes[] = new int[15]; + if(cameraCalibrated){ + int[] codes = new int[ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS]; + float[] translations = new float[ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS * 3]; + float[] rotations = new float[ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS * 9]; + MarkerData data; + Bitmap tFrame, mFrame; + Mat inImg = new Mat(); + Mat outImg = new Mat(); + + // Fill the codes array with -1 to indicate markers that were not found; + for(int i : codes) + codes[i] = -1; + + // Decode the input image and convert it to an OpenCV matrix. + tFrame = BitmapFactory.decodeByteArray(frame, 0, frame.length); + Utils.bitmapToMat(tFrame, inImg); + + // Find the markers in the input image. + getMarkerCodesAndLocations(inImg.getNativeObjAddr(), outImg.getNativeObjAddr(), codes, cameraMatrix.getNativeObjAddr(), distortionCoeffs.getNativeObjAddr(), translations, rotations); + + // Encode the output image as a JPEG image. + mFrame = Bitmap.createBitmap(outImg.cols(), outImg.rows(), Bitmap.Config.RGB_565); + Utils.matToBitmap(outImg, mFrame); + mFrame.compress(CompressFormat.JPEG, 100, outputStream); + + // Create and fill the output data structure. + data = new MarkerData(); + data.outFrame = outputStream.toByteArray(); + data.markerCodes = codes; + data.rotationMatrices = new Matrix3[ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS]; + data.translationVectors = new Vector3[ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS]; + + for(int i = 0, p = 0; i < ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS; i++, p += 3){ + data.translationVectors[i] = new Vector3(translations[p], translations[p + 1], translations[p + 2]); + } + + for(int k = 0; k < ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS; k++){ + data.rotationMatrices[k] = new Matrix3(); + for(int row = 0; row < 3; row++){ + for(int col = 0; col < 3; col++){ + data.rotationMatrices[k].val[col + (row * 3)] = rotations[col + (row * 3) + (9 * k)]; + } + } + } + + // Clean up memory. + tFrame.recycle(); + mFrame.recycle(); + outputStream.reset(); + + return data; + }else{ + Gdx.app.debug(TAG, CLASS_NAME + ".findMarkersInFrame(): The camera has not been calibrated."); + return null; + } + }else{ + Gdx.app.debug(TAG, CLASS_NAME + ".findMarkersInFrame(): OpenCV is not ready or failed to load."); + return null; + } + } + + @Override + public CalibrationData findCalibrationPattern(byte[] frame){ + if(ocvOn){ + boolean found; + float points[] = new float[ProjectConstants.CALIBRATION_PATTERN_POINTS * 2]; Bitmap tFrame, mFrame; + Mat inImg = new Mat(), outImg = new Mat(); + CalibrationData data = new CalibrationData(); + // Decode the input frame and convert it to an OpenCV Matrix. tFrame = BitmapFactory.decodeByteArray(frame, 0, frame.length); - - Mat inImg = new Mat(); - Mat outImg = new Mat(); Utils.bitmapToMat(tFrame, inImg); - getMarkerCodesAndLocations(inImg.getNativeObjAddr(), outImg.getNativeObjAddr(), codes); + // Attempt to find the calibration pattern in the input frame. + found = findCalibrationPattern(inImg.getNativeObjAddr(), outImg.getNativeObjAddr(), points); - Mat temp = new Mat(); - Imgproc.cvtColor(outImg, temp, Imgproc.COLOR_BGR2RGB); - - mFrame = Bitmap.createBitmap(temp.cols(), temp.rows(), Bitmap.Config.RGB_565); - Utils.matToBitmap(temp, mFrame); + // Encode the output image as a JPEG image. + mFrame = Bitmap.createBitmap(outImg.cols(), outImg.rows(), Bitmap.Config.RGB_565); + Utils.matToBitmap(outImg, mFrame); mFrame.compress(CompressFormat.JPEG, 100, outputStream); - CVData data = new CVData(); + // Prepare the output data structure. data.outFrame = outputStream.toByteArray(); - data.markerCodes = codes; + data.calibrationPoints = found ? points : null; + // Clean up memory. tFrame.recycle(); mFrame.recycle(); outputStream.reset(); return data; - }else{ - Gdx.app.debug(TAG, CLASS_NAME + ".processFrame(): OpenCV is not ready or failed to load."); + }else{ + Gdx.app.debug(TAG, CLASS_NAME + ".findCalibrationPattern(): OpenCV is not ready or failed to load."); return null; } } -} \ No newline at end of file + + @Override + public void calibrateCamera(float[][] calibrationSamples, byte[] frame) { + if(ocvOn){ + float[] calibrationPoints = new float[ProjectConstants.CALIBRATION_PATTERN_POINTS * 2 * ProjectConstants.CALIBRATION_SAMPLES]; + int w = ProjectConstants.CALIBRATION_PATTERN_POINTS * 2; + Bitmap tFrame; + Mat inImg = new Mat(); + + // Save the calibration points on a one dimensional array for easier parameter passing + // to the native code. + for(int i = 0; i < ProjectConstants.CALIBRATION_SAMPLES; i++){ + for(int j = 0, p = 0; j < ProjectConstants.CALIBRATION_PATTERN_POINTS; j++, p += 2){ + calibrationPoints[p + (w * i)] = calibrationSamples[i][p]; + calibrationPoints[(p + 1) + (w * i)] = calibrationSamples[i][p + 1]; + } + } + + // Decode the input image and convert it to an OpenCV matrix. + tFrame = BitmapFactory.decodeByteArray(frame, 0, frame.length); + Utils.bitmapToMat(tFrame, inImg); + + // Attempt to obtain the camera parameters. + double error = calibrateCameraParameters(cameraMatrix.getNativeObjAddr(), distortionCoeffs.getNativeObjAddr(), inImg.getNativeObjAddr(), calibrationPoints); + Gdx.app.log(TAG, CLASS_NAME + "calibrateCamera(): calibrateCameraParameters retured " + Double.toString(error)); + cameraCalibrated = true; + + }else{ + Gdx.app.debug(TAG, CLASS_NAME + ".calibrateCamera(): OpenCV is not ready or failed to load."); + } + } + + @Override + public byte[] undistortFrame(byte[] frame){ + if(ocvOn){ + if(cameraCalibrated){ + byte undistortedFrame[]; + Bitmap tFrame, mFrame; + Mat inImg = new Mat(), outImg = new Mat(); + + // Decode the input frame and convert it to an OpenCV Matrix. + tFrame = BitmapFactory.decodeByteArray(frame, 0, frame.length); + Utils.bitmapToMat(tFrame, inImg); + + // Apply the undistort correction to the input frame. + Imgproc.undistort(inImg, outImg, cameraMatrix, distortionCoeffs); + + // Encode the output image as a JPEG image. + mFrame = Bitmap.createBitmap(outImg.cols(), outImg.rows(), Bitmap.Config.RGB_565); + Utils.matToBitmap(outImg, mFrame); + mFrame.compress(CompressFormat.JPEG, 100, outputStream); + + // Prepare the return frame. + undistortedFrame = outputStream.toByteArray(); + + // Clean up memory. + tFrame.recycle(); + mFrame.recycle(); + outputStream.reset(); + + return undistortedFrame; + + }else{ + Gdx.app.debug(TAG, CLASS_NAME + ".undistortFrame(): Camera has not been calibrated."); + return null; + } + }else{ + Gdx.app.debug(TAG, CLASS_NAME + ".undistortFrame(): OpenCV is not ready or failed to load."); + return null; + } + } + + @Override + public boolean isCameraCalibrated() { + return ocvOn && cameraCalibrated; + } + + @Override + public float getFocalPointX() { + return ocvOn && cameraCalibrated ? (float)cameraMatrix.get(0, 0)[0] : 0.0f; + } + + @Override + public float getFocalPointY() { + return ocvOn && cameraCalibrated ? (float)cameraMatrix.get(1, 1)[0] : 0.0f; + } + + @Override + public float getCameraCenterX() { + return ocvOn && cameraCalibrated ? (float)cameraMatrix.get(0, 2)[0] : 0.0f; + } + + @Override + public float getCameraCenterY() { + return ocvOn && cameraCalibrated ? (float)cameraMatrix.get(1, 2)[0] : 0.0f; + } +}