So the short version is that I'm trying to do head tracking on an Android device to move/tilt a camera in an OpenGL ES 2.0 virtual environment. Should be simple enough but I can't get it to work
I'm new to almost all of this. It's my first time working with Android, OpenGL, VR, and the math (matrices) associated with it. That being said I understand the hardware and sensor side very well.
I'm using the rotation matrix from the sensors. I can't use the quaternion because the API version (15) doesn't support the w component, and google cardboard is out of the question. The matrix seems to work, but the axis the device is tilted along does not match up with the tilt axis of the camera.
This seems like something that should be really simple, especially given how many demos of the tech are out there, but I can't seem to get it to work properly. Can someone tell me what I'm doing wrong with the rotation matrix?
OpenGL Renderer Class:
/*
Based on Android tutorials
*/
package com.example.android.opengl;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
import android.opengl.Matrix;
import android.util.Log;
/**
* Provides drawing instructions for a GLSurfaceView object. This class
* must override the OpenGL ES drawing lifecycle methods:
* <ul>
* <li>{@link android.opengl.GLSurfaceView.Renderer#onSurfaceCreated}</li>
* <li>{@link android.opengl.GLSurfaceView.Renderer#onDrawFrame}</li>
* <li>{@link android.opengl.GLSurfaceView.Renderer#onSurfaceChanged}</li>
* </ul>
*/
public class MyGLRenderer implements GLSurfaceView.Renderer {
private static final String TAG = "MyGLRenderer";
private Square square1, square2, square3, square4;
private IMUData localIMUData = new IMUData();
// mMVPMatrix is an abbreviation for "Model View Projection Matrix"
private final float[] mMVPMatrix = new float[16];
private final float[] mProjectionMatrix = new float[16];
private final float[] mViewMatrix = new float[16];
@Override
public void onSurfaceCreated(GL10 unused, EGLConfig config) {
// Set the background frame color
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
square1 = new Square(
-.5f, 5f, .5f,
-.5f, 5f, -.5f,
.5f, 5f, -.5f,
.5f, 5f, .5f,
1f, 0f, 0f, 1f);
square2 = new Square(
.5f, -5f, -.5f,
-.5f, -5f, -.5f,
-.5f, -5f, .5f,
.5f, -5f, .5f,
0f, 1f, 0f, 1f);
square3 = new Square(
5f, .5f, -.5f,
5f, -.5f, -.5f,
5f, -.5f, .5f,
5f, .5f, .5f,
0f, 0f, 1f, 1f);
square4 = new Square(
-5f, .5f, -.5f,
-5f, -.5f, -.5f,
-5f, -.5f, .5f,
-5f, .5f, .5f,
.5f, 0f, .5f, 1f);
//initialize the sensor listeners
localIMUData.initSensorListeners();
}
@Override
public void onDrawFrame(GL10 unused) {
// Draw background color
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
//get the heading anle for calculations
float heading = localIMUData.getTheta();
// Set the camera position (View matrix)
Matrix.setLookAtM(mViewMatrix, 0, //result matrix, index to start at
// position
0, 0, 0,
//center of view
0 + (float) Math.sin(heading), 0 + (float) Math.cos(heading), 0,
//"up" vector
0f, 0f, 1f
);
//this is my attempt at modifying the matrices to get head tracking to work
float[] tempMatrix = localIMUData.getRotMat();
//set rotMat to identity matrix to stop null pointer error
float[] rotMat = new float[] {1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1};
//this should allow the program to work until the rotation matrix is created
if (tempMatrix != null){
//I know this is sloppy but it's all I can come up with for right now to pad the matrix
float[] t = localIMUData.getRotMat();
float[] i = {t[0],t[1],t[2],0, t[3],t[4],t[5],0, t[6],t[7],t[8],0, 0,0,0,1};
//and I know this is bad too but this is for a proof of concept and doesn't need to be good code yet
rotMat = i;
}
float[] adjustedView = new float[16];
Matrix.multiplyMM(adjustedView, 0, mViewMatrix, 0, rotMat, 0);
// Calculate the projection and view transformation
Matrix.multiplyMM(mMVPMatrix, 0, mProjectionMatrix, 0, adjustedView, 0);
// Draw squares
square1.draw(mMVPMatrix);
square2.draw(mMVPMatrix);
square3.draw(mMVPMatrix);
square4.draw(mMVPMatrix);
}
@Override
public void onSurfaceChanged(GL10 unused, int width, int height) {
// Adjust the viewport based on geometry changes,
// such as screen rotation
GLES20.glViewport(0, 0, width, height);
float ratio = (float) width / height;
// this projection matrix is applied to object coordinates
// in the onDrawFrame() method
Matrix.frustumM(mProjectionMatrix, 0, -ratio, ratio, -1, 1, 3, 7);
}
/**
* Utility method for compiling a OpenGL shader.
*
* <p><strong>Note:</strong> When developing shaders, use the checkGlError()
* method to debug shader coding errors.</p>
*
* @param type - Vertex or fragment shader type.
* @param shaderCode - String containing the shader code.
* @return - Returns an id for the shader.
*/
public static int loadShader(int type, String shaderCode){
// create a vertex shader type (GLES20.GL_VERTEX_SHADER)
// or a fragment shader type (GLES20.GL_FRAGMENT_SHADER)
int shader = GLES20.glCreateShader(type);
// add the source code to the shader and compile it
GLES20.glShaderSource(shader, shaderCode);
GLES20.glCompileShader(shader);
return shader;
}
/**
* Utility method for debugging OpenGL calls. Provide the name of the call
* just after making it:
*
* <pre>
* mColorHandle = GLES20.glGetUniformLocation(mProgram, "vColor");
* MyGLRenderer.checkGlError("glGetUniformLocation");</pre>
*
* If the operation is not successful, the check throws an error.
*
* @param glOperation - Name of the OpenGL call to check.
*/
public static void checkGlError(String glOperation) {
int error;
while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
Log.e(TAG, glOperation + ": glError " + error);
throw new RuntimeException(glOperation + ": glError " + error);
}
}
}
Sensor Listener Class:
package com.example.android.opengl;
import android.hardware.Sensor;
import android.hardware.SensorEvent;
import android.hardware.SensorEventListener;
import android.hardware.SensorManager;
import android.util.Log;
import java.util.ArrayList;
import java.util.Arrays;
public class IMUData implements SensorEventListener{
//create sensor manager
private static SensorManager sensorManager;
private final static String TAG = "IMUData";
//create variables for storing the most recent set of data for the calculations and initialize them to 0
public float[] locallacc = new float[] {0.0f, 0.0f, 0.0f};
public float[] acc = new float[] {0.0f, 0.0f, 0.0f};
public float[] gyro = new float[]{0.0f, 0.0f, 0.0f};
public float[] mag = new float[] {0.0f, 0.0f, 0.0f};
public float[] rot = new float[] {0.0f, 0.0f, 0.0f, 0.0f};
public float[] grav = new float[] {0.0f, 0.0f, 0.0f};
//these two need to have the size defined before they can be used
float[] lacc = new float[] {0.0f, 0.0f, 0.0f};
float[] angle = new float[] {0.0f, 0.0f, 0.0f};
public float theta = 0.0f; //this is for the heading (think compass)
public float[] RMatrix;
//public ArrayList<Float> l = new ArrayList<>(); // for dynamic sized arrays
//flags to keep orientation calculation from happening before all the data is collected
private boolean rot_flag = false;
@Override
public void onSensorChanged(SensorEvent event) {
//on sensor message, check what sensor it is and save data from sensor
switch (event.sensor.getType()){
case Sensor.TYPE_LINEAR_ACCELERATION:
locallacc = event.values;
//used in position estimates so left here for future compatibility
//change co-ordinate systems from local to global
lacc[0] = (float) ((locallacc[0]*Math.cos(theta)) + (locallacc[1]*Math.sin(theta)));
lacc[1] = (float) ((locallacc[1]*Math.cos(theta)) - (locallacc[0]*Math.sin(theta)));
lacc[2] = locallacc[2];
break;
case Sensor.TYPE_ACCELEROMETER:
acc = event.values;
break;
case Sensor.TYPE_GYROSCOPE:
gyro = event.values;
break;
case Sensor.TYPE_MAGNETIC_FIELD:
mag = event.values;
break;
case Sensor.TYPE_ROTATION_VECTOR:
rot_flag = true;
rot = event.values;
theta = 2 * (float)(Math.atan(rot[0]/rot[1]));
break;
case Sensor.TYPE_GRAVITY:
grav = event.values;
break;
default:
//hopefully will not hit this but hey let's catch all the mess-ups.
break;
}
//compute rotation angle (think compass)
boolean check = false;
float RMatrix[] = new float[9];
//if acceleromater and magnometer data are both not null
if(rot_flag) {
//see if the rotation matrix calculation works using data from accelerometer and magnometer
sensorManager.getRotationMatrixFromVector(RMatrix, rot);
}
if (check) {
//get euler angles
sensorManager.getOrientation(RMatrix, angle);
this.RMatrix = RMatrix;
}
}
@Override
public void onAccuracyChanged(Sensor sensor, int accuracy) {
//do nothing.
}
public void initSensorListeners(){
//setup sensor manager and sensor listeners
sensorManager=MyGLSurfaceView.getSensorManager();
//linear acceleration, basically acceleration sans gravity
sensorManager.registerListener(this,
sensorManager.getDefaultSensor(Sensor.TYPE_LINEAR_ACCELERATION),
sensorManager.SENSOR_DELAY_NORMAL);
//get gravity vector for the up vector
sensorManager.registerListener(this,
sensorManager.getDefaultSensor(Sensor.TYPE_GRAVITY),
sensorManager.SENSOR_DELAY_NORMAL);
//get data for the orientation setup later on
sensorManager.registerListener(this,
sensorManager.getDefaultSensor(Sensor.TYPE_ACCELEROMETER),
sensorManager.SENSOR_DELAY_NORMAL);
//gyroscope, this has been run through a few filters
sensorManager.registerListener(this,
sensorManager.getDefaultSensor(Sensor.TYPE_GYROSCOPE),
sensorManager.SENSOR_DELAY_NORMAL);
//magnometer, also run through a few filters
sensorManager.registerListener(this,
sensorManager.getDefaultSensor(Sensor.TYPE_MAGNETIC_FIELD),
sensorManager.SENSOR_DELAY_NORMAL);
//orientation vector, standard, relies on magnometer and gyro (I think, docs don't say)
sensorManager.registerListener(this,
sensorManager.getDefaultSensor(Sensor.TYPE_ROTATION_VECTOR),
sensorManager.SENSOR_DELAY_NORMAL);
}
//return functions
public float[] getLinAcc(){return locallacc;}
public float[] getAdjLinAcc(){return lacc;}
public float[] getAcc(){return acc;}
public float[] getGyro(){return gyro;}
public float[] getMag(){return mag;}
public float[] getRot(){return rot;}
public float[] getAngle(){return angle;}
public float getTheta(){return theta;}
public float[] getGravity(){return grav;}
public float[] getRotMat(){return RMatrix;}
}
