feat: camera pass-through mode

code/mobile/android/PhoneVR/app/CMakeLists.txt

@@ -21,6 +21,7 @@ file(GLOB_RECURSE GVR_SRC
 
 add_library(native-lib-alvr SHARED
     src/main/cpp/alvr_main.cpp
+    src/main/cpp/passthrough.cpp
     ${LIB_SRC}
 )
 

code/mobile/android/PhoneVR/app/build.gradle

@@ -95,7 +95,7 @@ android {
     }
     project.gradle.taskGraph.whenReady {
         android.productFlavors.all { flavor ->
-            // Capitalize (as Gralde is case-sensitive).
+            // Capitalize (as Gradle is case-sensitive).
             def flavorName = flavor.name.substring(0, 1).toUpperCase() + flavor.name.substring(1)
 
             // At last, configure.
@@ -132,6 +132,10 @@ android {
     }
 }
 
+repositories {
+    maven { url 'https://jitpack.io' }
+}
+
 dependencies {
     implementation fileTree(dir: 'libs', include: ['*.jar'])
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
@@ -140,6 +144,7 @@ dependencies {
     implementation 'com.google.android.material:material:1.2.1'
 
     implementation "androidx.tracing:tracing:1.1.0"
+    implementation 'androidx.preference:preference:1.2.1'
     testImplementation 'junit:junit:4.13.2'
     testImplementation 'junit:junit:4.12'
     androidTestImplementation "androidx.test.ext:junit-ktx:$extJUnitVersion"
@@ -159,6 +164,7 @@ dependencies {
 
     implementation platform('com.google.firebase:firebase-bom:26.0.0')
     implementation 'com.google.firebase:firebase-analytics-ktx'
+    implementation 'com.github.PhilJay:MPAndroidChart:v3.1.0'
 }
 
 // The dependencies for NDK builds live inside the .aar files so they need to

code/mobile/android/PhoneVR/app/src/main/AndroidManifest.xml

@@ -46,6 +46,10 @@
         android:theme="@style/AppTheme"
         tools:replace="android:extractNativeLibs"
         android:extractNativeLibs="true">
+        <activity
+            android:name=".PassthroughSettingsActivity"
+            android:exported="false"
+            android:label="@string/title_activity_passthrough_settings" />
         <activity
             android:name=".InitActivity"
             android:label="@string/app_name"

code/mobile/android/PhoneVR/app/src/main/cpp/alvr_main.cpp

@@ -11,6 +11,7 @@
 #include <vector>
 
 #include "nlohmann/json.hpp"
+#include "passthrough.h"
 #include "utils.h"
 
 using namespace nlohmann;
@@ -38,6 +39,7 @@ struct NativeContext {
 
     bool running = false;
     bool streaming = false;
+    PassthroughInfo passthroughInfo = {};
     std::thread inputThread;
 
     // Une one texture per eye, no need for swapchains.
@@ -188,6 +190,10 @@ extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_initia
 
     Cardboard_initializeAndroid(CTX.javaVm, CTX.javaContext);
     CTX.headTracker = CardboardHeadTracker_create();
+
+    CTX.passthroughInfo.screenWidth = &(CTX.screenWidth);
+    CTX.passthroughInfo.screenHeight = &(CTX.screenHeight);
+    passthrough_createPlane(&(CTX.passthroughInfo));
 }
 
 extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_destroyNative(JNIEnv *,
@@ -233,11 +239,24 @@ extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_pauseN
     CardboardHeadTracker_pause(CTX.headTracker);
 }
 
+extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_Passthrough_setPassthroughActiveNative(
+    JNIEnv *, jobject, jboolean activate) {
+    CTX.passthroughInfo.enabled = activate;
+    CTX.renderingParamsChanged = true;
+}
+
 extern "C" JNIEXPORT void JNICALL
+Java_viritualisres_phonevr_Passthrough_setPassthroughSizeNative(JNIEnv *, jobject, jfloat size) {
+    CTX.passthroughInfo.passthroughSize = size;
+    passthrough_createPlane(&(CTX.passthroughInfo));
+}
+
+extern "C" JNIEXPORT jint JNICALL
 Java_viritualisres_phonevr_ALVRActivity_surfaceCreatedNative(JNIEnv *, jobject) {
     alvr_initialize_opengl();
-
+    GLuint camTex = passthrough_init(&(CTX.passthroughInfo));
     CTX.glContextRecreated = true;
+    return camTex;
 }
 
 extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_setScreenResolutionNative(
@@ -308,38 +327,41 @@ extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_render
         if (CTX.renderingParamsChanged && !CTX.glContextRecreated) {
             info("Pausing ALVR since glContext is not recreated, deleting textures");
             alvr_pause_opengl();
-
+            passthrough_cleanup(&(CTX.passthroughInfo));
             GL(glDeleteTextures(2, CTX.lobbyTextures));
         }
 
         if (CTX.renderingParamsChanged || CTX.glContextRecreated) {
-            info("Rebuilding, binding textures, Resuming ALVR since glContextRecreated %b, "
-                 "renderingParamsChanged %b",
-                 CTX.renderingParamsChanged,
-                 CTX.glContextRecreated);
-            GL(glGenTextures(2, CTX.lobbyTextures));
-
-            for (auto &lobbyTexture : CTX.lobbyTextures) {
-                GL(glBindTexture(GL_TEXTURE_2D, lobbyTexture));
-                GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE));
-                GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE));
-                GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR));
-                GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR));
-                GL(glTexImage2D(GL_TEXTURE_2D,
-                                0,
-                                GL_RGB,
-                                CTX.screenWidth / 2,
-                                CTX.screenHeight,
-                                0,
-                                GL_RGB,
-                                GL_UNSIGNED_BYTE,
-                                nullptr));
-            }
-
-            const uint32_t *targetViews[2] = {(uint32_t *) &CTX.lobbyTextures[0],
-                                              (uint32_t *) &CTX.lobbyTextures[1]};
-            alvr_resume_opengl(CTX.screenWidth / 2, CTX.screenHeight, targetViews, 1, true);
+            if (CTX.passthroughInfo.enabled) {
+                passthrough_setup(&(CTX.passthroughInfo));
+            } else {
+                info("Rebuilding, binding textures, Resuming ALVR since glContextRecreated %b, "
+                     "renderingParamsChanged %b",
+                     CTX.renderingParamsChanged,
+                     CTX.glContextRecreated);
+                GL(glGenTextures(2, CTX.lobbyTextures));
+
+                for (auto &lobbyTexture : CTX.lobbyTextures) {
+                    GL(glBindTexture(GL_TEXTURE_2D, lobbyTexture));
+                    GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE));
+                    GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE));
+                    GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR));
+                    GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR));
+                    GL(glTexImage2D(GL_TEXTURE_2D,
+                                    0,
+                                    GL_RGB,
+                                    CTX.screenWidth / 2,
+                                    CTX.screenHeight,
+                                    0,
+                                    GL_RGB,
+                                    GL_UNSIGNED_BYTE,
+                                    nullptr));
+                }
 
+                const uint32_t *targetViews[2] = {(uint32_t *) &CTX.lobbyTextures[0],
+                                                  (uint32_t *) &CTX.lobbyTextures[1]};
+                alvr_resume_opengl(CTX.screenWidth / 2, CTX.screenHeight, targetViews, 1, true);
+            }
             CTX.renderingParamsChanged = false;
             CTX.glContextRecreated = false;
         }
@@ -481,7 +503,9 @@ extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_render
             viewsDesc.bottom_v = 0.0;
         }
 
-        if (CTX.streaming) {
+        if (CTX.passthroughInfo.enabled) {
+            passthrough_render(&(CTX.passthroughInfo), viewsDescs);
+        } else if (CTX.streaming) {
             void *streamHardwareBuffer = nullptr;
 
             AlvrViewParams dummyViewParams;

code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp

@@ -0,0 +1,198 @@
+#include "alvr_client_core.h"
+// clang-format off
+#include <GLES3/gl3.h>
+#include <GLES2/gl2ext.h>
+// clang-format on
+#include <vector>
+
+#include "passthrough.h"
+#include "utils.h"
+
+GLuint LoadGLShader(GLenum type, const char *shader_source) {
+    GLuint shader = GL(glCreateShader(type));
+
+    GL(glShaderSource(shader, 1, &shader_source, nullptr));
+    GL(glCompileShader(shader));
+
+    // Get the compilation status.
+    GLint compile_status;
+    GL(glGetShaderiv(shader, GL_COMPILE_STATUS, &compile_status));
+
+    // If the compilation failed, delete the shader and show an error.
+    if (compile_status == 0) {
+        GLint info_len = 0;
+        GL(glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &info_len));
+        if (info_len == 0) {
+            return 0;
+        }
+
+        std::vector<char> info_string(info_len);
+        GL(glGetShaderInfoLog(shader, info_string.size(), nullptr, info_string.data()));
+        // LOGE("Could not compile shader of type %d: %s", type, info_string.data());
+        GL(glDeleteShader(shader));
+        return 0;
+    } else {
+        return shader;
+    }
+}
+
+namespace {
+    // Simple shaders to render camera Texture files without any lighting.
+    constexpr const char *camVertexShader =
+        R"glsl(
+    uniform mat4 u_MVP;
+    attribute vec4 a_Position;
+    attribute vec2 a_UV;
+    varying vec2 v_UV;
+
+    void main() {
+      v_UV = a_UV;
+      gl_Position = a_Position;
+    })glsl";
+
+    constexpr const char *camFragmentShader =
+        R"glsl(
+    #extension GL_OES_EGL_image_external : require
+    precision mediump float;
+    varying vec2 v_UV;
+    uniform samplerExternalOES sTexture;
+    void main() {
+        gl_FragColor = texture2D(sTexture, v_UV);
+    })glsl";
+
+    static int passthroughProgram_ = 0;
+    static int texturePositionParam_ = 0;
+    static int textureUvParam_ = 0;
+    static int textureMvpParam_ = 0;
+
+    float passthroughTexCoords[] = {0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0};
+}   // namespace
+
+void passthrough_createPlane(PassthroughInfo *info) {
+    float size = info->passthroughSize;
+    float x0 = -size, y0 = size;    // Top left
+    float x1 = size, y1 = size;     // Top right
+    float x2 = size, y2 = -size;    // Bottom right
+    float x3 = -size, y3 = -size;   // Bottom left
+
+    info->passthroughVertices[0] = x3;
+    info->passthroughVertices[1] = y3;
+    info->passthroughVertices[2] = x2;
+    info->passthroughVertices[3] = y2;
+    info->passthroughVertices[4] = x0;
+    info->passthroughVertices[5] = y0;
+    info->passthroughVertices[6] = x1;
+    info->passthroughVertices[7] = y1;
+}
+
+GLuint passthrough_init(PassthroughInfo *info) {
+    const int obj_vertex_shader = LoadGLShader(GL_VERTEX_SHADER, camVertexShader);
+    const int obj_fragment_shader = LoadGLShader(GL_FRAGMENT_SHADER, camFragmentShader);
+
+    passthroughProgram_ = GL(glCreateProgram());
+    GL(glAttachShader(passthroughProgram_, obj_vertex_shader));
+    GL(glAttachShader(passthroughProgram_, obj_fragment_shader));
+    GL(glLinkProgram(passthroughProgram_));
+
+    GL(glUseProgram(passthroughProgram_));
+    texturePositionParam_ = GL(glGetAttribLocation(passthroughProgram_, "a_Position"));
+    textureUvParam_ = GL(glGetAttribLocation(passthroughProgram_, "a_UV"));
+    textureMvpParam_ = GL(glGetUniformLocation(passthroughProgram_, "u_MVP"));
+
+    GL(glGenTextures(1, &(info->cameraTexture)));
+    GL(glActiveTexture(GL_TEXTURE0));
+
+    GL(glBindTexture(GL_TEXTURE_EXTERNAL_OES, info->cameraTexture));
+    GL(glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST));
+    GL(glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR));
+    GL(glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE));
+    GL(glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE));
+
+    return info->cameraTexture;
+}
+
+void passthrough_cleanup(PassthroughInfo *info) {
+    if (info->passthroughDepthRenderBuffer != 0) {
+        GL(glDeleteRenderbuffers(1, &(info->passthroughDepthRenderBuffer)));
+        info->passthroughDepthRenderBuffer = 0;
+    }
+    if (info->passthroughFramebuffer != 0) {
+        GL(glDeleteFramebuffers(1, &info->passthroughFramebuffer));
+        info->passthroughFramebuffer = 0;
+    }
+    if (info->passthroughTexture != 0) {
+        GL(glDeleteTextures(1, &(info->passthroughTexture)));
+        info->passthroughTexture = 0;
+    }
+}
+
+void passthrough_setup(PassthroughInfo *info) {
+    // Create render texture.
+    GL(glGenTextures(1, &(info->passthroughTexture)));
+    GL(glBindTexture(GL_TEXTURE_2D, info->passthroughTexture));
+    GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR));
+    GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR));
+    GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE));
+    GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE));
+
+    GL(glTexImage2D(GL_TEXTURE_2D,
+                    0,
+                    GL_RGB,
+                    *(info->screenWidth),
+                    *(info->screenHeight),
+                    0,
+                    GL_RGB,
+                    GL_UNSIGNED_BYTE,
+                    0));
+
+    // Generate depth buffer to perform depth test.
+    GL(glGenRenderbuffers(1, &(info->passthroughDepthRenderBuffer)));
+    GL(glBindRenderbuffer(GL_RENDERBUFFER, info->passthroughDepthRenderBuffer));
+    GL(glRenderbufferStorage(
+        GL_RENDERBUFFER, GL_DEPTH_COMPONENT16, *(info->screenWidth), *(info->screenHeight)));
+
+    // Create render target.
+    GL(glGenFramebuffers(1, &(info->passthroughFramebuffer)));
+    GL(glBindFramebuffer(GL_FRAMEBUFFER, info->passthroughFramebuffer));
+    GL(glFramebufferTexture2D(
+        GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, info->passthroughTexture, 0));
+    GL(glFramebufferRenderbuffer(
+        GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, info->passthroughDepthRenderBuffer));
+}
+
+void passthrough_render(PassthroughInfo *info, CardboardEyeTextureDescription viewsDescs[]) {
+    GL(glBindFramebuffer(GL_FRAMEBUFFER, info->passthroughFramebuffer));
+
+    GL(glEnable(GL_DEPTH_TEST));
+    GL(glEnable(GL_CULL_FACE));
+    GL(glDisable(GL_SCISSOR_TEST));
+    GL(glEnable(GL_BLEND));
+    GL(glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA));
+    GL(glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT));
+
+    // Draw Passthrough video for each eye
+    for (int eye = 0; eye < 2; ++eye) {
+        GL(glViewport(eye == kLeft ? 0 : *(info->screenWidth) / 2,
+                      0,
+                      *(info->screenWidth) / 2,
+                      *(info->screenHeight)));
+
+        GL(glUseProgram(passthroughProgram_));
+        GL(glActiveTexture(GL_TEXTURE0));
+        GL(glBindTexture(GL_TEXTURE_EXTERNAL_OES, info->cameraTexture));
+
+        // Draw Mesh
+        GL(glEnableVertexAttribArray(texturePositionParam_));
+        GL(glVertexAttribPointer(
+            texturePositionParam_, 2, GL_FLOAT, false, 0, info->passthroughVertices));
+        GL(glEnableVertexAttribArray(textureUvParam_));
+        GL(glVertexAttribPointer(textureUvParam_, 2, GL_FLOAT, false, 0, passthroughTexCoords));
+
+        GL(glDrawArrays(GL_TRIANGLE_STRIP, 0, 4));
+
+        viewsDescs[eye].left_u = 0.5 * eye;          // 0 for left, 0.5 for right
+        viewsDescs[eye].right_u = 0.5 + 0.5 * eye;   // 0.5 for left, 1.0 for right
+    }
+    viewsDescs[0].texture = info->passthroughTexture;
+    viewsDescs[1].texture = info->passthroughTexture;
+}