Movement SDK for OpenXRでのフェイストラッキング

adb uninstall com.oculus.sdk.xrface
cd XrSamples/XrFace/Projects/Android
../../../../gradlew installDebug

<manifest xmlns:android="http://schemas.android.com/apk/res/android" ><!-- Tell the system this app can handle face tracking --><uses-feature android:name="oculus.software.face_tracking" android:required="true" /><uses-permission android:name="com.oculus.permission.FACE_TRACKING" /><!-- Tell the system this app can use audio for face tracking --><uses-permission android:name="android.permission.RECORD_AUDIO" /><!-- Tell the system this app can handle eye tracking --><uses-feature android:name="oculus.software.eye_tracking" android:required="true" /><uses-permission android:name="com.oculus.permission.EYE_TRACKING" />

  ....
</manifest>

  private static final String PERMISSION_FACE_TRACKING = "com.oculus.permission.FACE_TRACKING";
  private static final String PERMISSION_EYE_TRACKING = "com.oculus.permission.EYE_TRACKING";
  private static final String PERMISSION_RECORD_AUDIO = "android.permission.RECORD_AUDIO";
  private static final int REQUEST_CODE_PERMISSION_FACE_AND_EYE_TRACKING = 1;

   @Override
  protected void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);

    requestFaceAndEyeTrackingPermissionIfNeeded();
  }

  private void requestFaceAndEyeTrackingPermissionIfNeeded() {
    List<String> permissionsToRequest = new ArrayList<>();
    if (checkSelfPermission(PERMISSION_EYE_TRACKING) != PackageManager.PERMISSION_GRANTED) {
      permissionsToRequest.add(PERMISSION_EYE_TRACKING);
    }
    if (checkSelfPermission(PERMISSION_FACE_TRACKING) != PackageManager.PERMISSION_GRANTED) {
      permissionsToRequest.add(PERMISSION_FACE_TRACKING);
    }
    if (checkSelfPermission(PERMISSION_RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) {
      permissionsToRequest.add(PERMISSION_RECORD_AUDIO);
    }

    if (!permissionsToRequest.isEmpty()) {
      String[] permissionsAsArray =
          permissionsToRequest.toArray(new String[permissionsToRequest.size()]);
      requestPermissions(permissionsAsArray, REQUEST_CODE_PERMISSION_FACE_AND_EYE_TRACKING);
    }
  }

XrInstance instance;
XrSystemId system;
XrSession session;
XrSpace sceneSpace;

std::vector<const char*> extensions;

XrInstance instance = XR_NULL_HANDLE;

    XrInstanceCreateInfo instanceCreateInfo = {XR_TYPE_INSTANCE_CREATE_INFO};
    ....
    instanceCreateInfo.enabledExtensionCount = extensions.size();
    instanceCreateInfo.enabledExtensionNames = extensions.data();

    ....
    OXR(initResult = xrCreateInstance(&instanceCreateInfo, &instance));

   #include <openxr/openxr.h>

    XrSession Session;
    XrSpace StageSpace;

typedef struct XrSystemFaceTrackingProperties2FB {
    XrStructureType    type;
    void* XR_MAY_ALIAS next;
    XrBool32           supportsVisualFaceTracking;
    XrBool32           supportsAudioFaceTracking;
} XrSystemFaceTrackingProperties2FB;

    XrSystemFaceTrackingProperties2FB faceTrackingSystemProperties{
        XR_TYPE_SYSTEM_FACE_TRACKING_PROPERTIES2_FB};
    XrSystemProperties systemProperties{
        XR_TYPE_SYSTEM_PROPERTIES, &faceTrackingSystemProperties};
    OXR(xrGetSystemProperties(GetInstance(), GetSystemId(), &systemProperties));
    if (faceTrackingSystemProperties.supportsAudioFaceTracking ||
        faceTrackingSystemProperties.supportsVisualFaceTracking) {
        // face tracking is supported!
    }

    PFN_xrCreateFaceTracker2FB xrCreateFaceTrackerFB_ = nullptr;
    PFN_xrDestroyFaceTracker2FB xrDestroyFaceTrackerFB_ = nullptr;
    PFN_xrGetFaceExpressionWeights2FB xrGetFaceExpressionWeightsFB_ = nullptr;

    OXR(xrGetInstanceProcAddr(
        GetInstance(),
        "xrCreateFaceTracker2FB",
        (PFN_xrVoidFunction*)(&xrCreateFaceTrackerFB_)));
    OXR(xrGetInstanceProcAddr(
        GetInstance(),
        "xrDestroyFaceTracker2FB",
        (PFN_xrVoidFunction*)(&xrDestroyFaceTrackerFB_)));
    OXR(xrGetInstanceProcAddr(
        GetInstance(),
        "xrGetFaceExpressionWeights2FB",
        (PFN_xrVoidFunction*)(&xrGetFaceExpressionWeightsFB_)));

XrResult xrCreateFaceTracker2FB(
   XrSession session,
   const XrFaceTrackerCreateInfo2FB* createInfo,
   XrFaceTracker2FB* faceTracker);

    XrFaceTracker2FB faceTracker_ = XR_NULL_HANDLE;

    XrFaceTrackerCreateInfo2FB createInfo{XR_TYPE_FACE_TRACKER_CREATE_INFO2_FB};
    createInfo.faceExpressionSet = XR_FACE_EXPRESSION_SET2_DEFAULT_FB;
    createInfo.requestedDataSourceCount = 2;
    XrFaceTrackingDataSource2FB dataSources[2] = {
        XR_FACE_TRACKING_DATA_SOURCE2_VISUAL_FB,
        XR_FACE_TRACKING_DATA_SOURCE2_AUDIO_FB};
    createInfo.requestedDataSources = dataSources;

    OXR(xrCreateFaceTracker2FB_(GetSession(), &createInfo, &faceTracker_));

    float weights_[XR_FACE_EXPRESSION2_COUNT_FB] = {};
    float confidence_[XR_FACE_CONFIDENCE2_COUNT_FB] = {};

XrResult XRAPI_CALL xrGetFaceExpressionWeights2FB(
   XrFaceTracker2FB faceTracker,
   const XrFaceExpressionInfo2FB* expressionInfo,
   XrFaceExpressionWeights2FB* expressionWeights);

typedef struct XrFaceExpressionInfo2FB {
   XrStructureType type;
   const void*     XR_MAY_ALIAS next;
   XrTime          time;
} XrFaceExpressionInfo2FB;

typedef struct XrFaceExpressionWeights2FB {
    XrStructureType             type;
    void* XR_MAY_ALIAS          next;
    uint32_t                    weightCount;
    float*                      weights;
    uint32_t                    confidenceCount;
    float*                      confidences;
    XrBool32                    isValid;
    XrBool32                    isEyeFollowingBlendshapesValid;
    XrFaceTrackingDataSource2FB dataSource;
    XrTime                      time;
} XrFaceExpressionWeights2FB;

    XrFaceExpressionWeights2FB expressionWeights{XR_TYPE_FACE_EXPRESSION_WEIGHTS2_FB};
    expressionWeights.next = nullptr;
    expressionWeights.weights = weights_;
    expressionWeights.confidences = confidence_;
    expressionWeights.weightCount = XR_FACE_EXPRESSION2_COUNT_FB;
    expressionWeights.confidenceCount = XR_FACE_CONFIDENCE2_COUNT_FB;

    XrFaceExpressionInfo2FB expressionInfo{XR_TYPE_FACE_EXPRESSION_INFO2_FB};
    expressionInfo.time = GetPredictedDisplayTime();

    OXR(xrGetFaceExpressionWeights2FB_(faceTracker_, &expressionInfo, &expressionWeights));

    for (uint32_t i = 0; i < XR_FACE_EXPRESSION2_COUNT_FB; ++i) {
        // weights_[i] contains one specific weight
        ....
    }

    OXR(xrDestroyFaceTracker2FB_(faceTracker_));

typedef struct XrSystemFaceTrackingVisemesPropertiesMETA {
    XrStructureType    type;
    void*              next;
    XrBool32           supportsVisemes;
} XrSystemFaceTrackingVisemesPropertiesMETA;

    XrSystemFaceTrackingVisemesPropertiesMETA faceTrackingVisemesSystemProperties{
        XR_TYPE_SYSTEM_FACE_TRACKING_VISEMES_PROPERTIES_META};
    XrSystemProperties systemProperties{XR_TYPE_SYSTEM_PROPERTIES,
        &faceTrackingVisemesSystemProperties};
    OXR(xrGetSystemProperties(instance, systemId, &systemProperties));
    if (faceTrackingVisemesSystemProperties.supportsVisemes) {
        // visemes are supported!
    }

typedef struct XrFaceTrackingVisemesMETA {
    XrStructureType    type;
    const void*        next;
    XrBool32           isValid;
    float              visemes[XR_FACE_TRACKING_VISEME_COUNT_META];
} XrFaceTrackingVisemesMETA;

    XrFaceExpressionWeights2FB expressionWeights{XR_TYPE_FACE_EXPRESSION_WEIGHTS2_FB};
    expressionWeights.weightCount = 0;
    expressionWeights.confidenceCount = 0;

    XrFaceTrackingVisemesMETA visemeInfo{XR_TYPE_FACE_TRACKING_VISEMES_META};
    expressionWeights.next = &visemeInfo;

    XrFaceExpressionInfo2FB expressionInfo{XR_TYPE_FACE_EXPRESSION_INFO2_FB};
    expressionInfo.time = GetPredictedDisplayTime();

    OXR(xrGetFaceExpressionWeights2FB_(faceTracker_, &expressionInfo, &expressionWeights));

    if (visemeInfo.isValid) {
        for (uint32_t i = 0; i < XR_FACE_TRACKING_VISEME_COUNT_META; ++i) {
            // visemeInfo.visemes[i] contains a weight of specific visemes
        }
    }