AR+ 实时音视频通话，虚拟与现实无缝结合

git clone git@github.com:pili-engineering/QNRTC-Android.git

git clone git@github.com:google-ar/arcore-android-sdk.git

implementation fileTree(include: ['*.jar'], dir: 'libs')

（1）<uses-permission android:name="android.permission.INTERNET"/>
（2）<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE"/>

public class HelloArActivity extends AppCompatActivity implements GLSurfaceView.Renderer, QNRoomEventListener {    private boolean mPublished = false; // 标识本地是否发布成功
    ...
    @Override    public void onJoinedRoom() {        mRTCManager.publish(); // 加入房间成功后，尝试发布    }
    @Override    public void onLocalPublished() {        mPublished = true; // 发布成功后，标识为 true    }
    ...}

protected void onCreate(Bundle savedInstanceState) {    ...    QNRTCSetting setting = new QNRTCSetting();    setting.setExternalVideoInputEnabled(true); // 开启外部视频导入功能
    mRTCManager.setRoomEventListener(this); // 设置房间事件监听器    mRTCManager.initialize(this, setting); // 七牛 RTN SDK 初始化
    mRTCManager.joinRoom(###Your Room Token###); // 通过 Room Token 加入指定房间}

public class HelloArActivity extends AppCompatActivity implements GLSurfaceView.Renderer, QNRoomEventListener {    /**     * 显示 Surface 创建完成时回调    **/    public void onSurfaceCreated(GL10 gl, EGLConfig config) {    }
    ...
   /**     * 显示 Surface 尺寸大小改变时回调    **/    public void onSurfaceChanged(GL10 gl, int width, int height) {    }
    ...
    /**     * 显示 Surface 创建完成时回调    **/    public void onDrawFrame(GL10 gl) {    }}

surfaceView = findViewById(R.id.surfaceview); // 从布局 xml 中解析 GLSurfaceView...surfaceView.setRenderer(this); // 设置 Renderer

protected void onResume() {    session = new Session(/* context= */ this); // AR 系统初始化    ...    session.resume(); // 开始 AR 会话，尝试开启摄像头，如摄像头被占用，会抛出 CameraNotAvailableException 异常}

// 绘制背景private final BackgroundRenderer backgroundRenderer = new BackgroundRenderer();// 绘制物体private final ObjectRenderer virtualObject = new ObjectRenderer();// 绘制物体阴影private final ObjectRenderer virtualObjectShadow = new ObjectRenderer();// 绘制平面private final PlaneRenderer planeRenderer = new PlaneRenderer();// 绘制云点private final PointCloudRenderer pointCloudRenderer = new PointCloudRenderer();
public void onDrawFrame(GL10 gl) {    frame = session.update(); // 获取摄像头原始数据帧（阻塞方法）
    // Handle one tap per frame.    handleTap(frame, camera); // 检测是否有平面点击事件，如有则在相应位置放置 Android 模型
    ...    // Draw background.    backgroundRenderer.draw(frame); // 将摄像头预览数据作为背景图绘制
    ...    // Visualize tracked points.    PointCloud pointCloud = frame.acquirePointCloud();    pointCloudRenderer.update(pointCloud);    pointCloudRenderer.draw(viewmtx, projmtx); // 绘制点云
    ...    // Visualize planes.    planeRenderer.drawPlanes(session.getAllTrackables(Plane.class), camera.getDisplayOrientedPose(), projmtx); // 绘制平面
    ...    // Update and draw the model and its shadow.    virtualObject.updateModelMatrix(anchorMatrix, scaleFactor);    virtualObjectShadow.updateModelMatrix(anchorMatrix, scaleFactor);    virtualObject.draw(viewmtx, projmtx, colorCorrectionRgba, coloredAnchor.color); // 绘制 Android 模型    virtualObjectShadow.draw(viewmtx, projmtx, colorCorrectionRgba, coloredAnchor.color); // 绘制 Android 模型的阴影}

public void onDrawFrame(GL10 gl) {    ...
    if (mPublished) { // 只在七牛 RTN 发布流成功后才导入 AR 数据        // 将 AR 增强画面 的数据从 GPU 中读取出来        GLES20.glReadPixels(0, 0, mSurfaceWidth, mSurfaceHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, mBufferRGBA);
        // RGBA 转为 NV21（篇幅原因，不在此展开算法）        mBufferNV21 = RGBAToNV21(mBufferRGBA, mSurfaceWidth, mSurfaceHeight);
        // 通过 "外部视频数据导入" 功能将 NV21 数据形式的 AR 增强画面 发布出去        mRTCManager.inputVideoFrame(mBufferNV21, mSurfaceWidth, mSurfaceHeight, 0, frame.getTimestamp());    }}