Citra-emu/citra-android
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

android: Add NDK camera implementation

Browse Source 
Not tested yet as my device doesn't support camera2...
This commit is contained in:
zhupengfei 2020-04-11 22:57:08 +08:00 committed by bunnei
parent fcf14b85fd
commit 232efb88d3
9 changed files with 481 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/android/app/src/main/java/org/citra/citra_emu/NativeLibrary.java
View File

@ -6,6 +6,7 @@
package org.citra.citra_emu;
import android.content.pm.PackageManager;
import android.content.res.Configuration;
import android.text.Html;
import android.text.method.LinkMovementMethod;
@ -20,9 +21,12 @@ import androidx.appcompat.app.AlertDialog;
import org.citra.citra_emu.activities.EmulationActivity;
import org.citra.citra_emu.utils.EmulationMenuSettings;
import org.citra.citra_emu.utils.Log;
import org.citra.citra_emu.utils.PermissionsHandler;
import java.lang.ref.WeakReference;
import static android.Manifest.permission.CAMERA;
/**
* Class which contains methods that interact
* with the native side of the Citra code.
@ -401,6 +405,38 @@ public final class NativeLibrary {
sEmulationActivity.clear();
}
private static final Object cameraPermissionLock = new Object();
private static boolean cameraPermissionGranted = false;
public static final int REQUEST_CODE_NATIVE_CAMERA = 800;
public static boolean RequestCameraPermission() {
final EmulationActivity emulationActivity = sEmulationActivity.get();
if (emulationActivity == null) {
Log.error("[NativeLibrary] EmulationActivity not present");
return false;
}
if (ContextCompat.checkSelfPermission(emulationActivity, CAMERA) == PackageManager.PERMISSION_GRANTED) {
// Permission already granted
return true;
}
emulationActivity.requestPermissions(new String[]{CAMERA}, REQUEST_CODE_NATIVE_CAMERA);
// Wait until result is returned
synchronized (cameraPermissionLock) {
try {
cameraPermissionLock.wait();
} catch (InterruptedException ignored) {}
}
return cameraPermissionGranted;
}
public static void CameraPermissionResult(boolean granted) {
cameraPermissionGranted = granted;
synchronized (cameraPermissionLock) {
cameraPermissionLock.notify();
}
}
/**
* Button type for use in onTouchEvent
*/

13
src/android/app/src/main/java/org/citra/citra_emu/activities/EmulationActivity.java
View File

@ -4,6 +4,7 @@ import android.app.Activity;
import android.app.PendingIntent;
import android.content.Intent;
import android.content.SharedPreferences;
import android.content.pm.PackageManager;
import android.os.Bundle;
import android.os.Handler;
import android.preference.PreferenceManager;
@ -228,6 +229,18 @@ public final class EmulationActivity extends AppCompatActivity {
}
}
@Override
public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
switch (requestCode) {
case NativeLibrary.REQUEST_CODE_NATIVE_CAMERA:
NativeLibrary.CameraPermissionResult(grantResults[0] == PackageManager.PERMISSION_GRANTED);
break;
default:
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
break;
}
}
private void enableFullscreenImmersive() {
// It would be nice to use IMMERSIVE_STICKY, but that doesn't show the toolbar.
mDecorView.setSystemUiVisibility(

2
src/android/app/src/main/java/org/citra/citra_emu/utils/PermissionsHandler.java
View File

@ -8,10 +8,12 @@ import android.os.Build;
import androidx.core.content.ContextCompat;
import androidx.fragment.app.FragmentActivity;
import static android.Manifest.permission.CAMERA;
import static android.Manifest.permission.WRITE_EXTERNAL_STORAGE;
public class PermissionsHandler {
public static final int REQUEST_CODE_WRITE_PERMISSION = 500;
public static final int REQUEST_CODE_CAMERA = 700;
// We use permissions acceptance as an indicator if this is a first boot for the user.
public static boolean isFirstBoot(final FragmentActivity activity) {

4
src/android/app/src/main/jni/CMakeLists.txt
View File

@ -3,6 +3,8 @@ add_library(main SHARED
applets/swkbd.h
button_manager.cpp
button_manager.h
camera/ndk_camera.cpp
camera/ndk_camera.h
camera/still_image_camera.cpp
camera/still_image_camera.h
config.cpp
@ -21,6 +23,6 @@ add_library(main SHARED
)
target_link_libraries(main PRIVATE common core input_common network)
target_link_libraries(main PRIVATE android jnigraphics EGL glad inih log yuv)
target_link_libraries(main PRIVATE android camera2ndk mediandk jnigraphics EGL glad inih log yuv)
set(CPACK_PACKAGE_EXECUTABLES ${CPACK_PACKAGE_EXECUTABLES} main)

346
src/android/app/src/main/jni/camera/ndk_camera.cpp Normal file
View File

@ -0,0 +1,346 @@
// Copyright 2020 Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#include <mutex>
#include <camera/NdkCameraCaptureSession.h>
#include <camera/NdkCameraDevice.h>
#include <camera/NdkCameraManager.h>
#include <camera/NdkCameraMetadata.h>
#include <camera/NdkCaptureRequest.h>
#include <libyuv.h>
#include <media/NdkImageReader.h>
#include "common/scope_exit.h"
#include "core/frontend/camera/blank_camera.h"
#include "jni/camera/ndk_camera.h"
#include "jni/id_cache.h"
namespace Camera::NDK {
/**
* Implementation detail of NDK camera interface, holding a ton of different structs.
* As long as the object lives, the camera is opened and capturing image. To turn off the camera,
* one needs to destruct the object.
* It captures at the maximum resolution supported by the device, capped at the 3DS camera's maximum
* resolution (640x480). The pixel format is I420.
*/
class CaptureSession final {
public:
explicit CaptureSession(ACameraManager* manager, const std::string& id);
private:
std::pair<int, int> selected_resolution{};
ACameraDevice_StateCallbacks device_callbacks{};
AImageReader_ImageListener listener{};
ACameraCaptureSession_stateCallbacks session_callbacks{};
std::array<ACaptureRequest*, 1> requests{};
#define MEMBER(type, name, func) \
struct type##Deleter { \
void operator()(type* ptr) { \
type##_##func(ptr); \
} \
}; \
std::unique_ptr<type, type##Deleter> name
MEMBER(ACameraDevice, device, close);
MEMBER(AImageReader, image_reader, delete);
// This window doesn't need to be destructed as it is managed by AImageReader
ANativeWindow* native_window{};
MEMBER(ACaptureSessionOutputContainer, outputs, free);
MEMBER(ACaptureSessionOutput, output, free);
MEMBER(ACameraOutputTarget, target, free);
MEMBER(ACaptureRequest, request, free);
// Put session last to close the session before we destruct everything else
MEMBER(ACameraCaptureSession, session, close);
#undef MEMBER
bool ready = false;
std::mutex data_mutex;
// Clang does not yet have shared_ptr to arrays support. Managed data are actually arrays.
std::array<std::shared_ptr<u8>, 3> data; // I420 format, planes are Y, U, V.
friend class Interface;
friend void ImageCallback(void* context, AImageReader* reader);
};
static void OnDisconnected(void* context, ACameraDevice* device) {
LOG_ERROR(Service_CAM, "Camera device disconnected");
// TODO: Do something here?
// CaptureSession* that = reinterpret_cast<CaptureSession*>(context);
// that->CloseCamera();
}
static void OnError(void* context, ACameraDevice* device, int error) {
LOG_ERROR(Service_CAM, "Camera device error {}", error);
// TODO: Do something here?
// CaptureSession* that = reinterpret_cast<CaptureSession*>(context);
// that->CloseCamera();
}
#define MEDIA_CALL(func) \
{ \
auto ret = func; \
if (ret != AMEDIA_OK) { \
LOG_ERROR(Service_CAM, "Call " #func " returned error {}", ret); \
return; \
} \
}
#define CAMERA_CALL(func) \
{ \
auto ret = func; \
if (ret != ACAMERA_OK) { \
LOG_ERROR(Service_CAM, "Call " #func " returned error {}", ret); \
return; \
} \
}
void ImageCallback(void* context, AImageReader* reader) {
AImage* image{};
MEDIA_CALL(AImageReader_acquireLatestImage(reader, &image))
SCOPE_EXIT({ AImage_delete(image); });
std::array<std::shared_ptr<u8>, 3> data;
for (const int plane : {0, 1, 2}) {
u8* ptr;
int size;
MEDIA_CALL(AImage_getPlaneData(image, plane, &ptr, &size));
data[plane].reset(new u8[size], std::default_delete<u8[]>());
std::memcpy(data[plane].get(), ptr, static_cast<size_t>(size));
}
{
CaptureSession* that = reinterpret_cast<CaptureSession*>(context);
std::lock_guard lock{that->data_mutex};
that->data = data;
}
}
#define CREATE(type, name, statement) \
{ \
type* raw; \
statement; \
name.reset(raw); \
}
CaptureSession::CaptureSession(ACameraManager* manager, const std::string& id) {
device_callbacks = {
/*context*/ nullptr,
/*onDisconnected*/ &OnDisconnected,
/*onError*/ &OnError,
};
CREATE(ACameraDevice, device,
CAMERA_CALL(ACameraManager_openCamera(manager, id.c_str(), &device_callbacks, &raw)));
ACameraMetadata* metadata;
CAMERA_CALL(ACameraManager_getCameraCharacteristics(manager, id.c_str(), &metadata));
ACameraMetadata_const_entry entry;
CAMERA_CALL(ACameraMetadata_getConstEntry(
metadata, ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, &entry));
selected_resolution = {};
for (std::size_t i = 0; i < entry.count; i += 4) {
// (format, width, height, input?)
if (entry.data.i32[i + 3] & ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_INPUT) {
// This is an input stream
continue;
}
int format = entry.data.i32[i + 0];
if (format == AIMAGE_FORMAT_YUV_420_888) {
int width = entry.data.i32[i + 1];
int height = entry.data.i32[i + 2];
if (width <= 640 && height <= 480) { // Maximum size the 3DS supports
selected_resolution = std::max(selected_resolution, std::make_pair(width, height));
}
}
}
ACameraMetadata_free(metadata);
if (selected_resolution == std::pair<int, int>{}) {
LOG_ERROR(Service_CAM, "Device does not support any YUV output format");
return;
}
CREATE(AImageReader, image_reader,
MEDIA_CALL(AImageReader_new(selected_resolution.first, selected_resolution.second,
AIMAGE_FORMAT_YUV_420_888, 4, &raw)));
listener = {
/*context*/ this,
/*onImageAvailable*/ &ImageCallback,
};
MEDIA_CALL(AImageReader_setImageListener(image_reader.get(), &listener));
MEDIA_CALL(AImageReader_getWindow(image_reader.get(), &native_window));
CREATE(ACaptureSessionOutput, output,
CAMERA_CALL(ACaptureSessionOutput_create(native_window, &raw)));
CREATE(ACaptureSessionOutputContainer, outputs,
CAMERA_CALL(ACaptureSessionOutputContainer_create(&raw)));
CAMERA_CALL(ACaptureSessionOutputContainer_add(outputs.get(), output.get()));
CREATE(ACameraCaptureSession, session,
CAMERA_CALL(ACameraDevice_createCaptureSession(device.get(), outputs.get(),
&session_callbacks, &raw)));
CREATE(ACaptureRequest, request,
CAMERA_CALL(ACameraDevice_createCaptureRequest(device.get(), TEMPLATE_PREVIEW, &raw)));
ANativeWindow_acquire(native_window);
CREATE(ACameraOutputTarget, target,
CAMERA_CALL(ACameraOutputTarget_create(native_window, &raw)));
CAMERA_CALL(ACaptureRequest_addTarget(request.get(), target.get()));
requests = {request.get()};
CAMERA_CALL(ACameraCaptureSession_setRepeatingRequest(session.get(), nullptr, 1,
requests.data(), nullptr));
ready = true;
}
#undef MEDIA_CALL
#undef CAMERA_CALL
#undef CREATE
Interface::Interface(Factory& factory_, const std::string& id_, const Service::CAM::Flip& flip_)
: factory(factory_), id(id_), flip(flip_) {}
Interface::~Interface() = default;
void Interface::StartCapture() {
session = factory.CreateCaptureSession(id);
}
void Interface::StopCapture() {
session.reset();
}
void Interface::SetResolution(const Service::CAM::Resolution& resolution_) {
resolution = resolution_;
}
void Interface::SetFlip(Service::CAM::Flip flip_) {
flip = flip_;
}
void Interface::SetFormat(Service::CAM::OutputFormat format_) {
format = format_;
}
std::vector<u16> Interface::ReceiveFrame() {
std::array<std::shared_ptr<u8>, 3> data;
{
std::lock_guard lock{session->data_mutex};
data = session->data;
}
auto [width, height] = session->selected_resolution;
int crop_width{}, crop_height{};
if (resolution.width * height > resolution.height * width) {
crop_width = width;
crop_height = width * resolution.height / resolution.width;
} else {
crop_height = height;
crop_width = height * resolution.width / resolution.height;
}
int crop_x = (width - crop_width) / 2;
int crop_y = (height - crop_height) / 2;
int offset = crop_y * width + crop_x;
std::vector<u8> scaled_y(resolution.width * resolution.height);
std::vector<u8> scaled_u(resolution.width * resolution.height / 4ul);
std::vector<u8> scaled_v(resolution.width * resolution.height / 4ul);
// Crop and scale
libyuv::I420Scale(data[0].get() + offset, width, data[1].get() + offset / 4, width / 4,
data[2].get() + offset / 4, width / 4, crop_width, crop_height,
scaled_y.data(), resolution.width, scaled_u.data(), resolution.width / 4,
scaled_v.data(), resolution.width / 4, resolution.width, resolution.height,
libyuv::kFilterBilinear);
// TODO: Record and apply flip
std::vector<u16> output(resolution.width * resolution.height);
if (format == Service::CAM::OutputFormat::RGB565) {
libyuv::I420ToRGB565(scaled_y.data(), resolution.width, scaled_u.data(),
resolution.width / 4, scaled_v.data(), resolution.width / 4,
reinterpret_cast<u8*>(output.data()), resolution.width * 2,
resolution.width, resolution.height);
} else {
libyuv::I420ToYUY2(scaled_y.data(), resolution.width, scaled_u.data(), resolution.width / 4,
scaled_v.data(), resolution.width / 4,
reinterpret_cast<u8*>(output.data()), resolution.width * 2,
resolution.width, resolution.height);
}
return output;
}
bool Interface::IsPreviewAvailable() {
return session && session->ready;
}
Factory::Factory() = default;
Factory::~Factory() = default;
std::shared_ptr<CaptureSession> Factory::CreateCaptureSession(const std::string& id) {
if (opened_camera_map.count(id) && !opened_camera_map.at(id).expired()) {
return opened_camera_map.at(id).lock();
}
const auto& session = std::make_shared<CaptureSession>(manager.get(), id);
opened_camera_map.insert_or_assign(id, session);
return session;
}
std::unique_ptr<CameraInterface> Factory::Create(const std::string& config,
const Service::CAM::Flip& flip) {
if (!manager) {
JNIEnv* env = IDCache::GetEnvForThread();
jboolean result = env->CallStaticBooleanMethod(IDCache::GetNativeLibraryClass(),
IDCache::GetRequestCameraPermission());
if (result != JNI_TRUE) {
LOG_ERROR(Service_CAM, "Camera permissions denied");
return std::make_unique<Camera::BlankCamera>();
}
manager.reset(ACameraManager_create());
}
ACameraIdList* id_list = nullptr;
auto ret = ACameraManager_getCameraIdList(manager.get(), &id_list);
if (ret != ACAMERA_OK) {
LOG_ERROR(Service_CAM, "Failed to get camera ID list: ret {}", ret);
return std::make_unique<Camera::BlankCamera>();
}
SCOPE_EXIT({ ACameraManager_deleteCameraIdList(id_list); });
if (id_list->numCameras <= 0) {
LOG_ERROR(Service_CAM, "No camera devices found");
return std::make_unique<Camera::BlankCamera>();
}
if (config.empty()) {
LOG_WARNING(Service_CAM, "Camera ID not set, using default camera");
return std::make_unique<Interface>(*this, id_list->cameraIds[0], flip);
}
for (int i = 0; i < id_list->numCameras; ++i) {
const char* id = id_list->cameraIds[i];
if (config == id) {
return std::make_unique<Interface>(*this, id, flip);
}
}
LOG_ERROR(Service_CAM, "Camera ID {} not found", config);
return std::make_unique<Camera::BlankCamera>();
}
} // namespace Camera::NDK

71
src/android/app/src/main/jni/camera/ndk_camera.h Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2020 Citra Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#pragma once
#include <memory>
#include <unordered_map>
#include <camera/NdkCameraManager.h>
#include "common/common_types.h"
#include "core/frontend/camera/factory.h"
#include "core/frontend/camera/interface.h"
namespace Camera::NDK {
class CaptureSession;
class Factory;
class Interface : public CameraInterface {
public:
Interface(Factory& factory, const std::string& id, const Service::CAM::Flip& flip);
~Interface() override;
void StartCapture() override;
void StopCapture() override;
void SetResolution(const Service::CAM::Resolution& resolution) override;
void SetFlip(Service::CAM::Flip flip) override;
void SetEffect(Service::CAM::Effect effect) override{};
void SetFormat(Service::CAM::OutputFormat format) override;
void SetFrameRate(Service::CAM::FrameRate frame_rate) override{};
std::vector<u16> ReceiveFrame() override;
bool IsPreviewAvailable() override;
private:
// jstring path;
Factory& factory;
std::shared_ptr<CaptureSession> session;
std::string id;
Service::CAM::Resolution resolution;
Service::CAM::Flip flip;
Service::CAM::OutputFormat format;
// std::vector<u16> image; // Data fetched from the frontend
// bool opened{}; // Whether the camera was successfully opened
};
class Factory final : public CameraFactory {
public:
explicit Factory();
~Factory() override;
std::unique_ptr<CameraInterface> Create(const std::string& config,
const Service::CAM::Flip& flip) override;
private:
std::shared_ptr<CaptureSession> CreateCaptureSession(const std::string& id);
// The session is cached, to avoid opening the same camera twice.
// This is weak_ptr so that the session is destructed when all cameras are closed
std::unordered_map<std::string, std::weak_ptr<CaptureSession>> opened_camera_map;
struct ACameraManagerDeleter {
void operator()(ACameraManager* manager) {
ACameraManager_delete(manager);
}
};
std::unique_ptr<ACameraManager, ACameraManagerDeleter> manager;
friend class Interface;
};
} // namespace Camera::NDK

7
src/android/app/src/main/jni/id_cache.cpp
View File

@ -24,6 +24,7 @@ static jmethodID s_alert_prompt_button;
static jmethodID s_is_portrait_mode;
static jmethodID s_landscape_screen_layout;
static jmethodID s_exit_emulation_activity;
static jmethodID s_request_camera_permission;
namespace IDCache {
@ -74,6 +75,10 @@ jmethodID GetExitEmulationActivity() {
return s_exit_emulation_activity;
}
jmethodID GetRequestCameraPermission() {
return s_request_camera_permission;
}
} // namespace IDCache
#ifdef __cplusplus
@ -112,6 +117,8 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) {
env->GetStaticMethodID(s_native_library_class, "landscapeScreenLayout", "()I");
s_exit_emulation_activity =
env->GetStaticMethodID(s_native_library_class, "exitEmulationActivity", "(I)V");
s_request_camera_permission =
env->GetStaticMethodID(s_native_library_class, "RequestCameraPermission", "()Z");
SoftwareKeyboard::InitJNI(env);
Camera::StillImage::InitJNI(env);

1
src/android/app/src/main/jni/id_cache.h
View File

@ -16,5 +16,6 @@ jmethodID GetAlertPromptButton();
jmethodID GetIsPortraitMode();
jmethodID GetLandscapeScreenLayout();
jmethodID GetExitEmulationActivity();
jmethodID GetRequestCameraPermission();
} // namespace IDCache

2
src/android/app/src/main/jni/native.cpp
View File

@ -22,6 +22,7 @@
#include "core/settings.h"
#include "jni/applets/swkbd.h"
#include "jni/button_manager.h"
#include "jni/camera/ndk_camera.h"
#include "jni/camera/still_image_camera.h"
#include "jni/config.h"
#include "jni/emu_window/emu_window.h"
@ -109,6 +110,7 @@ static Core::System::ResultStatus RunCitra(const std::string& filepath) {
Settings::Apply();
Camera::RegisterFactory("image", std::make_unique<Camera::StillImage::Factory>());
Camera::RegisterFactory("ndk", std::make_unique<Camera::NDK::Factory>());
// Register frontend applets
Frontend::RegisterDefaultApplets();