diff --git a/src/android/app/src/main/java/org/citra/citra_emu/NativeLibrary.java b/src/android/app/src/main/java/org/citra/citra_emu/NativeLibrary.java index 37edbb1a9..b1b1e2903 100644 --- a/src/android/app/src/main/java/org/citra/citra_emu/NativeLibrary.java +++ b/src/android/app/src/main/java/org/citra/citra_emu/NativeLibrary.java @@ -6,6 +6,7 @@ package org.citra.citra_emu; +import android.content.pm.PackageManager; import android.content.res.Configuration; import android.text.Html; import android.text.method.LinkMovementMethod; @@ -20,9 +21,12 @@ import androidx.appcompat.app.AlertDialog; import org.citra.citra_emu.activities.EmulationActivity; import org.citra.citra_emu.utils.EmulationMenuSettings; import org.citra.citra_emu.utils.Log; +import org.citra.citra_emu.utils.PermissionsHandler; import java.lang.ref.WeakReference; +import static android.Manifest.permission.CAMERA; + /** * Class which contains methods that interact * with the native side of the Citra code. @@ -401,6 +405,38 @@ public final class NativeLibrary { sEmulationActivity.clear(); } + private static final Object cameraPermissionLock = new Object(); + private static boolean cameraPermissionGranted = false; + public static final int REQUEST_CODE_NATIVE_CAMERA = 800; + + public static boolean RequestCameraPermission() { + final EmulationActivity emulationActivity = sEmulationActivity.get(); + if (emulationActivity == null) { + Log.error("[NativeLibrary] EmulationActivity not present"); + return false; + } + if (ContextCompat.checkSelfPermission(emulationActivity, CAMERA) == PackageManager.PERMISSION_GRANTED) { + // Permission already granted + return true; + } + emulationActivity.requestPermissions(new String[]{CAMERA}, REQUEST_CODE_NATIVE_CAMERA); + + // Wait until result is returned + synchronized (cameraPermissionLock) { + try { + cameraPermissionLock.wait(); + } catch (InterruptedException ignored) {} + } + return cameraPermissionGranted; + } + + public static void CameraPermissionResult(boolean granted) { + cameraPermissionGranted = granted; + synchronized (cameraPermissionLock) { + cameraPermissionLock.notify(); + } + } + /** * Button type for use in onTouchEvent */ diff --git a/src/android/app/src/main/java/org/citra/citra_emu/activities/EmulationActivity.java b/src/android/app/src/main/java/org/citra/citra_emu/activities/EmulationActivity.java index 297c78560..ca0c0ac1b 100644 --- a/src/android/app/src/main/java/org/citra/citra_emu/activities/EmulationActivity.java +++ b/src/android/app/src/main/java/org/citra/citra_emu/activities/EmulationActivity.java @@ -4,6 +4,7 @@ import android.app.Activity; import android.app.PendingIntent; import android.content.Intent; import android.content.SharedPreferences; +import android.content.pm.PackageManager; import android.os.Bundle; import android.os.Handler; import android.preference.PreferenceManager; @@ -228,6 +229,18 @@ public final class EmulationActivity extends AppCompatActivity { } } + @Override + public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { + switch (requestCode) { + case NativeLibrary.REQUEST_CODE_NATIVE_CAMERA: + NativeLibrary.CameraPermissionResult(grantResults[0] == PackageManager.PERMISSION_GRANTED); + break; + default: + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + break; + } + } + private void enableFullscreenImmersive() { // It would be nice to use IMMERSIVE_STICKY, but that doesn't show the toolbar. mDecorView.setSystemUiVisibility( diff --git a/src/android/app/src/main/java/org/citra/citra_emu/utils/PermissionsHandler.java b/src/android/app/src/main/java/org/citra/citra_emu/utils/PermissionsHandler.java index a29e23e8d..b1cfc542d 100644 --- a/src/android/app/src/main/java/org/citra/citra_emu/utils/PermissionsHandler.java +++ b/src/android/app/src/main/java/org/citra/citra_emu/utils/PermissionsHandler.java @@ -8,10 +8,12 @@ import android.os.Build; import androidx.core.content.ContextCompat; import androidx.fragment.app.FragmentActivity; +import static android.Manifest.permission.CAMERA; import static android.Manifest.permission.WRITE_EXTERNAL_STORAGE; public class PermissionsHandler { public static final int REQUEST_CODE_WRITE_PERMISSION = 500; + public static final int REQUEST_CODE_CAMERA = 700; // We use permissions acceptance as an indicator if this is a first boot for the user. public static boolean isFirstBoot(final FragmentActivity activity) { diff --git a/src/android/app/src/main/jni/CMakeLists.txt b/src/android/app/src/main/jni/CMakeLists.txt index 3ce02730b..c86d1a686 100644 --- a/src/android/app/src/main/jni/CMakeLists.txt +++ b/src/android/app/src/main/jni/CMakeLists.txt @@ -3,6 +3,8 @@ add_library(main SHARED applets/swkbd.h button_manager.cpp button_manager.h + camera/ndk_camera.cpp + camera/ndk_camera.h camera/still_image_camera.cpp camera/still_image_camera.h config.cpp @@ -21,6 +23,6 @@ add_library(main SHARED ) target_link_libraries(main PRIVATE common core input_common network) -target_link_libraries(main PRIVATE android jnigraphics EGL glad inih log yuv) +target_link_libraries(main PRIVATE android camera2ndk mediandk jnigraphics EGL glad inih log yuv) set(CPACK_PACKAGE_EXECUTABLES ${CPACK_PACKAGE_EXECUTABLES} main) diff --git a/src/android/app/src/main/jni/camera/ndk_camera.cpp b/src/android/app/src/main/jni/camera/ndk_camera.cpp new file mode 100644 index 000000000..124a8ef3b --- /dev/null +++ b/src/android/app/src/main/jni/camera/ndk_camera.cpp @@ -0,0 +1,346 @@ +// Copyright 2020 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include +#include +#include +#include +#include +#include +#include +#include +#include "common/scope_exit.h" +#include "core/frontend/camera/blank_camera.h" +#include "jni/camera/ndk_camera.h" +#include "jni/id_cache.h" + +namespace Camera::NDK { + +/** + * Implementation detail of NDK camera interface, holding a ton of different structs. + * As long as the object lives, the camera is opened and capturing image. To turn off the camera, + * one needs to destruct the object. + * It captures at the maximum resolution supported by the device, capped at the 3DS camera's maximum + * resolution (640x480). The pixel format is I420. + */ +class CaptureSession final { +public: + explicit CaptureSession(ACameraManager* manager, const std::string& id); + +private: + std::pair selected_resolution{}; + + ACameraDevice_StateCallbacks device_callbacks{}; + AImageReader_ImageListener listener{}; + ACameraCaptureSession_stateCallbacks session_callbacks{}; + std::array requests{}; + +#define MEMBER(type, name, func) \ + struct type##Deleter { \ + void operator()(type* ptr) { \ + type##_##func(ptr); \ + } \ + }; \ + std::unique_ptr name + + MEMBER(ACameraDevice, device, close); + MEMBER(AImageReader, image_reader, delete); + + // This window doesn't need to be destructed as it is managed by AImageReader + ANativeWindow* native_window{}; + + MEMBER(ACaptureSessionOutputContainer, outputs, free); + MEMBER(ACaptureSessionOutput, output, free); + MEMBER(ACameraOutputTarget, target, free); + MEMBER(ACaptureRequest, request, free); + + // Put session last to close the session before we destruct everything else + MEMBER(ACameraCaptureSession, session, close); +#undef MEMBER + + bool ready = false; + + std::mutex data_mutex; + + // Clang does not yet have shared_ptr to arrays support. Managed data are actually arrays. + std::array, 3> data; // I420 format, planes are Y, U, V. + + friend class Interface; + friend void ImageCallback(void* context, AImageReader* reader); +}; + +static void OnDisconnected(void* context, ACameraDevice* device) { + LOG_ERROR(Service_CAM, "Camera device disconnected"); + // TODO: Do something here? + // CaptureSession* that = reinterpret_cast(context); + // that->CloseCamera(); +} + +static void OnError(void* context, ACameraDevice* device, int error) { + LOG_ERROR(Service_CAM, "Camera device error {}", error); + // TODO: Do something here? + // CaptureSession* that = reinterpret_cast(context); + // that->CloseCamera(); +} + +#define MEDIA_CALL(func) \ + { \ + auto ret = func; \ + if (ret != AMEDIA_OK) { \ + LOG_ERROR(Service_CAM, "Call " #func " returned error {}", ret); \ + return; \ + } \ + } + +#define CAMERA_CALL(func) \ + { \ + auto ret = func; \ + if (ret != ACAMERA_OK) { \ + LOG_ERROR(Service_CAM, "Call " #func " returned error {}", ret); \ + return; \ + } \ + } + +void ImageCallback(void* context, AImageReader* reader) { + AImage* image{}; + MEDIA_CALL(AImageReader_acquireLatestImage(reader, &image)) + SCOPE_EXIT({ AImage_delete(image); }); + + std::array, 3> data; + for (const int plane : {0, 1, 2}) { + u8* ptr; + int size; + MEDIA_CALL(AImage_getPlaneData(image, plane, &ptr, &size)); + data[plane].reset(new u8[size], std::default_delete()); + std::memcpy(data[plane].get(), ptr, static_cast(size)); + } + + { + CaptureSession* that = reinterpret_cast(context); + std::lock_guard lock{that->data_mutex}; + that->data = data; + } +} + +#define CREATE(type, name, statement) \ + { \ + type* raw; \ + statement; \ + name.reset(raw); \ + } + +CaptureSession::CaptureSession(ACameraManager* manager, const std::string& id) { + device_callbacks = { + /*context*/ nullptr, + /*onDisconnected*/ &OnDisconnected, + /*onError*/ &OnError, + }; + + CREATE(ACameraDevice, device, + CAMERA_CALL(ACameraManager_openCamera(manager, id.c_str(), &device_callbacks, &raw))); + + ACameraMetadata* metadata; + CAMERA_CALL(ACameraManager_getCameraCharacteristics(manager, id.c_str(), &metadata)); + + ACameraMetadata_const_entry entry; + CAMERA_CALL(ACameraMetadata_getConstEntry( + metadata, ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, &entry)); + selected_resolution = {}; + for (std::size_t i = 0; i < entry.count; i += 4) { + // (format, width, height, input?) + if (entry.data.i32[i + 3] & ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_INPUT) { + // This is an input stream + continue; + } + + int format = entry.data.i32[i + 0]; + if (format == AIMAGE_FORMAT_YUV_420_888) { + int width = entry.data.i32[i + 1]; + int height = entry.data.i32[i + 2]; + if (width <= 640 && height <= 480) { // Maximum size the 3DS supports + selected_resolution = std::max(selected_resolution, std::make_pair(width, height)); + } + } + } + ACameraMetadata_free(metadata); + + if (selected_resolution == std::pair{}) { + LOG_ERROR(Service_CAM, "Device does not support any YUV output format"); + return; + } + + CREATE(AImageReader, image_reader, + MEDIA_CALL(AImageReader_new(selected_resolution.first, selected_resolution.second, + AIMAGE_FORMAT_YUV_420_888, 4, &raw))); + + listener = { + /*context*/ this, + /*onImageAvailable*/ &ImageCallback, + }; + MEDIA_CALL(AImageReader_setImageListener(image_reader.get(), &listener)); + + MEDIA_CALL(AImageReader_getWindow(image_reader.get(), &native_window)); + CREATE(ACaptureSessionOutput, output, + CAMERA_CALL(ACaptureSessionOutput_create(native_window, &raw))); + + CREATE(ACaptureSessionOutputContainer, outputs, + CAMERA_CALL(ACaptureSessionOutputContainer_create(&raw))); + CAMERA_CALL(ACaptureSessionOutputContainer_add(outputs.get(), output.get())); + + CREATE(ACameraCaptureSession, session, + CAMERA_CALL(ACameraDevice_createCaptureSession(device.get(), outputs.get(), + &session_callbacks, &raw))); + CREATE(ACaptureRequest, request, + CAMERA_CALL(ACameraDevice_createCaptureRequest(device.get(), TEMPLATE_PREVIEW, &raw))); + + ANativeWindow_acquire(native_window); + CREATE(ACameraOutputTarget, target, + CAMERA_CALL(ACameraOutputTarget_create(native_window, &raw))); + CAMERA_CALL(ACaptureRequest_addTarget(request.get(), target.get())); + + requests = {request.get()}; + CAMERA_CALL(ACameraCaptureSession_setRepeatingRequest(session.get(), nullptr, 1, + requests.data(), nullptr)); + + ready = true; +} + +#undef MEDIA_CALL +#undef CAMERA_CALL +#undef CREATE + +Interface::Interface(Factory& factory_, const std::string& id_, const Service::CAM::Flip& flip_) + : factory(factory_), id(id_), flip(flip_) {} + +Interface::~Interface() = default; + +void Interface::StartCapture() { + session = factory.CreateCaptureSession(id); +} + +void Interface::StopCapture() { + session.reset(); +} + +void Interface::SetResolution(const Service::CAM::Resolution& resolution_) { + resolution = resolution_; +} + +void Interface::SetFlip(Service::CAM::Flip flip_) { + flip = flip_; +} + +void Interface::SetFormat(Service::CAM::OutputFormat format_) { + format = format_; +} + +std::vector Interface::ReceiveFrame() { + std::array, 3> data; + { + std::lock_guard lock{session->data_mutex}; + data = session->data; + } + + auto [width, height] = session->selected_resolution; + + int crop_width{}, crop_height{}; + if (resolution.width * height > resolution.height * width) { + crop_width = width; + crop_height = width * resolution.height / resolution.width; + } else { + crop_height = height; + crop_width = height * resolution.width / resolution.height; + } + + int crop_x = (width - crop_width) / 2; + int crop_y = (height - crop_height) / 2; + int offset = crop_y * width + crop_x; + std::vector scaled_y(resolution.width * resolution.height); + std::vector scaled_u(resolution.width * resolution.height / 4ul); + std::vector scaled_v(resolution.width * resolution.height / 4ul); + // Crop and scale + libyuv::I420Scale(data[0].get() + offset, width, data[1].get() + offset / 4, width / 4, + data[2].get() + offset / 4, width / 4, crop_width, crop_height, + scaled_y.data(), resolution.width, scaled_u.data(), resolution.width / 4, + scaled_v.data(), resolution.width / 4, resolution.width, resolution.height, + libyuv::kFilterBilinear); + // TODO: Record and apply flip + + std::vector output(resolution.width * resolution.height); + if (format == Service::CAM::OutputFormat::RGB565) { + libyuv::I420ToRGB565(scaled_y.data(), resolution.width, scaled_u.data(), + resolution.width / 4, scaled_v.data(), resolution.width / 4, + reinterpret_cast(output.data()), resolution.width * 2, + resolution.width, resolution.height); + } else { + libyuv::I420ToYUY2(scaled_y.data(), resolution.width, scaled_u.data(), resolution.width / 4, + scaled_v.data(), resolution.width / 4, + reinterpret_cast(output.data()), resolution.width * 2, + resolution.width, resolution.height); + } + return output; +} + +bool Interface::IsPreviewAvailable() { + return session && session->ready; +} + +Factory::Factory() = default; + +Factory::~Factory() = default; + +std::shared_ptr Factory::CreateCaptureSession(const std::string& id) { + if (opened_camera_map.count(id) && !opened_camera_map.at(id).expired()) { + return opened_camera_map.at(id).lock(); + } + const auto& session = std::make_shared(manager.get(), id); + opened_camera_map.insert_or_assign(id, session); + return session; +} + +std::unique_ptr Factory::Create(const std::string& config, + const Service::CAM::Flip& flip) { + + if (!manager) { + JNIEnv* env = IDCache::GetEnvForThread(); + jboolean result = env->CallStaticBooleanMethod(IDCache::GetNativeLibraryClass(), + IDCache::GetRequestCameraPermission()); + if (result != JNI_TRUE) { + LOG_ERROR(Service_CAM, "Camera permissions denied"); + return std::make_unique(); + } + + manager.reset(ACameraManager_create()); + } + ACameraIdList* id_list = nullptr; + + auto ret = ACameraManager_getCameraIdList(manager.get(), &id_list); + if (ret != ACAMERA_OK) { + LOG_ERROR(Service_CAM, "Failed to get camera ID list: ret {}", ret); + return std::make_unique(); + } + + SCOPE_EXIT({ ACameraManager_deleteCameraIdList(id_list); }); + + if (id_list->numCameras <= 0) { + LOG_ERROR(Service_CAM, "No camera devices found"); + return std::make_unique(); + } + if (config.empty()) { + LOG_WARNING(Service_CAM, "Camera ID not set, using default camera"); + return std::make_unique(*this, id_list->cameraIds[0], flip); + } + + for (int i = 0; i < id_list->numCameras; ++i) { + const char* id = id_list->cameraIds[i]; + if (config == id) { + return std::make_unique(*this, id, flip); + } + } + + LOG_ERROR(Service_CAM, "Camera ID {} not found", config); + return std::make_unique(); +} + +} // namespace Camera::NDK diff --git a/src/android/app/src/main/jni/camera/ndk_camera.h b/src/android/app/src/main/jni/camera/ndk_camera.h new file mode 100644 index 000000000..56156f209 --- /dev/null +++ b/src/android/app/src/main/jni/camera/ndk_camera.h @@ -0,0 +1,71 @@ +// Copyright 2020 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include +#include +#include "common/common_types.h" +#include "core/frontend/camera/factory.h" +#include "core/frontend/camera/interface.h" + +namespace Camera::NDK { + +class CaptureSession; +class Factory; + +class Interface : public CameraInterface { +public: + Interface(Factory& factory, const std::string& id, const Service::CAM::Flip& flip); + ~Interface() override; + void StartCapture() override; + void StopCapture() override; + void SetResolution(const Service::CAM::Resolution& resolution) override; + void SetFlip(Service::CAM::Flip flip) override; + void SetEffect(Service::CAM::Effect effect) override{}; + void SetFormat(Service::CAM::OutputFormat format) override; + void SetFrameRate(Service::CAM::FrameRate frame_rate) override{}; + std::vector ReceiveFrame() override; + bool IsPreviewAvailable() override; + +private: + // jstring path; + Factory& factory; + std::shared_ptr session; + std::string id; + + Service::CAM::Resolution resolution; + Service::CAM::Flip flip; + Service::CAM::OutputFormat format; + // std::vector image; // Data fetched from the frontend + // bool opened{}; // Whether the camera was successfully opened +}; + +class Factory final : public CameraFactory { +public: + explicit Factory(); + ~Factory() override; + + std::unique_ptr Create(const std::string& config, + const Service::CAM::Flip& flip) override; + +private: + std::shared_ptr CreateCaptureSession(const std::string& id); + + // The session is cached, to avoid opening the same camera twice. + // This is weak_ptr so that the session is destructed when all cameras are closed + std::unordered_map> opened_camera_map; + + struct ACameraManagerDeleter { + void operator()(ACameraManager* manager) { + ACameraManager_delete(manager); + } + }; + std::unique_ptr manager; + + friend class Interface; +}; + +} // namespace Camera::NDK diff --git a/src/android/app/src/main/jni/id_cache.cpp b/src/android/app/src/main/jni/id_cache.cpp index e87656a8c..9f0130fb2 100644 --- a/src/android/app/src/main/jni/id_cache.cpp +++ b/src/android/app/src/main/jni/id_cache.cpp @@ -24,6 +24,7 @@ static jmethodID s_alert_prompt_button; static jmethodID s_is_portrait_mode; static jmethodID s_landscape_screen_layout; static jmethodID s_exit_emulation_activity; +static jmethodID s_request_camera_permission; namespace IDCache { @@ -74,6 +75,10 @@ jmethodID GetExitEmulationActivity() { return s_exit_emulation_activity; } +jmethodID GetRequestCameraPermission() { + return s_request_camera_permission; +} + } // namespace IDCache #ifdef __cplusplus @@ -112,6 +117,8 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) { env->GetStaticMethodID(s_native_library_class, "landscapeScreenLayout", "()I"); s_exit_emulation_activity = env->GetStaticMethodID(s_native_library_class, "exitEmulationActivity", "(I)V"); + s_request_camera_permission = + env->GetStaticMethodID(s_native_library_class, "RequestCameraPermission", "()Z"); SoftwareKeyboard::InitJNI(env); Camera::StillImage::InitJNI(env); diff --git a/src/android/app/src/main/jni/id_cache.h b/src/android/app/src/main/jni/id_cache.h index ac9f24d97..86a9a5703 100644 --- a/src/android/app/src/main/jni/id_cache.h +++ b/src/android/app/src/main/jni/id_cache.h @@ -16,5 +16,6 @@ jmethodID GetAlertPromptButton(); jmethodID GetIsPortraitMode(); jmethodID GetLandscapeScreenLayout(); jmethodID GetExitEmulationActivity(); +jmethodID GetRequestCameraPermission(); } // namespace IDCache diff --git a/src/android/app/src/main/jni/native.cpp b/src/android/app/src/main/jni/native.cpp index a80d9d0ec..522b300f2 100644 --- a/src/android/app/src/main/jni/native.cpp +++ b/src/android/app/src/main/jni/native.cpp @@ -22,6 +22,7 @@ #include "core/settings.h" #include "jni/applets/swkbd.h" #include "jni/button_manager.h" +#include "jni/camera/ndk_camera.h" #include "jni/camera/still_image_camera.h" #include "jni/config.h" #include "jni/emu_window/emu_window.h" @@ -109,6 +110,7 @@ static Core::System::ResultStatus RunCitra(const std::string& filepath) { Settings::Apply(); Camera::RegisterFactory("image", std::make_unique()); + Camera::RegisterFactory("ndk", std::make_unique()); // Register frontend applets Frontend::RegisterDefaultApplets();