diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 3e327e48..73574905 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -263,6 +263,8 @@ if (ARCHITECTURE_x86_64)
     else()
         target_sources(dynarmic PRIVATE backend/x64/exception_handler_generic.cpp)
     endif()
+
+    target_link_libraries(dynarmic PRIVATE xbyak)
 elseif(ARCHITECTURE_Aarch64)
     target_sources(dynarmic PRIVATE
          backend/A64/emitter/a64_emitter.cpp
@@ -287,9 +289,11 @@ target_link_libraries(dynarmic
         boost
         fmt::fmt
         mp
-        xbyak
         $<$<BOOL:DYNARMIC_USE_LLVM>:${llvm_libs}>
 )
+if (ARCHITECTURE_x86_64)
+    target_link_libraries(dynarmic PRIVATE xbyak)
+endif()
 if (DYNARMIC_ENABLE_CPU_FEATURE_DETECTION)
     target_compile_definitions(dynarmic PRIVATE DYNARMIC_ENABLE_CPU_FEATURE_DETECTION=1)
 endif()
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 29026bee..9a460375 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -39,7 +39,12 @@ include(CreateDirectoryGroups)
 create_target_directory_groups(dynarmic_tests)
 create_target_directory_groups(dynarmic_print_info)
 
-target_link_libraries(dynarmic_tests PRIVATE dynarmic boost catch fmt mp xbyak)
+target_link_libraries(dynarmic_tests PRIVATE dynarmic boost catch fmt mp)
+
+if (ARCHITECTURE_x86_64)
+    target_link_libraries(dynarmic_tests PRIVATE xbyak)
+endif()
+
 target_include_directories(dynarmic_tests PRIVATE . ../src)
 target_compile_options(dynarmic_tests PRIVATE ${DYNARMIC_CXX_FLAGS})
 target_compile_definitions(dynarmic_tests PRIVATE FMT_USE_USER_DEFINED_LITERALS=0)