diff --git a/src/core/hid/emulated_controller.cpp b/src/core/hid/emulated_controller.cpp
index 0a7777732..f9cfa7b88 100644
--- a/src/core/hid/emulated_controller.cpp
+++ b/src/core/hid/emulated_controller.cpp
@@ -111,16 +111,25 @@ void EmulatedController::ReloadFromSettings() {
 
     ring_params[0] = Common::ParamPackage(Settings::values.ringcon_analogs);
 
-    // Other or debug controller should always be a pro controller
-    if (npad_id_type != NpadIdType::Other) {
-        SetNpadStyleIndex(MapSettingsTypeToNPad(player.controller_type));
-        original_npad_type = npad_type;
-    } else {
-        SetNpadStyleIndex(NpadStyleIndex::ProController);
-        original_npad_type = npad_type;
+    // Player 1 shares config with handheld. Disable controller when handheld is selected
+    // if (npad_id_type == NpadIdType::Player1 && npad_type == NpadStyleIndex::Handheld) {
+    //    Disconnect();
+    //    ReloadInput();
+    //    return;
+    //}
+
+    // Handheld shares config with player 1. Disable controller when handheld isn't selected
+    if (npad_id_type == NpadIdType::Handheld && npad_type != NpadStyleIndex::Handheld) {
+        Disconnect();
+        ReloadInput();
+        return;
     }
 
     Disconnect();
+
+    SetNpadStyleIndex(MapSettingsTypeToNPad(player.controller_type));
+    original_npad_type = npad_type;
+
     if (player.connected) {
         Connect();
     }
@@ -593,8 +602,14 @@ bool EmulatedController::IsConfiguring() const {
 }
 
 void EmulatedController::SaveCurrentConfig() {
-    const auto player_index = NpadIdTypeToIndex(npad_id_type);
+    // Other and Handheld can't alter the config from here
+    if (npad_id_type == NpadIdType::Other || npad_id_type == NpadIdType::Handheld) {
+        return;
+    }
+
+    const auto player_index = NpadIdTypeToConfigIndex(npad_id_type);
     auto& player = Settings::values.players.GetValue()[player_index];
+
     player.connected = is_connected;
     player.controller_type = MapNPadToSettingsType(npad_type);
     for (std::size_t index = 0; index < player.buttons.size(); ++index) {
diff --git a/src/yuzu/configuration/configure_input.cpp b/src/yuzu/configuration/configure_input.cpp
index 7fce85bca..2c4a68ecb 100644
--- a/src/yuzu/configuration/configure_input.cpp
+++ b/src/yuzu/configuration/configure_input.cpp
@@ -196,6 +196,7 @@ void ConfigureInput::ApplyConfiguration() {
     }
 
     advanced->ApplyConfiguration();
+    system.HIDCore().ReloadInputDevices();
 
     const bool pre_docked_mode = Settings::values.use_docked_mode.GetValue();
     Settings::values.use_docked_mode.SetValue(ui->radioDocked->isChecked());
diff --git a/src/yuzu/configuration/configure_input_per_game.cpp b/src/yuzu/configuration/configure_input_per_game.cpp
index 78e65d468..b417765db 100644
--- a/src/yuzu/configuration/configure_input_per_game.cpp
+++ b/src/yuzu/configuration/configure_input_per_game.cpp
@@ -89,17 +89,12 @@ void ConfigureInputPerGame::LoadConfiguration() {
 
         emulated_controller->ReloadFromSettings();
 
-        if (player_index > 0) {
+        if (player_index != HANDHELD_INDEX) {
             continue;
         }
+
         // Handle Handheld cases
-        auto& handheld_player = Settings::values.players.GetValue()[HANDHELD_INDEX];
-        auto* handheld_controller = hid_core.GetEmulatedController(Core::HID::NpadIdType::Handheld);
-        if (player.controller_type == Settings::ControllerType::Handheld) {
-            handheld_player = player;
-        } else {
-            handheld_player = {};
-        }
+        auto handheld_controller = hid_core.GetEmulatedController(Core::HID::NpadIdType::Handheld);
         handheld_controller->ReloadFromSettings();
     }
 }
diff --git a/src/yuzu/configuration/configure_input_player.cpp b/src/yuzu/configuration/configure_input_player.cpp
index 2c2e7e47b..c632f51b6 100644
--- a/src/yuzu/configuration/configure_input_player.cpp
+++ b/src/yuzu/configuration/configure_input_player.cpp
@@ -296,26 +296,11 @@ ConfigureInputPlayer::ConfigureInputPlayer(QWidget* parent, std::size_t player_i
       is_powered_on{is_powered_on_}, input_subsystem{input_subsystem_}, profiles(profiles_),
       timeout_timer(std::make_unique<QTimer>()),
       poll_timer(std::make_unique<QTimer>()), bottom_row{bottom_row_}, hid_core{hid_core_} {
-    if (player_index == 0) {
-        auto* emulated_controller_p1 =
-            hid_core.GetEmulatedController(Core::HID::NpadIdType::Player1);
-        auto* emulated_controller_handheld =
-            hid_core.GetEmulatedController(Core::HID::NpadIdType::Handheld);
-        emulated_controller_p1->SaveCurrentConfig();
-        emulated_controller_p1->EnableConfiguration();
-        emulated_controller_handheld->SaveCurrentConfig();
-        emulated_controller_handheld->EnableConfiguration();
-        if (emulated_controller_handheld->IsConnected(true)) {
-            emulated_controller_p1->Disconnect();
-            emulated_controller = emulated_controller_handheld;
-        } else {
-            emulated_controller = emulated_controller_p1;
-        }
-    } else {
-        emulated_controller = hid_core.GetEmulatedControllerByIndex(player_index);
-        emulated_controller->SaveCurrentConfig();
-        emulated_controller->EnableConfiguration();
-    }
+
+    emulated_controller = hid_core.GetEmulatedControllerByIndex(player_index);
+    emulated_controller->SaveCurrentConfig();
+    emulated_controller->EnableConfiguration();
+
     ui->setupUi(this);
 
     setFocusPolicy(Qt::ClickFocus);
@@ -747,29 +732,6 @@ ConfigureInputPlayer::ConfigureInputPlayer(QWidget* parent, std::size_t player_i
         UpdateMotionButtons();
         const Core::HID::NpadStyleIndex type =
             GetControllerTypeFromIndex(ui->comboControllerType->currentIndex());
-
-        if (player_index == 0) {
-            auto* emulated_controller_p1 =
-                hid_core.GetEmulatedController(Core::HID::NpadIdType::Player1);
-            auto* emulated_controller_handheld =
-                hid_core.GetEmulatedController(Core::HID::NpadIdType::Handheld);
-            bool is_connected = emulated_controller->IsConnected(true);
-
-            emulated_controller_p1->SetNpadStyleIndex(type);
-            emulated_controller_handheld->SetNpadStyleIndex(type);
-            if (is_connected) {
-                if (type == Core::HID::NpadStyleIndex::Handheld) {
-                    emulated_controller_p1->Disconnect();
-                    emulated_controller_handheld->Connect(true);
-                    emulated_controller = emulated_controller_handheld;
-                } else {
-                    emulated_controller_handheld->Disconnect();
-                    emulated_controller_p1->Connect(true);
-                    emulated_controller = emulated_controller_p1;
-                }
-            }
-            ui->controllerFrame->SetController(emulated_controller);
-        }
         emulated_controller->SetNpadStyleIndex(type);
     });
 
@@ -805,32 +767,10 @@ ConfigureInputPlayer::ConfigureInputPlayer(QWidget* parent, std::size_t player_i
 }
 
 ConfigureInputPlayer::~ConfigureInputPlayer() {
-    if (player_index == 0) {
-        auto* emulated_controller_p1 =
-            hid_core.GetEmulatedController(Core::HID::NpadIdType::Player1);
-        auto* emulated_controller_handheld =
-            hid_core.GetEmulatedController(Core::HID::NpadIdType::Handheld);
-        emulated_controller_p1->DisableConfiguration();
-        emulated_controller_handheld->DisableConfiguration();
-    } else {
-        emulated_controller->DisableConfiguration();
-    }
+    emulated_controller->DisableConfiguration();
 }
 
 void ConfigureInputPlayer::ApplyConfiguration() {
-    if (player_index == 0) {
-        auto* emulated_controller_p1 =
-            hid_core.GetEmulatedController(Core::HID::NpadIdType::Player1);
-        auto* emulated_controller_handheld =
-            hid_core.GetEmulatedController(Core::HID::NpadIdType::Handheld);
-        emulated_controller_p1->DisableConfiguration();
-        emulated_controller_p1->SaveCurrentConfig();
-        emulated_controller_p1->EnableConfiguration();
-        emulated_controller_handheld->DisableConfiguration();
-        emulated_controller_handheld->SaveCurrentConfig();
-        emulated_controller_handheld->EnableConfiguration();
-        return;
-    }
     emulated_controller->DisableConfiguration();
     emulated_controller->SaveCurrentConfig();
     emulated_controller->EnableConfiguration();