Update GDBStub.cpp

Source/3rdParty/CMakeLists.txt

@@ -41,10 +41,10 @@ ExternalProject_Add(mupen64plus-core
     CONFIGURE_COMMAND ""
     INSTALL_COMMAND ""
 
-    BUILD_COMMAND make all -f ${M64P_CORE_DIR}/projects/unix/Makefile 
-        SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/mupen64plus-core/src 
-        SUBDIR=${CMAKE_CURRENT_SOURCE_DIR}/mupen64plus-core/subprojects 
-        OSD=0 NEW_DYNAREC=1 KEYBINDINGS=0 ACCURATE_FPU=1 
+    BUILD_COMMAND make all -f ${M64P_CORE_DIR}/projects/unix/Makefile
+        SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/mupen64plus-core/src
+        SUBDIR=${CMAKE_CURRENT_SOURCE_DIR}/mupen64plus-core/subprojects
+        OSD=0 NEW_DYNAREC=1 KEYBINDINGS=0 ACCURATE_FPU=1 DEBUGGER=1
         TARGET=${CORE_FILE} DEBUG=$<CONFIG:Debug> CC=${MAKE_CC_COMPILER} CXX=${MAKE_CXX_COMPILER}
     BUILD_IN_SOURCE False
     BUILD_ALWAYS True

Source/RMG-Core/CMakeLists.txt

@@ -24,6 +24,7 @@ set(RMG_CORE_SOURCES
     m64p/Api.cpp
     m64p/CoreApi.cpp
     m64p/ConfigApi.cpp
+    m64p/DebugApi.cpp
     m64p/PluginApi.cpp
     CachedRomHeaderAndSettings.cpp
     ConvertStringEncoding.cpp
@@ -38,6 +39,8 @@ set(RMG_CORE_SOURCES
     Emulation.cpp
     SaveState.cpp
     Callback.cpp
+    # TODO: make optional
+    GDBStub.cpp
     Plugins.cpp
     Version.cpp
     Cheats.cpp

Source/RMG-Core/Core.cpp

@@ -120,6 +120,14 @@ bool CoreInit(void)
         return false;
     }
 
+    ret = m64p::Debug.Hook(l_CoreLibHandle);
+    if (!ret)
+    {
+        error = m64p::Debug.GetLastError();
+        CoreSetError(error);
+        return false;
+    }
+
     m64p_ret = m64p::Core.Startup(FRONTEND_API_VERSION, CoreGetUserConfigDirectory().string().c_str(), CoreGetSharedDataDirectory().string().c_str(), (void*)l_CoreContextString, CoreDebugCallback, nullptr, CoreStateCallback);
     if (m64p_ret != M64ERR_SUCCESS)
     {

Source/RMG-Core/Emulation.cpp

@@ -12,6 +12,7 @@
 #include "RomSettings.hpp"
 #include "Emulation.hpp"
 #include "m64p/Api.hpp"
+#include "GDBStub.hpp"
 #include "Plugins.hpp"
 #include "Cheats.hpp"
 #include "Error.hpp"
@@ -88,12 +89,22 @@ static void apply_game_coresettings_overlay(void)
 // Exported Functions
 //
 
+#include <iostream>
 bool CoreStartEmulation(std::filesystem::path n64rom, std::filesystem::path n64ddrom)
 {
     std::string error;
     m64p_error  ret;
     CoreRomType type;
 
+    // TODO: don't hardcode it here
+    // the overlay will be copied later here
+    CoreSettingsSetValue(SettingsID::CoreOverlay_EnableDebugger, true);
+
+    if (!CoreGDBStubInit(5001))
+    {
+        return false;
+    }
+
     if (!CoreOpenRom(n64rom))
     {
         return false;
@@ -160,6 +171,8 @@ bool CoreStartEmulation(std::filesystem::path n64rom, std::filesystem::path n64d
         error += m64p::Core.ErrorMessage(ret);
     }
 
+    CoreGDBStubShutdown();
+
     CoreClearCheats();
     CoreDetachPlugins();
     CoreCloseRom();

Source/RMG-Core/GDBStub.hpp

@@ -0,0 +1,19 @@
+// Taken from https://github.com/Gillou68310/mupen64plus-ui-console/blob/gdbstub/src/gdbstub.h
+// And modified for RMG-Core
+
+// Copyright 2013 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+// Originally written by Sven Peter <sven@fail0verflow.com> for anergistic.
+
+#ifndef CORE_GDBSTUB_H
+#define CORE_GDBSTUB_H
+
+// attempts to initialize GDBStub
+bool CoreGDBStubInit(int port);
+
+// shuts down GDB stub
+void CoreGDBStubShutdown(void);
+
+#endif // CORE_GDBSTUB_H

Source/RMG-Core/m64p/Api.cpp

@@ -13,4 +13,5 @@ namespace m64p
 {
 m64p::CoreApi Core;
 m64p::ConfigApi Config;
+m64p::DebugApi Debug;
 } // namespace m64p

Source/RMG-Core/m64p/Api.hpp

@@ -12,11 +12,13 @@
 
 #include "ConfigApi.hpp"
 #include "CoreApi.hpp"
+#include "DebugApi.hpp"
 
 namespace m64p
 {
 extern m64p::CoreApi Core;
 extern m64p::ConfigApi Config;
+extern m64p::DebugApi Debug;
 } // namespace m64p
 
 #endif // M64P_API_HPP

Source/RMG-Core/m64p/DebugApi.cpp

@@ -0,0 +1,93 @@
+/*
+ * Rosalie's Mupen GUI - https://github.com/Rosalie241/RMG
+ *  Copyright (C) 2020 Rosalie Wanders <rosalie@mailbox.org>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License version 3.
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+#include "DebugApi.hpp"
+#include "Macros.hpp"
+
+using namespace m64p;
+
+DebugApi::DebugApi(void)
+{
+    this->Unhook();
+}
+
+DebugApi::~DebugApi(void)
+{
+}
+
+bool DebugApi::Hook(m64p_dynlib_handle handle)
+{
+    this->errorMessage = "DebugApi::Hook Failed: ";
+
+    HOOK_FUNC(handle, Debug, SetCallbacks);
+    HOOK_FUNC(handle, Debug, SetRunState);
+    HOOK_FUNC(handle, Debug, GetState);
+    HOOK_FUNC(handle, Debug, Step);
+    HOOK_FUNC(handle, Debug, DecodeOp);
+    HOOK_FUNC(handle, Debug, MemGetRecompInfo);
+    HOOK_FUNC(handle, Debug, MemGetMemInfo);
+    HOOK_FUNC(handle, Debug, MemGetPointer);
+    HOOK_FUNC(handle, Debug, MemRead64);
+    HOOK_FUNC(handle, Debug, MemRead32);
+    HOOK_FUNC(handle, Debug, MemRead16);
+    HOOK_FUNC(handle, Debug, MemRead8);
+    HOOK_FUNC(handle, Debug, MemWrite64);
+    HOOK_FUNC(handle, Debug, MemWrite32);
+    HOOK_FUNC(handle, Debug, MemWrite16);
+    HOOK_FUNC(handle, Debug, MemWrite8);
+    HOOK_FUNC(handle, Debug, GetCPUDataPtr);
+    HOOK_FUNC(handle, Debug, BreakpointLookup);
+    HOOK_FUNC(handle, Debug, BreakpointCommand);
+
+    this->handle = handle;
+    this->hooked = true;
+    return true;
+}
+
+bool DebugApi::Unhook(void)
+{
+    UNHOOK_FUNC(Debug, SetCallbacks);
+    UNHOOK_FUNC(Debug, SetRunState);
+    UNHOOK_FUNC(Debug, GetState);
+    UNHOOK_FUNC(Debug, Step);
+    UNHOOK_FUNC(Debug, DecodeOp);
+    UNHOOK_FUNC(Debug, MemGetRecompInfo);
+    UNHOOK_FUNC(Debug, MemGetMemInfo);
+    UNHOOK_FUNC(Debug, MemGetPointer);
+    UNHOOK_FUNC(Debug, MemRead64);
+    UNHOOK_FUNC(Debug, MemRead32);
+    UNHOOK_FUNC(Debug, MemRead16);
+    UNHOOK_FUNC(Debug, MemRead8);
+    UNHOOK_FUNC(Debug, MemWrite64);
+    UNHOOK_FUNC(Debug, MemWrite32);
+    UNHOOK_FUNC(Debug, MemWrite16);
+    UNHOOK_FUNC(Debug, MemWrite8);
+    UNHOOK_FUNC(Debug, GetCPUDataPtr);
+    UNHOOK_FUNC(Debug, BreakpointLookup);
+    UNHOOK_FUNC(Debug, BreakpointCommand);
+
+    this->handle = nullptr;
+    this->hooked = false;
+    return true;
+}
+
+bool DebugApi::IsHooked(void)
+{
+    return this->hooked;
+}
+
+m64p_dynlib_handle DebugApi::GetHandle(void)
+{
+    return this->handle;
+}
+
+std::string DebugApi::GetLastError(void)
+{
+    return this->errorMessage;
+}

Source/RMG-Core/m64p/DebugApi.hpp

@@ -0,0 +1,61 @@
+/*
+ * Rosalie's Mupen GUI - https://github.com/Rosalie241/RMG
+ *  Copyright (C) 2020 Rosalie Wanders <rosalie@mailbox.org>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License version 3.
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+#ifndef M64P_DEBUGAPI_HPP
+#define M64P_DEBUGAPI_HPP
+
+#include "api/m64p_common.h"
+#include "api/m64p_debugger.h"
+
+#include <string>
+
+namespace m64p
+{
+class DebugApi
+{
+  public:
+    DebugApi();
+    ~DebugApi();
+
+    bool Hook(m64p_dynlib_handle handle);
+    bool Unhook(void);
+    bool IsHooked(void);
+
+    m64p_dynlib_handle GetHandle(void);
+    std::string GetLastError(void);
+
+    ptr_DebugSetCallbacks SetCallbacks;
+    ptr_DebugSetRunState SetRunState;
+    ptr_DebugGetState GetState;
+    ptr_DebugStep Step;
+    ptr_DebugDecodeOp DecodeOp;
+    ptr_DebugMemGetRecompInfo MemGetRecompInfo;
+    ptr_DebugMemGetMemInfo MemGetMemInfo;
+    ptr_DebugMemGetPointer MemGetPointer;
+    ptr_DebugMemRead64 MemRead64;
+    ptr_DebugMemRead32 MemRead32;
+    ptr_DebugMemRead16 MemRead16;
+    ptr_DebugMemRead8 MemRead8;
+    ptr_DebugMemWrite64 MemWrite64;
+    ptr_DebugMemWrite32 MemWrite32;
+    ptr_DebugMemWrite16 MemWrite16;
+    ptr_DebugMemWrite8 MemWrite8;
+    ptr_DebugGetCPUDataPtr GetCPUDataPtr;
+    ptr_DebugBreakpointLookup BreakpointLookup;
+    ptr_DebugBreakpointCommand BreakpointCommand;
+
+  private:
+    bool hooked = false;
+
+    std::string errorMessage;
+    m64p_dynlib_handle handle;
+};
+} // namespace m64p
+
+#endif // M64P_DEBUGAPI_HPP