Core: Run _malloc_init to fix LLE internal mallocs (#4622)

* Initialize heap API and run malloc_init

Purely for testing

* Update linker.cpp

* Fix heap function definitions

* Run malloc_init before initializing libs

Allows LLE libSceMoveTracker.

* Fix for rebase

* Some other fixups

* Some other rebase/bug fixes

* Claaaaaang

* Add a memory type for system memory

Don't want these system areas merging with things they shouldn't.

* mmap with MAP_SYSTEM flag

* Add FW 1.70 check to sceKernelMapNamedSystemFlexibleMemory

* regmgr call stub

Best we at least have some clue what this thing is doing

* Remove forced LibcInternal HLEs

These are no longer needed since running malloc_init LLE allows internal mallocs to succeed.

* Remove heap api pre-init

This is a hack.

src/core/libraries/kernel/kernel.cpp

@@ -326,6 +326,11 @@ s32 PS4_SYSV_ABI sceKernelGetProcessType(s32 pid) {
     return 0;
 }
 
+s32 PS4_SYSV_ABI __sys_regmgr_call(u32 op, u32 key, void* result, void* value, u64 len) {
+    LOG_ERROR(Lib_Kernel, "(STUBBED) called, op: {:#x}, key: {}, len: {}", op, key, len);
+    return ORBIS_OK;
+}
+
 // Nominally: long sysconf(int name);
 u64 PS4_SYSV_ABI posix_sysconf(s32 name) {
     switch (name) {
@@ -481,6 +486,7 @@ void RegisterLib(Core::Loader::SymbolsResolver* sym) {
     LIB_FUNCTION("9BcDykPmo1I", "libkernel", 1, "libkernel", __Error);
     LIB_FUNCTION("k+AXqu2-eBc", "libkernel", 1, "libkernel", posix_getpagesize);
     LIB_FUNCTION("k+AXqu2-eBc", "libScePosix", 1, "libkernel", posix_getpagesize);
+    LIB_FUNCTION("7NwggrWJ5cA", "libkernel", 1, "libkernel", __sys_regmgr_call);
 
     LIB_FUNCTION("mkawd0NA9ts", "libkernel", 1, "libkernel", posix_sysconf);
     LIB_FUNCTION("mkawd0NA9ts", "libScePosix", 1, "libkernel", posix_sysconf);

src/core/libraries/kernel/memory.cpp

@@ -342,6 +342,45 @@ s32 PS4_SYSV_ABI sceKernelMapFlexibleMemory(void** addr_in_out, u64 len, s32 pro
     return sceKernelMapNamedFlexibleMemory(addr_in_out, len, prot, flags, "anon");
 }
 
+s32 PS4_SYSV_ABI sceKernelMapNamedSystemFlexibleMemory(void** addr_in_out, u64 len, s32 prot,
+                                                       s32 flags, const char* name) {
+    LOG_INFO(Kernel_Vmm, "in_addr = {}, len = {:#x}, prot = {:#x}, flags = {:#x}, name = '{}'",
+             fmt::ptr(*addr_in_out), len, prot, flags, name);
+    if (len == 0 || !Common::Is16KBAligned(len)) {
+        LOG_ERROR(Kernel_Vmm, "len is 0 or not 16kb multiple");
+        return ORBIS_KERNEL_ERROR_EINVAL;
+    }
+
+    if (name == nullptr) {
+        LOG_ERROR(Kernel_Vmm, "name is invalid!");
+        return ORBIS_KERNEL_ERROR_EFAULT;
+    }
+
+    auto map_flags = static_cast<Core::MemoryMapFlags>(flags);
+    VAddr in_addr = reinterpret_cast<VAddr>(*addr_in_out);
+    if (True(map_flags & Core::MemoryMapFlags::Fixed) && in_addr == 0) {
+        if (Common::ElfInfo::FW_170 <= g_sdk_version) {
+            return ORBIS_KERNEL_ERROR_EINVAL;
+        }
+        map_flags &= ~Core::MemoryMapFlags::Fixed;
+    }
+
+    if (std::strlen(name) >= ORBIS_KERNEL_MAXIMUM_NAME_LENGTH) {
+        LOG_ERROR(Kernel_Vmm, "name exceeds 32 bytes!");
+        return ORBIS_KERNEL_ERROR_ENAMETOOLONG;
+    }
+
+    if (in_addr == 0) {
+        in_addr = 0x880000000;
+    }
+    const auto mem_prot = static_cast<Core::MemoryProt>(prot);
+    auto* memory = Core::Memory::Instance();
+    const auto ret = memory->MapMemory(addr_in_out, in_addr, len, mem_prot, map_flags,
+                                       Core::VMAType::System, name);
+    LOG_INFO(Kernel_Vmm, "out_addr = {}", fmt::ptr(*addr_in_out));
+    return ret;
+}
+
 s32 PS4_SYSV_ABI sceKernelQueryMemoryProtection(void* addr, void** start, void** end, u32* prot) {
     auto* memory = Core::Memory::Instance();
     return memory->QueryProtection(std::bit_cast<VAddr>(addr), start, end, prot);
@@ -728,9 +767,12 @@ void* PS4_SYSV_ABI posix_mmap(void* addr, u64 len, s32 prot, s32 flags, s32 fd,
 
     s32 result = ORBIS_OK;
     if (True(mem_flags & Core::MemoryMapFlags::Anon)) {
-        // Maps flexible memory
+        // Maps flexible or system memory
+        const Core::VMAType mapping_type = True(mem_flags & Core::MemoryMapFlags::System)
+                                               ? Core::VMAType::System
+                                               : Core::VMAType::Flexible;
         result = memory->MapMemory(&addr_out, aligned_addr, aligned_size, mem_prot, mem_flags,
-                                   Core::VMAType::Flexible, "anon", false);
+                                   mapping_type, "anon", false);
     } else if (True(mem_flags & Core::MemoryMapFlags::Stack)) {
         // Maps stack memory
         result = memory->MapMemory(&addr_out, aligned_addr, aligned_size, mem_prot, mem_flags,
@@ -876,6 +918,7 @@ void RegisterMemory(Core::Loader::SymbolsResolver* sym) {
     LIB_FUNCTION("PGhQHd-dzv8", "libkernel", 1, "libkernel", sceKernelMmap);
     LIB_FUNCTION("cQke9UuBQOk", "libkernel", 1, "libkernel", sceKernelMunmap);
     LIB_FUNCTION("mL8NDH86iQI", "libkernel", 1, "libkernel", sceKernelMapNamedFlexibleMemory);
+    LIB_FUNCTION("kc+LEEIYakc", "libkernel", 1, "libkernel", sceKernelMapNamedSystemFlexibleMemory);
     LIB_FUNCTION("aNz11fnnzi4", "libkernel", 1, "libkernel", sceKernelAvailableFlexibleMemorySize);
     LIB_FUNCTION("aNz11fnnzi4", "libkernel_avlfmem", 1, "libkernel",
                  sceKernelAvailableFlexibleMemorySize);

src/core/libraries/libc_internal/libc_internal.cpp

@@ -24,8 +24,4 @@ void RegisterLib(Core::Loader::SymbolsResolver* sym) {
     RegisterlibSceLibcInternalThreads(sym);
 }
 
-void ForceRegisterLib(Core::Loader::SymbolsResolver* sym) {
-    // Used to forcibly enable HLEs for broken LLE functions.
-    ForceRegisterlibSceLibcInternalIo(sym);
-}
 } // namespace Libraries::LibcInternal

src/core/libraries/libc_internal/libc_internal.h

@@ -15,5 +15,4 @@ namespace Libraries::LibcInternal {
 // so everything is just in the .cpp file
 
 void RegisterLib(Core::Loader::SymbolsResolver* sym);
-void ForceRegisterLib(Core::Loader::SymbolsResolver* sym);
 } // namespace Libraries::LibcInternal

src/core/libraries/libc_internal/libc_internal_io.cpp

@@ -465,7 +465,6 @@ s32 PS4_SYSV_ABI internal_fclose(OrbisFILE* file) {
 
 void RegisterlibSceLibcInternalIo(Core::Loader::SymbolsResolver* sym) {
     LIB_FUNCTION("eLdDw6l0-bU", "libSceLibcInternal", 1, "libSceLibcInternal", internal_snprintf);
-    LIB_FUNCTION("MUjC4lbHrK4", "libSceLibcInternal", 1, "libSceLibcInternal", internal_fflush);
     LIB_FUNCTION("xGT4Mc55ViQ", "libSceLibcInternal", 1, "libSceLibcInternal", internal__Fofind);
     LIB_FUNCTION("dREVnZkAKRE", "libSceLibcInternal", 1, "libSceLibcInternal", internal__Foprep);
     LIB_FUNCTION("sQL8D-jio7U", "libSceLibcInternal", 1, "libSceLibcInternal", internal__Fopen);
@@ -477,13 +476,11 @@ void RegisterlibSceLibcInternalIo(Core::Loader::SymbolsResolver* sym) {
                  internal__Lockfilelock);
     LIB_FUNCTION("0x7rx8TKy2Y", "libSceLibcInternal", 1, "libSceLibcInternal",
                  internal__Unlockfilelock);
-}
 
-void ForceRegisterlibSceLibcInternalIo(Core::Loader::SymbolsResolver* sym) {
-    // Goal is to be minimally intrusive here to allow LLE for printf/stdout writes.
     LIB_FUNCTION("xeYO4u7uyJ0", "libSceLibcInternal", 1, "libSceLibcInternal", internal_fopen);
     LIB_FUNCTION("rQFVBXp-Cxg", "libSceLibcInternal", 1, "libSceLibcInternal", internal_fseek);
     LIB_FUNCTION("lbB+UlZqVG0", "libSceLibcInternal", 1, "libSceLibcInternal", internal_fread);
+    LIB_FUNCTION("MUjC4lbHrK4", "libSceLibcInternal", 1, "libSceLibcInternal", internal_fflush);
     LIB_FUNCTION("uodLYyUip20", "libSceLibcInternal", 1, "libSceLibcInternal", internal_fclose);
 }
 

src/core/libraries/libc_internal/libc_internal_io.h

@@ -94,5 +94,4 @@ u64 PS4_SYSV_ABI internal_fread(char* ptr, u64 size, u64 nmemb, OrbisFILE* file)
 s32 PS4_SYSV_ABI internal_fclose(OrbisFILE* file);
 
 void RegisterlibSceLibcInternalIo(Core::Loader::SymbolsResolver* sym);
-void ForceRegisterlibSceLibcInternalIo(Core::Loader::SymbolsResolver* sym);
 } // namespace Libraries::LibcInternal

src/core/libraries/libs.cpp

@@ -83,7 +83,6 @@ namespace Libraries {
 void InitHLELibs(Core::Loader::SymbolsResolver* sym) {
     LOG_INFO(Lib_Kernel, "Initializing HLE libraries");
     Libraries::Kernel::RegisterLib(sym);
-    Libraries::LibcInternal::ForceRegisterLib(sym);
     Libraries::GnmDriver::RegisterLib(sym);
     Libraries::VideoOut::RegisterLib(sym);
     Libraries::UserService::RegisterLib(sym);

src/core/libraries/sysmodule/sysmodule_internal.cpp

@@ -434,6 +434,13 @@ s32 preloadModulesForLibkernel() {
             continue;
         }
 
+        if (module_index == 1 || module_index == 2) {
+            // libkernel and libSceLibcInternal aren't directly loaded here.
+            // All we do for those is increment is_loaded
+            g_modules_array[module_index].is_loaded++;
+            continue;
+        }
+
         // Load the actual module
         s32 result = loadModuleInternal(module_index, 0, nullptr, nullptr);
         if (result != ORBIS_OK) {

src/core/linker.cpp

@@ -72,11 +72,38 @@ void Linker::Execute(const std::vector<std::string>& args) {
     Module* module = m_modules[0].get();
     static_tls_size = module->tls.offset = module->tls.image_size;
 
+    // Map libSceLibcInternal
+    const auto& libc_internal_path =
+        EmulatorSettings.GetSysModulesDir() / "libSceLibcInternal.sprx";
+    if (std::filesystem::exists(libc_internal_path)) {
+        LoadModule(libc_internal_path);
+    }
+
     // Relocate all modules
     for (const auto& m : m_modules) {
         Relocate(m.get());
     }
 
+    // Before we can run guest code, we need to properly initialize the heap API and
+    // libSceLibcInternal. libSceLibcInternal's _malloc_init serves as an additional initialization
+    // function called by libkernel.
+    heap_api = new HeapAPI{};
+    static PS4_SYSV_ABI s32 (*malloc_init)() = nullptr;
+
+    for (const auto& m : m_modules) {
+        const auto& mod = m.get();
+        if (mod->name.contains("libSceLibcInternal.sprx")) {
+            // Found libSceLibcInternal, now search through function exports.
+            // Looking for _malloc_init to init libSceLibcInternal properly
+            // and for all the memory allocating functions, so we can initialize our heap API
+            for (const auto& sym : mod->export_sym.GetSymbols()) {
+                if (sym.nid_name.compare("_malloc_init") == 0) {
+                    malloc_init = reinterpret_cast<PS4_SYSV_ABI s32 (*)()>(sym.virtual_address);
+                }
+            }
+        }
+    }
+
     // Configure the direct and flexible memory regions.
     u64 fmem_size = ORBIS_KERNEL_FLEXIBLE_MEMORY_SIZE;
     bool use_extended_mem1 = true, use_extended_mem2 = true;
@@ -125,6 +152,15 @@ void Linker::Execute(const std::vector<std::string>& args) {
             ipc.WaitForStart();
         }
 
+        // Load libSceLibcInternal, run malloc_init.
+        LoadLibcInternal();
+
+        if (malloc_init != nullptr) {
+            // Call _malloc_init
+            s32 ret = malloc_init();
+            ASSERT_MSG(ret == 0, "malloc_init failed");
+        }
+
         // Have libSceSysmodule preload our libraries.
         Libraries::SysModule::sceSysmodulePreloadModuleForLibkernel();
 

src/core/linker.h

@@ -131,6 +131,14 @@ public:
         }
     }
 
+    void LoadLibcInternal() {
+        for (auto& module : m_modules) {
+            if (module->name.contains("libSceLibcInternal")) {
+                module->Start(0, nullptr, nullptr);
+            }
+        }
+    }
+
     void SetHeapAPI(void* func[]) {
         heap_api = reinterpret_cast<AppHeapAPI>(func);
     }

src/core/memory.h

@@ -52,6 +52,7 @@ enum class MemoryMapFlags : u32 {
     Stack = 0x400,
     NoSync = 0x800,
     Anon = 0x1000,
+    System = 0x2000,
     NoCore = 0x20000,
     NoCoalesce = 0x400000,
 };
@@ -100,6 +101,7 @@ enum class VMAType : u32 {
     Stack = 6,
     Code = 7,
     File = 8,
+    System = 9,
 };
 
 struct VirtualMemoryArea {