CMake: Use DT_RPATH instead of DT_RUNPATH on Linux

Preloaded libraries that interpose dlopen() (e.g. NoMachine) prevented our
dependencies from being found, since only the caller's RUNPATH is searched.

CMakeLists.txt

@@ -84,6 +84,11 @@ if(LINUX)
   get_origin_relative_path(deps_rpath "${DEPS_PATH}/lib")
   set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
   set(CMAKE_INSTALL_RPATH "${deps_rpath}")
+
+  # Use DT_RPATH instead of DT_RUNPATH. dlopen() only searches the RUNPATH of the calling object, so a preloaded
+  # library that interposes dlopen() (e.g. NoMachine's libnxegl.so) stops us from finding our own libraries.
+  # The executable's RPATH is searched regardless of the caller.
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--disable-new-dtags")
 endif()
 
 # Enable large file support on Linux 32-bit platforms.

scripts/appimage/make-appimage.sh

@@ -131,6 +131,10 @@ $LINUXDEPLOY --plugin qt --appdir="$OUTDIR" --executable="$BUILDDIR/bin/duckstat
 --desktop-file="$ROOTDIR/scripts/appimage/org.duckstation.DuckStation.desktop" \
 --icon-file="$ROOTDIR/scripts/appimage/org.duckstation.DuckStation.png" \
 
+# linuxdeploy sets DT_RUNPATH, which dlopen() ignores when called through an interposing preloaded library.
+echo "Patching RPATH in $BINARY..."
+"$PATCHELF" --force-rpath --set-rpath '$ORIGIN/../lib' "$OUTDIR/usr/bin/$BINARY"
+
 echo "Copying resources into AppDir..."
 cp -a "$BUILDDIR/bin/resources" "$OUTDIR/usr/bin"
 

scripts/appimage/make-cross-appimage.sh

@@ -223,7 +223,7 @@ cp -a "$BUILDDIR/bin/duckstation-qt" "$BUILDDIR/bin/resources" "$BUILDDIR/bin/tr
 
 # Patch RPATH so the binary goes hunting for shared libraries in the AppDir instead of system.
 echo "Patching RPATH in duckstation-qt..."
-patchelf --set-rpath '$ORIGIN/../lib' "$OUTDIR/usr/bin/duckstation-qt"
+patchelf --force-rpath --set-rpath '$ORIGIN/../lib' "$OUTDIR/usr/bin/duckstation-qt"
 
 # Libraries we pull in from the system.
 echo "Copying system libraries..."