From 232c4b482afbcad757e59100277b0f61aa4253fe Mon Sep 17 00:00:00 2001
From: lightningterror <18107717+lightningterror@users.noreply.github.com>
Date: Sat, 26 Dec 2020 11:09:38 +0100
Subject: [PATCH] tools: Remove GameIndex.dbf-tool.

Deprecated with yaml db, maybe a new alternative can be made in the
future.
---
 .../pcsx2-GameIndex.dbf-updater.html          | 455 ------------------
 1 file changed, 455 deletions(-)
 delete mode 100644 tools/GameIndex.dbf-tool/pcsx2-GameIndex.dbf-updater.html
diff --git a/tools/GameIndex.dbf-tool/pcsx2-GameIndex.dbf-updater.html b/tools/GameIndex.dbf-tool/pcsx2-GameIndex.dbf-updater.html
deleted file mode 100644
index 853e026862..0000000000
--- a/tools/GameIndex.dbf-tool/pcsx2-GameIndex.dbf-updater.html
+++ /dev/null
@@ -1,455 +0,0 @@
-<html>
-<meta charset="utf-8">
-<title>PCSX2 games index compatibility updater</title>
-<head>
-<script>
-/*
-Quick and (very) dirty gameindex.dbf update tool from website compatibility db data.csv.
-- works in Firefox since Chrome doesn't allow loading local files.
-- updates compatibility info at the dbf from data.csv
-- add entries to the dbf if they only appear on data.csv
-- sorts the dbf entries according to the serial number
-- keeps hacks/comments/etc untouched (except when modifying compatibility info - and prints a warning)
-- can import external html files with games info and compare the name/region to our own data
-*/
-var CONFIG_SKIP_SORT_1 = 0; // sort after initial dbf import
-var CONFIG_SKIP_REGIONS_SUMMARY = 0;
-var CONFIG_SKIP_IMPORT_CSV = 0;
-var CONFIG_SKIP_ADD_MISSING_FROM_CSV = 0;
-var CONFIG_SKIP_LOG_CSV_MISSING = 0; // files which have compat info at the dbf but not at the csv
-var CONFIG_SKIP_LOG_IMPROVEMENTS = 0; // the common case of data.csv having better compat than the current dbf.
-var CONFIG_SKIP_SORT_2 = 0; // sort after adding entries from data.csv
-var CONFIG_SKIP_HTML_INDEX_IMPORTS = 1;
-
-var DBF_PATH = "GameIndex.dbf";
-var COMPAT_PATH = "data.csv";
-var HTML_INDEX_FILES = ["serials_1.htm", "serials_2.htm", "serials_3.htm"];
-
-var COMPAT_DELIMITER = "\t";
-var COMPAT_COLUMNS = {Serial: 0, Compat: 1, Name: 5, Region: 6};
-
-var DBF_KEYS =    ["Serial", "Name", "Region", "Compat"];
-var DBF_DELIMITER = "\n---------------------------------------------\n";
-var DBF_HEADER_END = DBF_DELIMITER + "-- Game List" + DBF_DELIMITER;
-
-
-function escapeHtml(str) { return str.split("&").join("&amp;")
-                                     .split("<").join("&lt;")
-                                     .split(">").join("&gt;")
-                                     .split("\"").join("&quot;"); }
-function $(id){ return document.getElementById(id); }
-function log(text) { $("logout").innerHTML += escapeHtml(text); }
-function logln(text) { log(text + "\n"); }
-
-function serPretty(ser) { ser = ser.toUpperCase(); return ser.substr(0,4) + "-" + ser.substr(4); }
-function noComment(txt) { return txt.split("//")[0].trim(); }
-function serNor(ser) { return noComment(ser).split("-").join("").toLowerCase(); }
-
-function pad(txt, len, ch) {
-  txt = "" + txt;
-  ch = ch || " ";
-  while (txt.length < len)
-    txt += ch;
-  return txt;
-}
-
-// Synchronous. Returns "" on failure, or file content on success.
-// Use Firefox. chrome doesn't allow loading local files even if at the same dir as this file.
-function getFile(URI, failQuietly) {
-  var result = "";
-
-  try {
-    function reqListener () {
-      result = this.responseText;
-      logln("Got file '" + URI + "', length: " + result.length);
-    }
-
-    var oReq = new XMLHttpRequest();
-    oReq.onload = reqListener;
-    oReq.open("get", URI, false);
-    oReq.overrideMimeType("text/plain; charset=x-user-defined");
-    oReq.send();
-  } catch(e) {
-      if (!failQuietly)
-        logln("Error file '" + URI + "' (" + e + ")");
-      return "";
-  }
-
-  return result;
-}
-
-var exportData = [];
-function parseDbf(dbf, startLine) {
-
-  function importGame(section, firstLine) {
-    if (!firstLine) firstLine = 0;
-    var result = {};
-    var extra = [];
-    var lines = String(section).trim().split("\n");
-    for (var i in lines) {
-      var line = lines[i].trim();
-      if (line.indexOf("--------") == 0 ){
-        logln("Warning: invalid separator? : '" + line + "'");
-      }
-      var lineLow = line.toLowerCase();
-      var foundKey = false;
-
-      for (var k in DBF_KEYS) {
-        var rest = line.substr(DBF_KEYS[k].length).trim();
-        if (lineLow.indexOf(DBF_KEYS[k].toLowerCase()) == 0 && rest.indexOf("=") == 0) {
-          // known key
-          var key = DBF_KEYS[k];
-          var value = rest.substr(1).trim();
-          if (!(key in result)) {
-            if (key == "Serial") {
-              value = value.split("-").join("");
-            }
-            result[key] = value;
-            foundKey = true;
-            break;
-
-          } else {
-            logln ("Error: rejecting DBF Line: " + (firstLine + i) + " redefine: " + DBF_KEYS[k]);
-          }
-        } else if (lineLow.indexOf(DBF_KEYS[k].toLowerCase()) == 0) {
-          // known yet without '=', warn.
-          logln("Warning: Line " + (firstLine + i) + ", no '='? '" + line + "'");
-        }
-      }
-
-      if (!foundKey) {
-        // unknown/no key (comment, patch, hack, empty, etc)
-        extra.push(lines[i]); // because 'line' is trimmed
-      }
-    } // lines
-
-    result["extra"] = extra;
-    return result;
-  }
-
-  var result = {header: "", games: []};
-
-	logln("import-dbf-start");
-  var dbf = getFile(DBF_PATH);
-
-	var dbfs = dbf.split(DBF_HEADER_END);
-  logln("found main sections: " + dbfs.length);
-
-  var gameSections = String(dbfs[1]).split(DBF_DELIMITER);
-  if (gameSections[gameSections.length-1].length == 0) {
-    gameSections.pop();
-  }
-  logln("Found game sections: " + gameSections.length);
-
-  logln("Importing games ...");
-  var gamesDb = [];
-  var serialIndex= {};
-  var compatInfo = [];
-  var dbfRegions = {};
-  for (var g in gameSections) {
-    var game = importGame(gameSections[g]);
-    if (!("Serial" in game) && !("Name" in game) && !("Region" in game) && !("Compat" in game) && game.extra.length == 1) {
-      logln("Warning: Empty section (skipping)");
-    } else {
-      if (!("Serial" in game) || !("Name" in game) || !("Region" in game)) {
-        logln("Warning: Incomplete: ser="+game.Serial.substr(0,4)+"-"+game.Serial.substr(4)+" name="+game.Name+ " region="+game.Region);
-      }
-      gamesDb.push(game);
-      if (game.Serial) {
-        var ser = serNor(game.Serial);
-        if (ser in serialIndex) {
-          logln("Warning: duplicate serial at dbf: " + serPretty(ser));
-        } else {
-          serialIndex[ser] = gamesDb.length - 1;
-        }
-      }
-      if ("Compat" in game) {
-        var c = game.Compat.substr(0,1);
-        if (!(c in compatInfo)) compatInfo[c] = 0;
-        compatInfo[c]++;
-      }
-    }
-    if (!dbfRegions[game.Region])
-      dbfRegions[game.Region] = 0;
-    dbfRegions[game.Region]++;
-  }
-  logln("Imported entries: " + gamesDb.length);
-  for (var i in compatInfo)
-    logln("Compatibility " + i + ": " + compatInfo[i] + " games");
-  
-  if (!CONFIG_SKIP_REGIONS_SUMMARY) {
-    for (var r in dbfRegions)
-      logln("" + r + ": " + dbfRegions[r]);
-  }
-    
-
-  function sortDbf() {
-    gamesDb.sort(function(a, b) {
-      // no need for equal since there are no duplicates
-      return serNor(a.Serial) > serNor(b.Serial) ? 1 : -1;
-    });
-    for (var i = 0; i < gamesDb.length; i++) {
-      serialIndex[serNor(gamesDb[i].Serial)] = i;
-    }
-  }
-
-  logln("Checking sort ...");
-  var prevg = "";
-  var sorted = true;
-  for (var g in serialIndex) {
-    if (g < prevg) {
-      logln ("bad sort: " + serPretty(prevg) + " before " + serPretty(g));
-      sorted = false;
-    }
-    prevg = g;
-  }
-  
-  if (sorted) {
-    logln("dbf sort: no need (already sorted)");
-  } else {
-    if (CONFIG_SKIP_SORT_1) {
-      logln("skipping sort1");
-    } else {
-      logln("Sorting dbf...");
-      sortDbf();
-      logln("Done Sorting dbf.");
-    }
-  }
-
-  logln("import-dbf-end");
-
-  function dbfRegionFromCompatRegion(csvRegion) {
-    // currently data.csv only has NTSC/PAL/JAP
-    // we're translating to the dbf as: NTSC-U/PAL-Unk/NTSC-J respectively
-    // can later manually fix or set more specific region data.
-    // return undefined if unexpected region name at data.csv
-    var trans = {
-      "NTSC": "NTSC-U",
-      "PAL" : "PAL-Unk",
-      "JAP" : "NTSC-J"
-    };
-    return trans[csvRegion];
-  }
-
-  logln("");
-  if (CONFIG_SKIP_IMPORT_CSV) {
-    logln("skipping data.csv import");
-  } else {
-    logln("start import compatibility ...");
-    var compat = getFile(COMPAT_PATH);
-    compat = compat.split("\n")
-    logln("Found " + compat.length + " compatibility info lines");
-    var updateSummary = {same: 0, "new": 0, better: 0, worse: 0, missingAtCsv: 0};
-    var compatVerification = {};
-
-    var compatRegions = {};
-    for (var i in compat) {
-      if (!compat[i].trim().length) continue;
-      var cols = compat[i].split(COMPAT_DELIMITER);
-      var found = false;
-      var ser = cols[COMPAT_COLUMNS.Serial].trim().toLowerCase();
-      var com = Number(cols[COMPAT_COLUMNS.Compat].trim());
-      var reg = cols[COMPAT_COLUMNS.Region].trim();
-
-      if (!compatRegions[reg])
-        compatRegions[reg] = 0;
-      compatRegions[reg]++;
-
-      if (!(ser in compatVerification)) {
-        compatVerification[ser] = com;
-      } else {
-        logln("Warning (skipping): duplicate compat info (" + compatVerification[ser] + "/" + com + ") for: " + serPretty(ser));
-        continue;
-      }
-      if (ser.toLowerCase() in serialIndex) {
-        var game = gamesDb[serialIndex[ser]];
-        var dbser = serNor(game.Serial);
-        var dbcom = Number(("Compat" in game) ? game.Compat.split("//")[0].trim() : "0");
-        if (dbser != ser) {
-          logln("Error: internal: invalid index at serialIndex for " + serPretty(ser));
-          continue;
-        }
-        
-        if (com != dbcom) {
-          //logln("Updating compat from " + dbcom + " to " + com + " for " + ser + ": " + game.Name);
-          if (("Compat" in game) && game.Compat.split("//").length > 1) {
-            logln("Warning: removing compat comment for serial " + serPretty(ser) + " : //" + game.Compat.split("//")[1]);
-          }
-          game.Compat = com;
-          
-          if (com > dbcom) {
-            updateSummary.better++
-            if (!CONFIG_SKIP_LOG_IMPROVEMENTS)
-              logln("--> ++ Improving compat from " + dbcom + " to " + com + " for " + serPretty(ser) + ": " + game.Name);
-          } else {
-            updateSummary.worse++;
-            logln("--> -- Degrading compat from " + dbcom + " to " + com + " for " + serPretty(ser) + ": " + game.Name);
-          }
-        } else {
-          updateSummary.same++;
-        }
-      } else {
-        updateSummary.new++;
-        logln ("not at dbf: " + serPretty(ser) + " - " + (CONFIG_SKIP_ADD_MISSING_FROM_CSV ? "skipping" : "adding:"));
-        if (!CONFIG_SKIP_ADD_MISSING_FROM_CSV) {
-          var newDbfReg = dbfRegionFromCompatRegion(reg);
-          if (newDbfReg) {
-            gamesDb.push({Serial: ser, Compat: com, Region: newDbfReg, Name: cols[COMPAT_COLUMNS.Name].trim(), extra: ""});
-            log(getExportEntry(gamesDb[gamesDb.length - 1]) + DBF_DELIMITER);
-          } else {
-            logln("Error: skipping due to unexpected region at data.csv: " + reg);
-          }
-        }
-      }
-    }
-    if (compat && compat.length > 1) {
-      if (updateSummary.new) {
-        if (CONFIG_SKIP_SORT_2) {
-          logln("skipping sort after add");
-        } else {
-          logln("sorting new entries into the dbf...");
-          sortDbf();
-          logln("sort - done.");
-        }
-      }
-      logln("-- Update summary --");
-
-      if (!CONFIG_SKIP_REGIONS_SUMMARY) {
-        logln("data.csv regions:");
-        for (var r in compatRegions)
-          logln("" + r + ": " + compatRegions[r]);
-      }
-      if (!CONFIG_SKIP_LOG_CSV_MISSING) {
-        for (var csvSer in serialIndex) {
-          var game = gamesDb[serialIndex[csvSer]];
-          if (game.Compat && !compatVerification[serNor(csvSer)]) {
-            logln("serial missing from CSV: " + pad(serPretty(csvSer), 12) + " / " + pad(noComment(game.Region), 8) + " / " + game.Compat + " / " + game.Name);
-            updateSummary.missingAtCsv++;
-          }
-        }
-      }
-   
-      logln("Not at dbf (and " +(CONFIG_SKIP_ADD_MISSING_FROM_CSV ? "skipped" : "added")+ "): " + updateSummary.new);
-      logln("Not at csv (but have compat info at the dbf): " + updateSummary.missingAtCsv);
-      logln("Unchanged: " + updateSummary.same);
-      logln("Better compat: " + updateSummary.better);
-      logln("Worse compat: " + updateSummary.worse);
-      logln("End import compatibility.");
-    }
-  }
-
-
-  logln("");
-  if (CONFIG_SKIP_HTML_INDEX_IMPORTS) {
-    logln("skipping html index files");
-  } else {
-    logln("Scanning html index files");
-    var in_incomplete = 0;
-    var in_identical = 0;
-    var in_diff = 0;
-    var in_new = 0;
-    var in_err = 0;
-    var in_usBetter = 0;
-    var files = HTML_INDEX_FILES;
-    for (var f in files) {
-      var serials = getFile(files[f]);
-      logln("file sections: " + serials.split('<table border="1"').length);
-      if (!serials.length) {
-        logln("Warning: file not found. skipping.");
-        continue;
-      }
-      var list = serials.split('<table border="1"')[1].split("</tr>");
-      logln("found rows: " + list.length);
-      for (var l in list) {
-        var s = list[l].split("</td>");
-        if (!(s.length==4)) {
-          in_incomplete++;
-          continue;
-        }
-        try {
-          var ser = serNor(s[0].split('.htm">')[1].split("</pre>")[0]);
-          var name = s[1].split(' 0">')[1].split("</pre>")[0];
-          var region = s[2].split(' 0">')[1].split("</pre>")[0];
-
-          if (ser in serialIndex) {
-            var game = gamesDb[serialIndex[ser]];
-            function norName(n) {
-              n = n.toLowerCase().split("//")[0];
-              var cs = [", ", ".", "-", ":"];
-              for (var i in cs)
-                n = n.split(cs[i]).join("");
-              return n.split("  ").join("").split(" ").join("").trim();
-            }
-            if (norName(game.Name) == norName(name) && norName(game.Region) == norName(region)) {
-              in_identical++
-            } else {
-              if ((norName(region).indexOf("unk")>=0 && norName(game.Region).indexOf("unk")<0)|| norName(name).indexOf("zzz")==0) 
-                in_usBetter++
-              else {
-                logln("diff: " + serPretty(ser) + " DB: [" + game.Region.split("//")[0].trim() + "] " + game.Name.split("//")[0].trim());
-                logln("              index: [" + region + "] "+ name);
-                logln("");
-                in_diff++;
-              }
-            }
-          } else {
-            in_new++;
-          }
-        } catch (e) {
-          in_err++;
-          logln(e);
-        }
-      }
-    }
-    logln("--- Html index summary: ---");
-    logln("errors: " + in_err);
-    logln("incomplete data: " + in_incomplete);
-    logln("identical: " + in_identical);
-    logln("we're better: " + in_usBetter);
-    logln("different: "+ in_diff);
-    logln("not in db: " + in_new);
-  }
-
-
-  // start: test export dbf
-  function getExportEntry(gameItem) {
-    var entry = [];
-    for (var k in DBF_KEYS) {
-      if (DBF_KEYS[k] in gameItem) {
-        var v = gameItem[DBF_KEYS[k]];
-        var fill = "";
-        if (DBF_KEYS[k] == "Serial") v = (v.substr(0,4) + "-" + v.substr(4)).toUpperCase();
-        if (DBF_KEYS[k] == "Name") fill = "  ";
-        entry.push(DBF_KEYS[k] + fill + " = " + v);
-      }
-    }
-    if (gameItem.extra.length) {
-      entry.push(gameItem.extra.join("\n"));
-    }
-    return entry.join("\n");
-  }
-  exportData = [];
-  for (var g in gamesDb) {
-    exportData.push(getExportEntry(gamesDb[g]));
-  }
-  exportData = dbfs[0] + DBF_HEADER_END + exportData.join(DBF_DELIMITER) + DBF_DELIMITER;
-  $("saveUpdated").href = "data:bin/plain;base64," + window.btoa(exportData);
-  $("saveUpdatedWrapper").style.display = "inline";
-  // end: export
-
-}
-
-</script>
-</head>
-
-<body onload="javascript:parseDbf()">
-Please copy GameIndex.dbf (from the bin folder) and data.csv (compat info) to the folder of this html file<br/>
-<span id="saveUpdatedWrapper" style="display:none;">
-<a id="saveUpdated" href="" download="GameIndex.new.dbf">Save updated DBF</a>
-</span>
-
-<h4>Log</h4>
-<pre><div id="logout"></div></pre>
-
-</body>
-</html>
