Merge branch 'master' into upd_banner_fmt

Author: Gemba

.gitignore

@@ -73,3 +73,6 @@ platforms_idmap_local.csv
 # VS Code
 *.code-workspace
 .vscode
+
+# docker config
+docker/config.ini

VERSION.ini

@@ -1 +1 @@
-VERSION="3.18.4"
+VERSION="3.18.5"

docker/Dockerfile

@@ -23,7 +23,9 @@ RUN rm -rf /var/lib/apt/lists/*
 RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo
 
 # Building skyscraper
-COPY ./* ./
+RUN mkdir -p /opt/skyscraper
+WORKDIR /opt/skyscraper
+COPY . .
 RUN rm -f ./VERSION.ini
 RUN chmod +x update_skyscraper.sh  && ./update_skyscraper.sh
 

docker/process_dir.sh

@@ -1,16 +1,23 @@
 #!/bin/bash
 
 # Usage:
-#   ./process_dir.sh [ROMS_DIR]
+#   ./process_dir.sh [ROMS_DIR] {source1} {source2} ...
 # Example
 #   ./process_dir.sh ~/roms
-#   ./process_dir.sh ~/roms > output.txt
+#   ./process_dir.sh ~/roms screenscraper > output.txt
 #
-# Note: can be use for Emulation Station roms folder, but platform name can be different. ES's genesis is megadrive for skyscraper,
+# Note: can be use for Emulation Station roms folder, but platform name can be different. ES's genesis is megadrive for skyscraper.
+# Defaults to screenscraper and thegamesdb sources if none are specified.
 
 TAG=skyscraper
 
 PLATFORMS=$1/*
+shift
+
+SOURCES=("$@")
+if [ ${#SOURCES[@]} -eq 0 ]; then
+  SOURCES=(screenscraper thegamesdb)
+fi
 
 mkdir cache
 
@@ -23,7 +30,7 @@ for file_name in $PLATFORMS; do
     echo "Processing ${PLATFORM}..."
     echo "Mounting $file_name:./roms/$PLATFORM..."
 
-    for source in "screenscraper" "thegamesdb"; do
+    for source in "${SOURCES[@]}"; do
       echo "Processing $PLATFORM on $source..."
 
       ./scrape.sh $file_name $PLATFORM $source

docker/save.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 
 # Usage:
-#   ./scrape.sh [ROMS_DIR] [PLATFORM]
+#   ./save.sh [ROMS_DIR] [PLATFORM]
 # Example
-#   ./scrape.sh ./roms/snes snes
+#   ./save.sh ./roms/snes snes
 
 TAG=skyscraper
 
@@ -14,4 +14,4 @@ docker run \
   -v "$ROMS_DIR:/tmp/roms/$PLATFORM" \
   -v "$(pwd)/cache:/tmp/skyscraper_cache" \
   $TAG \
-  -p $PLATFORM -i /tmp/roms/$PLATFORM -d /tmp/skyscraper_cache --flags relative,unattendskip
+  -p $PLATFORM -i /tmp/roms/$PLATFORM -g /tmp/roms/$PLATFORM -o /tmp/roms/$PLATFORM/media -d /tmp/skyscraper_cache --flags relative,unattend

docker/scrape.sh

@@ -11,8 +11,17 @@ ROMS_DIR=$1
 PLATFORM=$2
 SOURCE=$3
 
-docker run \
-  -v "$ROMS_DIR:/tmp/roms/$PLATFORM" \
-  -v "$(pwd)/cache:/tmp/skyscraper_cache" \
-  $TAG \
-  -p $PLATFORM -s $SOURCE -i /tmp/roms/$PLATFORM -d /tmp/skyscraper_cache
+if [ -f ./config.ini ]; then
+  docker run \
+    -v "$ROMS_DIR:/tmp/roms/$PLATFORM" \
+    -v "$(pwd)/cache:/tmp/skyscraper_cache" \
+    -v "$(pwd)/config.ini:/root/.skyscraper/config.ini" \
+    $TAG \
+    -p $PLATFORM -s $SOURCE -i /tmp/roms/$PLATFORM -d /tmp/skyscraper_cache
+else
+  docker run \
+    -v "$ROMS_DIR:/tmp/roms/$PLATFORM" \
+    -v "$(pwd)/cache:/tmp/skyscraper_cache" \
+    $TAG \
+    -p $PLATFORM -s $SOURCE -i /tmp/roms/$PLATFORM -d /tmp/skyscraper_cache
+fi

docs/CONFIGINI.md

@@ -1056,6 +1056,13 @@ or
 userCreds="<Client-ID>:<Client-Secret>"
 ```
 
+or
+
+```ini
+[thegamesdb]
+userCreds="<your-private-api-key-of-64chars>"
+```
+
 Default value: unset  
 Allowed in sections: `[<SCRAPER>]`
 

docs/SCRAPINGMODULES.md

@@ -96,7 +96,7 @@ I strongly recommend supporting them by contributing data to the database, or by
 - Website: _[www.thegamesdb.net](http://www.thegamesdb.net)_
 - Type: _File name search based_
 - User credential support: _Not required_
-- API request limit: _Limited to 3000 requests per IP per month_
+- API request limit: _Limited to 1000 requests per IP per month_
 - Thread limit: _None_
 - Platform support: _[Link to list](https://thegamesdb.net/list_platforms.php) or see `tgdb_platforms.json` sibling to your `config.ini`_
 - Media support: `backcover`, `cover`, `fanart`, `marquee`, `screenshot`, `wheel`
@@ -107,7 +107,7 @@ I strongly recommend supporting them by contributing data to the database, or by
 
 For newer games there's no way around TheGames DB. It recently had a huge redesign and their database remains one of the best out there. I would recommend scraping your roms with `screenscraper` first, and then use `thegamesdb` to fill out the gaps in your cache.
 
-There's a small caveat to this module, as it has a monthly request limit (see above) per IP per month. But this should be plenty for most people.
+There's a small caveat to this module, as it has a monthly request limit (see above) per IP per month. As per Feb 17th of 2026 TGDB have changed they API usage policy: You may have to register for your private API key to increase the above limits. Put your private API key then in the [`userCreds=`](CONFIGINI.md#usercreds) section of `[thegamesdb]`, or use the `-u` option when scraping with this module.
 
 Their API is based on a file name search. This means that the returned results do have a chance of being faulty. Skyscraper does a lot internally to make sure accepted data is for the correct game. But it is impossible to ensure 100% correct results, so do keep that in mind when using it. Consider using the `--flags interactive` command line flag if you want complete control of the accepted entries.
 

src/settings.cpp

@@ -699,7 +699,13 @@ void RuntimeCfg::applyCli(bool &inputFolderSet, bool &gameListFolderSet,
     if (parser->isSet("cache")) {
         config->cacheOptions =
             parser->value("cache").simplified().replace(" ", "");
-        if (config->cacheOptions == "refresh") {
+        bool valid = validateCacheSubCommand(config->cacheOptions);
+        if (!valid) {
+            printf("\033[1;31mInvalid parameter '--cache %s'\033[0m, "
+                   "please check '--cache help' for more info.\n",
+                   config->cacheOptions.toStdString().c_str());
+            exit(2);
+        } else if (config->cacheOptions == "refresh") {
             config->refresh = true;
             config->cacheOptions = "";
         } else if (config->cacheOptions == "help") {
@@ -709,8 +715,7 @@ void RuntimeCfg::applyCli(bool &inputFolderSet, bool &gameListFolderSet,
             Cli::cacheReportMissingUsage();
             exit(0);
         } else if (config->cacheOptions.startsWith("purge:")) {
-            bool valid = validatePurgeParameters(config->cacheOptions);
-            if (!valid) {
+            if (!validatePurgeParameters(config->cacheOptions)) {
                 printf(
                     "\033[1;31mInvalid purge: parameter '--cache %s'\033[0m, "
                     "please check '--cache help' for more info.\n",

src/thegamesdb.cpp

@@ -65,33 +65,54 @@ void TheGamesDb::getSearchResults(QList<GameEntry> &gameEntries,
             pIds.append(QString::number(p));
         }
     }
-
-    QString req =
-        searchUrlPre +
-        StrTools::unMagic(
-            "187;161;217;126;172;149;202;122;163;197;163;219;162;171;203;197;"
-            "139;151;215;173;122;206;161;162;200;216;217;123;124;215;200;170;"
-            "171;132;158;155;215;120;149;169;140;164;122;154;178;174;160;172;"
-            "157;131;210;161;203;137;159;117;205;166;162;139;171;169;210;163") +
-        "&name=" + searchName + "&filter[platform]=" + pIds.join(",");
+    QString k = getKey();
+    QString req = searchUrlPre % k % "&name=" % searchName %
+                  "&filter[platform]=" % pIds.join(",");
     netComm->request(req);
+    qDebug() << req;
     q.exec();
+    int httpStatus = netComm->getHttpStatus();
+    qDebug() << "HTTP status" << httpStatus;
     data = netComm->getData();
-    qDebug() << req;
-
     jsonDoc = QJsonDocument::fromJson(data);
+    QString apiStatus = jsonDoc.object()["status"].toString();
+
+    if (httpStatus >= 400) {
+        printf(
+            "\033[1;31mServer response status %d\033[0m: Cannot continue! See "
+            "https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/"
+            "Status/%d\n",
+            httpStatus, httpStatus);
+        reqRemaining = 0;
+        if (!apiStatus.isEmpty()) {
+            printf("\033[1;31mAPI status message\033[0m: %s\n",
+                   apiStatus.toStdString().c_str());
+        }
+        return;
+    }
+
     if (jsonDoc.isEmpty()) {
         return;
     }
 
-    reqRemaining = jsonDoc.object()["remaining_monthly_allowance"].toInt();
+    if (!apiStatus.isEmpty() && apiStatus != "Success") {
+        printf("\033[1;31mAPI status message\033[0m: %s. Cannot continue...\n",
+               apiStatus.toStdString().c_str());
+        reqRemaining = 0;
+        return;
+    }
+
+    QJsonObject obj = jsonDoc.object();
+    reqRemaining = obj["extra_allowance"].toInt();
+    if (reqRemaining <= 0) {
+        reqRemaining = obj["remaining_monthly_allowance"].toInt();
+    }
+
     if (reqRemaining <= 0)
         printf("\033[1;31mYou've reached TheGamesdDb's request limit for this "
-               "month.\033[0m\n");
+               "month with %s API key.\033[0m\n",
+               config->userCreds.isEmpty() ? "the public" : "your private");
 
-    if (jsonDoc.object()["status"].toString() != "Success") {
-        return;
-    }
     if (jsonDoc.object()["data"].toObject()["count"].toInt() < 1) {
         return;
     }
@@ -109,17 +130,11 @@ void TheGamesDb::getSearchResults(QList<GameEntry> &gameEntries,
         GameEntry game;
 
         game.id = QString::number(jsonGame["id"].toInt());
-        game.url =
-            QString("%1/Games/ByGameID?id=%2&apikey=%3&fields=%4")
-                .arg(baseUrl)
-                .arg(game.id)
-                .arg(QString(StrTools::unMagic(
-                    "187;161;217;126;172;149;202;122;163;197;163;219;162;"
-                    "171;203;197;139;151;215;173;122;206;161;162;200;216;"
-                    "217;123;124;215;200;170;171;132;158;155;215;120;149;"
-                    "169;140;164;122;154;178;174;160;172;157;131;210;161;"
-                    "203;137;159;117;205;166;162;139;171;169;210;163")))
-                .arg(fields.join(","));
+        game.url = QString("%1/Games/ByGameID?id=%2&apikey=%3&fields=%4")
+                       .arg(baseUrl)
+                       .arg(game.id)
+                       .arg(k)
+                       .arg(fields.join(","));
 
         game.title = jsonGame["game_title"].toString();
         // Remove anything at the end with a parentheses. 'thegamesdb' has a
@@ -301,4 +316,16 @@ void TheGamesDb::loadMaps() {
 
 QVector<int> TheGamesDb::getPlatformId(const QString platform) {
     return Platform::get().getPlatformIdOnScraper(platform, config->scraper);
+}
+
+QString TheGamesDb::getKey() {
+    QString k = config->userCreds;
+    if (k.isEmpty()) {
+        k = StrTools::unMagic(
+            "189;159;218;132;218;153;203;170;166;155;168;177;211;217;203;153;"
+            "137;152;167;175;126;161;209;162;150;165;216;128;173;168;149;169;"
+            "213;137;203;161;163;170;196;215;182;168;118;156;132;171;202;165;"
+            "153;177;168;208;155;134;212;166;164;163;161;184;168;163;160;211");
+    }
+    return k;
 }