diff --git a/config.ini.example b/config.ini.example index c4e91e41..4b41d7b0 100644 --- a/config.ini.example +++ b/config.ini.example @@ -69,6 +69,7 @@ ;region="wor" ;langPrios="en,de,es" ;regionPrios="eu,us,ss,uk,wor,jp" +;regionFromFilename="inline" ;minMatch="0" ;artworkXml="" ;relativePaths="false" diff --git a/docs/CONFIGINI.md b/docs/CONFIGINI.md index 65ed1f01..83d83fe9 100644 --- a/docs/CONFIGINI.md +++ b/docs/CONFIGINI.md @@ -122,6 +122,7 @@ This is an alphabetical index of all configuration options their usage level and | [platform](CONFIGINI.md#platform) | Basic | Y | | | | | [pretend](CONFIGINI.md#pretend) | Basic | Y | Y | | | | [region](CONFIGINI.md#region) | Basic | Y | Y | | | +| [regionFromFilename](CONFIGINI.md#regionfromfilename) | Advanced | Y | Y | | | | [regionPrios](CONFIGINI.md#regionprios) | Expert | Y | Y | | | | [relativePaths](CONFIGINI.md#relativepaths) | Basic | Y | Y | | | | [scummIni](CONFIGINI.md#scummini) | Advanced | Y | | | | @@ -791,13 +792,62 @@ region="de" --- -#### regionPrios +#### regionFromFilename -Completely overwrites the internal region priority list inside of Skyscraper. Multiple regions can be configured here separated by commas. Read more about how [regions are handled in general](REGIONS.md). +With this parameter (introduced with Skyscraper 3.20) you can control at which position detected regions from a game filename will be put. `first` means all detected region will be put first as in the order they are within the filename (this is the pre Skyscraper 3.20 behaviour). `insert` will pigeon-hole a detected region at the position at the region prios list, and will put any additionally detected region which is not on the region prios list at the end of the region prios list. The region prio list is calculated for each game file before performing the scraping. +You can also disable the filename detection of regions by setting this to `off`. -!!! info +Accepted values: `first`, `inline`, `off` +Default value: `inline` +Allowed in sections: `[main]`, `[]` + +**Example(s)** + +Set to `inline` mode: + +```ini +[snes] +regionFromFilename="inline" +regionPrios="eu, br, us, jp" +``` +... with game filename 'Game A (Japan, USA).zip' +will result in the region prio list +"eu, br, us, jp". Note the different order between filename region and position in list. + +... with game filename 'Game B (USA, Europe).zip' +will result in the region prio list +"eu, br, us, jp". Note the different order between filename region and position in list. + +... with game filename 'Game C (USA, World, Europe).zip' +will result in the region prio list +"eu, br, us, jp, wor". Note the world at end of list as it was not on the configured `regionPrios`. + +When set to `first` mode: + +```ini +[snes] +regionFromFilename="first" +regionPrios="eu, br, us, jp" +``` +... with game filename 'Game A (Japan, USA).zip' +will result in the region prio list +"jp, us, eu, br". Note the regions in the order of the filename put first. + +... with game filename 'Game B (USA, Europe).zip' +will result in the region prio list +"us, eu, br, jp". Same here, but compare this to the inline example above. + +... with game filename 'Game C (USA, World, Europe).zip' +will result in the region prio list +"us, wor, eu, br, jp". Note "World" at the beginning and in order of the position in the filename. + + +--- + +#### regionPrios - Any region [auto-detected](REGIONS.md#region-auto-detection) from the file name will still be added to the top of this list. +Completely overwrites the internal region priority list inside of Skyscraper. Multiple regions can be configured here separated by commas. Read more about how [regions are handled in general](REGIONS.md). Do not configure the region prios too narrow, as you might not find a match for every game in your collection then, always put one or some fail-safe(s) at the end of the list. +Any region [auto-detected](REGIONS.md#region-auto-detection) from the file name will still be added to the end or the beginning of the region prios list unless it is in the region prios list already (see also [regionsFromFile](#regionsfromfile)). If a region from the filename is already in the region prios list, then the order is kept as defined. Default value: `eu, us, ss, uk, wor, jp, au, ame, de, cus, cn, kr, asi, br, sp, fr, gr, it, no, dk, nz, nl, pl, ru, se, tw, ca` Allowed in sections: `[main]`, `[]` diff --git a/docs/REGIONS.md b/docs/REGIONS.md index 15ff1c18..8d24ce2e 100644 --- a/docs/REGIONS.md +++ b/docs/REGIONS.md @@ -1,6 +1,6 @@ ## Overview -Some game information and / or game media is region-based. Skyscraper provides several ways of configuring these for your convenience. But most importantly; it supports region auto-detection directly from the file names. Read on for more information about how regions are handled by Skyscraper. +Some game information and / or game media is region-based (e.g., release date, artwork). Skyscraper provides several ways of configuring these for your convenience. But most importantly; it supports region auto-detection directly from the file names. Read on for more information about how regions are handled by Skyscraper. ## Scraping modules that support regions @@ -60,22 +60,26 @@ When configuring regions be sure to use the short-names as shown (eg. 'fr' for F ### Region auto-detection -Skyscraper will try to auto-detect the region from the file name. It will look for designations such as `(Europe)` or `(e)` and set the region accordingly. This currently works for the following regions and / or countries: +Skyscraper will try to auto-detect the region from the file name. It will look for designations in parenthesis such as `(Europe)` or `(e)` or combinations like `(USA, Japan)` and set the region priorities accordingly. This currently works for the following regions and / or countries: - asi, au, br, ca, cn - de, dk, eu, fr, it - jp, kr, nl, se, sp - tw, us, wor -So if your files are named like `Game Name (Europe).zip`, there's no need to configure regions manually. Skyscraper will recognize `Europe` and add it to the top of the internal region priority list. If info or media isn't found for the auto-detected region, it will move down the list and check the next region on the list until it finds one that has data for the requested resource. +So if your files are named like `Game Name (Europe).zip`, there's no need to configure regions manually. Skyscraper will recognize `Europe` and verfifies if it is on the region prios list, unless you disabled the region from filename detection (see configuration option [regionFromFilename](CONFIGINI.md#regionfromfilename)). The default behaviour is: +- If a detected region is in the region prios list, then the position in the configured region prios matters for finding a scraping match for the game. +- If it is not, the detected region from the filename is added to the end to the region prios list. +- If you set `regionFromFilename` to `"first"`, then every detected region is prepended to the region list in the order they appear in the filename. +Skyscraper will process the region prios list from begin to end and checks the region on the list until it finds one that has data for the requested resource. Do not configure the region prios too narrow, as you might not find a match for every game in your collection then, always put some fail-safes at the end of the list. ### Default Region Prioritization Skyscraper's default internal region priority list is as follows. Topmost region has highest priority and so forth. -- User-specified region set with `--region ` (command line) or `region=""` (config.ini) -- If no user-specified region is set, the [auto-detected](REGIONS.md#region-auto-detection) region will be added here -- Then this list is processed in order: eu, us, ss (Screenscraper specific), uk, wor, jp, au, ame, de, cus, cn, kr, asi, br, sp, fr, gr, it, no, dk, nz, nl, pl, ru, se, tw, ca +1. User-specified region set with `--region ` (command line) or `region=""` (config.ini). The `regionPrios=` setting is not applied in this case. +2. If no user-specified region is set, the [auto-detected](REGIONS.md#region-auto-detection) region(s) will be added at the end of the region prios in the order they appear in the filename, unless a detected region is already in the region prio list. In this case the priority for a region is according to the position in the region prio list. You can also prepend any detected region from the filename first on the region prios list or disable region detection from filename at all. See configuration option [regionFromFilename](CONFIGINI.md#regionfromfilename). +3. Then this list is processed in order by default: eu, us, ss (Screenscraper specific), uk, wor, jp, au, ame, de, cus (Screenscraper specific), cn, kr, asi, br, sp, fr, gr, it, no, dk, nz, nl, pl, ru, se, tw and ca. If you have configured a region prios list, the list will be processed from left to right. ## Configuring Region Manually diff --git a/src/abstractscraper.cpp b/src/abstractscraper.cpp index 1c98ae43..5d32d588 100644 --- a/src/abstractscraper.cpp +++ b/src/abstractscraper.cpp @@ -37,6 +37,10 @@ #include static const QRegularExpression RE_THE = QRegularExpression(", [Tt]he"); +static const QRegularExpression RE_VERSION = QRegularExpression( + " v[.]{0,1}([0-9]{1}[0-9]{0,2}[.]{0,1}[0-9]{1,4}|[IVX]{1,5})$"); +static const QRegularExpression RE_REGIONS = QRegularExpression( + "\\((\\D+?)\\)", QRegularExpression::CaseInsensitiveOption); AbstractScraper::AbstractScraper(Settings *config, QSharedPointer manager, @@ -609,9 +613,7 @@ QString AbstractScraper::getCompareTitle(const QFileInfo &info) { } // Remove "vX.XXX" versioning string if one is found - match = QRegularExpression( - " v[.]{0,1}([0-9]{1}[0-9]{0,2}[.]{0,1}[0-9]{1,4}|[IVX]{1,5})$") - .match(compareTitle); + match = RE_VERSION.match(compareTitle); if (match.hasMatch() && match.capturedStart(0) != -1) { compareTitle = compareTitle.left(match.capturedStart(0)).simplified(); } @@ -620,27 +622,82 @@ QString AbstractScraper::getCompareTitle(const QFileInfo &info) { } void AbstractScraper::detectRegionFromFilename(const QFileInfo &info) { + // the next statement redundant, but leave it here for the unit tests + regionPrios = config->regionPrios; + const QString fn = info.fileName(); - if (int leftParPos = fn.indexOf("("); leftParPos != -1) { - // Autodetect region and append to region priorities - QString regionString = fn.mid(leftParPos, fn.length()); - QListIterator> iter(regionMap()); - while (iter.hasNext()) { - QPair e = iter.next(); - QStringList keys = e.first.split("|"); - for (const auto &k : keys) { - if (regionString.contains(k, Qt::CaseInsensitive)) { - // regionMap is sorted from bigger regions to smaller - // prepend() assures smaller regions get higher priority - regionPrios.prepend(e.second); - if (keys.size() > 1) { - // append only one: "europe" or "(e)" - break; + QRegularExpressionMatchIterator matchIter = RE_REGIONS.globalMatch(fn); + QStringList addRegionPrios; + const bool regionsInline = config->regionFromFilename == "inline"; + + // loop over region infos from filename + while (matchIter.hasNext()) { + QString regionString = matchIter.next().captured().toLower(); + if (regionString == "(jue)") { + regionString = "japan|usa|europe"; + } else if (regionString == "(ue)") { + regionString = "usa|europe"; + } else if (regionString != "(e)" && regionString != "(j)" && + regionString != "(u)") { + // remove parenthesis + regionString = regionString.mid(1, regionString.length() - 2); + } + while (!regionString.isEmpty()) { + bool detectedRegion = false; + QListIterator> iter(regionMap()); + while (iter.hasNext()) { + QPair e = iter.next(); + QString fn_regio = e.first; + QString sky_regio_key = e.second; + if (regionString.startsWith(fn_regio)) { + qDebug() << "matched" << fn_regio; + // map to Skyscraper's short-names (sky_regio_key) + if (regionsInline) { + if (!regionPrios.contains(sky_regio_key) && + !addRegionPrios.contains(sky_regio_key)) { + addRegionPrios.append(sky_regio_key); + } + } else { + // regionFromFilename == "first" + if (!addRegionPrios.contains(sky_regio_key)) { + addRegionPrios.append(sky_regio_key); + } } + regionString = regionString.replace(fn_regio, ""); + if (!regionString.isEmpty()) { + // remove possible separators (comma et al.) if + // regionString was "europe, japan" -> retain "japan" + regionString = regionString.replace( + QRegularExpression("^([^a-z]+)?"), ""); + } + detectedRegion = true; + break; } } + if (!detectedRegion) { + // no match was found in regionMap() + break; + } } } + + QStringList rankedRegionPrios; + QStringList retainedRegionPrios = regionPrios; + for (int i = regionPrios.size() - 1; i >= 0; i--) { + const QString prioRegion = regionPrios.at(i); + if (addRegionPrios.contains(prioRegion)) { + if (regionsInline) { + rankedRegionPrios.prepend(prioRegion); + addRegionPrios.removeAt(addRegionPrios.indexOf(prioRegion)); + } + retainedRegionPrios.removeAt( + retainedRegionPrios.indexOf(prioRegion)); + } + } + if (regionsInline) + regionPrios = rankedRegionPrios + retainedRegionPrios + addRegionPrios; + else + regionPrios = addRegionPrios + retainedRegionPrios; } void AbstractScraper::runPasses(QList &gameEntries, @@ -648,7 +705,7 @@ void AbstractScraper::runPasses(QList &gameEntries, QString &debug) { // Reset region priorities to original list from Settings regionPrios = config->regionPrios; - if (config->region.isEmpty()) { + if (config->region.isEmpty() && config->regionFromFilename != "off") { detectRegionFromFilename(info); } @@ -747,10 +804,10 @@ QVariantMap AbstractScraper::readJson(const QString &filename) { "fix.\nNot scraping...\n\033[0m", filename.toUtf8().constData()); } else if (jsonObj.isEmpty()) { - ncprintf( - "\033[1;31mFile '%s' has invalid JSON format. Please fix.\nNot " - "scraping...\n\033[0m", - filename.toUtf8().constData()); + ncprintf("\033[1;31mFile '%s' has insky_regio_keyid JSON format. " + "Please fix.\nNot " + "scraping...\n\033[0m", + filename.toUtf8().constData()); } return m; } diff --git a/src/abstractscraper.h b/src/abstractscraper.h index 9db39cfe..6faebe46 100644 --- a/src/abstractscraper.h +++ b/src/abstractscraper.h @@ -172,10 +172,17 @@ public slots: const inline QList> regionMap() { // use list of pairs to maintain order return QList>{ - QPair("europe|(e)", "eu"), - QPair("usa|(u)", "us"), + QPair("europe", "eu"), + QPair("(e)", "eu"), + QPair("eu", "eu"), + QPair("usa", "us"), + QPair("(u)", "us"), + QPair("us", "us"), QPair("world", "wor"), - QPair("japan|(j)", "jp"), + QPair("wor", "wor"), + QPair("japan", "jp"), + QPair("(j)", "jp"), + QPair("jp", "jp"), QPair("brazil", "br"), QPair("korea", "kr"), QPair("taiwan", "tw"), @@ -185,6 +192,7 @@ public slots: QPair("spain", "sp"), QPair("china", "cn"), QPair("australia", "au"), + QPair("aus", "au"), QPair("sweden", "se"), QPair("canada", "ca"), QPair("netherlands", "nl"), diff --git a/src/igdb.cpp b/src/igdb.cpp index 72edb1c9..d991c59a 100644 --- a/src/igdb.cpp +++ b/src/igdb.cpp @@ -213,18 +213,20 @@ void Igdb::getGameData(GameEntry &game) { void Igdb::getReleaseDate(GameEntry &game) { QJsonArray jsonDates = jsonObj["release_dates"].toArray(); bool regionMatch = false; - QStringList skyscraperRegions = {"eu", "us", "au", "nz", "jp", - "cn", "asi", "wor", "kr", "br"}; + /* http --print bH https://api.igdb.com/v4/release_date_regions + * "Client-ID:" "Authorization: Bearer " --raw='fields region;' + */ + // regions as Skyscraper keywords + QStringList igdbRegions = {"eu", "us", "au", "nz", "jp", + "cn", "asi", "wor", "kr", "br"}; for (const auto ®ion : regionPrios) { for (const auto &jsonDate : jsonDates) { - /* http --print bH https://api.igdb.com/v4/release_date_regions - * "Client-ID:<>" "Authorization: Bearer <>" --raw='fields region;' - */ + int regionEnum = jsonDate.toObject()["region"].toInt(); QString curRegion = ""; - if (regionEnum > 0 && regionEnum < skyscraperRegions.length()) { + if (regionEnum > 0 && regionEnum < igdbRegions.length()) { // resolve to skyscraper region identifier - curRegion = skyscraperRegions.at(regionEnum); + curRegion = igdbRegions.at(regionEnum); } if (QString::number(jsonDate.toObject()["platform"].toInt()) == game.id.split(";").last() && diff --git a/src/settings.cpp b/src/settings.cpp index 97ea4f47..084b406e 100644 --- a/src/settings.cpp +++ b/src/settings.cpp @@ -357,6 +357,18 @@ void RuntimeCfg::applyConfigIni(CfgType type, QSettings *settings, config->region = v; continue; } + if (k == "regionFromFilename") { + QStringList allowed = QStringList({"first", "inline", "off"}); + if (allowed.contains(v)) { + config->regionFromFilename = v; + } else { + ncprintf("\033[1;33mValue '%s' of parameter %s is ignored. " + "Valid values are: %s.\n\033[0m", + v.toUtf8().constData(), k.toUtf8().constData(), + allowed.join(", ").toUtf8().constData()); + } + continue; + } if (k == "regionPrios") { config->regionPriosStr = v; continue; diff --git a/src/settings.h b/src/settings.h index 0f24250d..d1908dd6 100644 --- a/src/settings.h +++ b/src/settings.h @@ -161,6 +161,7 @@ struct Settings { QString region = ""; QString langPriosStr = ""; QString regionPriosStr = ""; + QString regionFromFilename = "inline"; QString searchName = ""; @@ -265,6 +266,7 @@ class RuntimeCfg : public QObject { {"platform", QPair("str", CfgType::MAIN )}, {"pretend", QPair("bool", CfgType::MAIN | CfgType::PLATFORM )}, {"region", QPair("str", CfgType::MAIN | CfgType::PLATFORM )}, + {"regionFromFilename", QPair("str", CfgType::MAIN | CfgType::PLATFORM )}, {"regionPrios", QPair("str", CfgType::MAIN | CfgType::PLATFORM )}, {"relativePaths", QPair("bool", CfgType::MAIN | CfgType::PLATFORM )}, {"scummIni", QPair("str", CfgType::MAIN )}, diff --git a/test/abstractscraper/test_abstractscraper.cpp b/test/abstractscraper/test_abstractscraper.cpp index cf37b06f..55cb0820 100644 --- a/test/abstractscraper/test_abstractscraper.cpp +++ b/test/abstractscraper/test_abstractscraper.cpp @@ -13,9 +13,12 @@ class TestAbstractScraper : public QObject { AbstractScraper *scraper; void match(QFileInfo &info, QList &expected) { + qDebug() << "From file:" << info.fileName() + << "(no regionPrios configured)"; scraper->detectRegionFromFilename(info); QCOMPARE(scraper->getRegionPrios().size(), expected.size()); QCOMPARE(scraper->getRegionPrios(), expected); + qDebug() << "Got:" << expected; } private slots: @@ -30,32 +33,107 @@ private slots: } void testDetectRegionsFromFilename2() { + // "world" shall not be detected as it is not in parenthesis scraper = new AbstractScraper(&settings, NULL); QFileInfo info("Gametitle (j) world.zip"); QList regionPriosExp; regionPriosExp.append("jp"); - regionPriosExp.append("wor"); match(info, regionPriosExp); } void testDetectRegionsFromFilename3() { + // "world" shall not be detected as it is not in parenthesis scraper = new AbstractScraper(&settings, NULL); QFileInfo info("Gametitle (france) wOrLD (j).zip"); QList regionPriosExp; regionPriosExp.append("fr"); regionPriosExp.append("jp"); - regionPriosExp.append("wor"); match(info, regionPriosExp); - qDebug() << info.completeBaseName(); - qDebug() << regionPriosExp; + } + + void testDetectRegionsFromFilename5() { + scraper = new AbstractScraper(&settings, NULL); + QFileInfo info("Gametitle (e) (u).zip"); + QList regionPriosExp = QStringList({"eu", "us"}); + match(info, regionPriosExp); } void testDetectRegionsFromFilename4() { scraper = new AbstractScraper(&settings, NULL); - QFileInfo info("Gametitle (usa) (u).zip"); + QFileInfo info("Gametitle (Usa) (u).zip"); QList regionPriosExp; regionPriosExp.prepend("us"); match(info, regionPriosExp); + info = QFileInfo("Gametitle (Usa).zip"); + match(info, regionPriosExp); + } + + QStringList + setupExpectedRegionPrios(const QStringList &configuredRegionPrios, + QString firstRegion) { + QStringList ret = configuredRegionPrios; + if (int idx = ret.lastIndexOf(firstRegion); idx > -1) { + ret.removeAt(idx); + } + ret.prepend(firstRegion); + return ret; + } + + void matchRegion(QString fn, QStringList regionPriosExp) { + QFileInfo game(fn); + qDebug() << "From file:" << game.fileName(); + qDebug() << "Expected: " << regionPriosExp; + scraper->detectRegionFromFilename(game); + qDebug() << "Actual: " << scraper->getRegionPrios(); + QCOMPARE(scraper->getRegionPrios(), regionPriosExp); + } + + void testDetectRegionsFromFilenameIssue242OptionInline() { + // "br" surplus + settings.regionPrios = QStringList({"eu", "br", "us", "jp"}); + settings.regionFromFilename = "inline"; + qDebug() << "Configured region prios:" << settings.regionPrios; + scraper = new AbstractScraper(&settings, NULL); + + QList regionPriosExp = settings.regionPrios; + matchRegion("Game A (Japan, USA).zip", regionPriosExp); + matchRegion("Game A' (us, jp).zip", regionPriosExp); + + matchRegion("Game B (USA, Europe).zip", regionPriosExp); + + // "wor" should be last as there is no match in regionPrios + regionPriosExp = settings.regionPrios + QStringList({"wor"}); + matchRegion("Game C (USA, World, Europe).zip", regionPriosExp); + + settings.regionPrios = QStringList({"jp", "eu"}); + regionPriosExp = QStringList({"jp", "eu", "us"}); + matchRegion("Game D (UE).zip", regionPriosExp); + settings.regionPrios = QStringList({"eu"}); + regionPriosExp = QStringList({"eu", "jp", "us"}); + matchRegion("Game D' (JUE).zip", regionPriosExp); + + settings.regionPrios = QStringList({"eu"}); + regionPriosExp = QStringList({"eu", "us"}); + matchRegion("Game X (USA, xyz).zip", regionPriosExp); + } + + void testDetectRegionsFromFilenameIssue242OptionFirst() { + // "br" surplus + settings.regionPrios = QStringList({"eu", "br", "us", "jp"}); + settings.regionFromFilename = "first"; + qDebug() << "Configured region prios:" << settings.regionPrios; + scraper = new AbstractScraper(&settings, NULL); + + QList regionPriosExp = QStringList({"jp", "us", "eu", "br"}); + matchRegion("Game A (Japan, USA).zip", regionPriosExp); + regionPriosExp = QStringList({"us", "jp", "eu", "br"}); + matchRegion("Game A' (us, jp).zip", regionPriosExp); + + regionPriosExp = QStringList({"us", "eu", "br", "jp"}); + matchRegion("Game B (USA, Europe).zip", regionPriosExp); + + regionPriosExp = QStringList({"us", "wor", "eu", "br", "jp"}); + matchRegion("Game C (USA, World, Europe).zip", regionPriosExp); } };