Release 2.3

Author: theypsilon

CHANGELOG.md

@@ -4,6 +4,23 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
+## Version 2.3 - 2025-11-21
+
+### Added
+- New `file_checking` configuration option, that will tell Downloader whether to verify the presence of installed files or not. By default it will take the value `balanced`, which is the recommended option. The option `balanced` is a new behavior that skips file presence checking when the free space in the storages (like the SD) haven't been increased, making the updating process much quicker as result. With option `exhaustive` you get the old behavior, it will always check file presence, which is useful for advanced users that usually manipulate installed files. There are more options documented in the [README.md file](README.md#options).
+
+### Changed
+- Reworked NTP logic in launcher. It now adds a few extra NTP servers that are helpful to avoid connectivity issues in users from China.
+- Adjusted timeouts in HTTP gateway to be more responsive when a HTTP Proxy is not working and in other scenarios.
+- Fixed locale errors in native builds.
+- Fixed launcher for headless runs.
+- Default `downloader_threads_limit` value changed to 6.
+- Documentation now reflects the latest architectural changes introduced with the `file_checking` feature.
+- Improved logs during connectivity issues.
+
+### Removed
+- The legacy *"force check"* flag triggered by removing the file `Scripts/.config/downloader/downloader.last_successful_run` is now **DEPRECATED** and will be removed in **May 2026**. To get the same behavior (verifying all file hashes), use `file_checking` with the option `verify_integrity`.
+
 ## Version 2.2.1 - 2025-10-30
 
 ### Added

README.md

@@ -16,6 +16,17 @@ Here you can see the default parameters and the options that you may change:
 
 ```ini
 [MiSTer]
+; verbose: when true, will make Downloader output to display additional debug information
+;   This is also necessary to be active to display benchmark information.
+verbose = false
+
+; file_checking options:
+;   'fastest' -> Only check file presences when there are database changes. Fastest.
+;   'balanced' -> Check file presences when there are database changes or the free space in the storages have increased noticeably. Recommended for most users.
+;   'exhaustive' -> Always check file presences. Option for users who remove/add cores manually.
+;   'verify_integrity' -> Always check file presences and hashes (much slower). Useful to verify that installed files are not corrupted (ExFAT partitions can become corrupt in rare cases).
+file_checking = 'balanced'
+
 ; storage_priority defines how this tool will use external storage during the installation
 ;         of files that are designed for external locations (usually games & docs files).
 ;         
@@ -35,47 +46,45 @@ Here you can see the default parameters and the options that you may change:
 ;   'off' -> Disables this feature. Affected files will always be installed in your SD.
 storage_priority = 'prefer_sd'
 
-; update_linux options:
-;   true -> Updates Linux when there is a new update (very recommended).
-;   false -> Doesn't update Linux.
-update_linux = true
-
-; allow_reboot options:
-;   0 -> Don't allow this tool to ever reboot automatically.
-;   1 -> Allow this tool to reboot the system after any system file has been updated.
-;   2 -> Allow this tool to reboot the system only after Linux has been updated.
-allow_reboot = 1
-
-; allow_delete options:
-;   0 -> Don't allow this tool to delete anything at all.
-;   1 -> Allow this tool to delete any old file from previous updates.
-;   2 -> Allow this tool to delete only old cores that receive a new version.
-allow_delete = 1
-
 ; minimum_system_free_space_mb: Fee space needed to install files with Downloader
 ;   This minimum applies solely to the system partition (SD card).
-;   Reducing this value is not advised.
+;   Reducing this value is highly discouraged.
 minimum_system_free_space_mb = 512
 
 ; minimum_external_free_space_mb: Free space needed to install files in external storages
 ;   This minimum applies to all external storages (USBs & CIFS).
-;   Reducing this value is not advised.
+;   Reducing this value is highly discouraged.
 minimum_external_free_space_mb = 128
 
 ; downloader_timeout: Can be tweaked to increase the timeout time in seconds
 ;   It is useful to increase this value for users with slow connections.
+;   Reducing this value is highly discouraged.
 downloader_timeout = 300
 
 ; downloader_retries: Can be tweaked to increase the retries per failed download
 ;   It is useful to increase this value for users with very unstable connections.
+;   Reducing this value is highly discouraged.
 downloader_retries = 3
 
-; verbose: when true, will make Downloader output to display additional debug information
-;   This is also necessary to be active to display benchmark information.
-verbose = false
+; update_linux options:
+;   true -> Updates Linux when there is a new update.
+;   false -> Doesn't update Linux (highly discouraged).
+update_linux = true
+
+; allow_reboot options:
+;   0 -> Don't allow this tool to ever reboot automatically (highly discouraged).
+;   1 -> Allow this tool to reboot the system after any system file has been updated.
+;   2 -> Allow this tool to reboot the system only after Linux has been updated (highly discouraged).
+allow_reboot = 1
+
+; allow_delete options:
+;   0 -> Don't allow this tool to delete anything at all (highly discouraged).
+;   1 -> Allow this tool to delete any old file from previous updates.
+;   2 -> Allow this tool to delete only old cores that receive a new version (highly discouraged).
+allow_delete = 1
 
 ; http_proxy: Routes all downloads through a proxy server (advanced, rarely needed)
-;   Format: 'http://proxy-server:port'
+;   Format: 'http://proxy-server:port' (it also supports basic auth)
 http_proxy = ''
 ```
 
@@ -87,7 +96,7 @@ http_proxy = ''
 - [x] Configurable custom download filters
 - [x] Storage Priority Resolution for automatically detecting connected drives
 - [x] Free space checks
-- [ ] Opt-in parameter to bypass strict file checks
+- [x] Opt-in parameter to bypass strict file checks
 - [ ] Fast check for update availability (without triggering an actual update)
 - [ ] HTTP cookie support
 - [ ] Uninstall database feature

docs/custom-databases.md

@@ -12,6 +12,13 @@ The `db_url` entry points to a JSON file that a maintainer has published. The `*
 The format of the aforementioned JSON file should be as follow:
 ```js
 {
+    /**
+     * [Mandatory] Version of the database format (int)
+     * - Must be set to 1 to match the spec in this document.
+     * - Versions >1 are rejected; the downloader will prompt a self-update.
+     * - Omitted defaults to version 0, which still works but is slower.
+     */
+    "v": 1,
     /**
      * [Mandatory] It should match the corresponding section_id defined in the INI file (string)
      *  Important: This ID must identify unequivocally your collection in the whole MiSTer ecosystem.

docs/jobs_diagram.png

@@ -1,5 +1,7 @@
 from graphviz import Digraph
 
+BACKUP_COLOR = '#888888'
+
 # Initialize the Digraph
 dot = Digraph(
     comment='Workflow Diagram',
@@ -10,19 +12,23 @@
         'fillcolor': 'lightgrey',
         'fontname': 'Helvetica'
     },
-    edge_attr={'color': 'black'}
+    edge_attr={'color': 'black', 'fontname': 'Helvetica-Narrow', 'fontsize': '10'}
 )
 
 # NON-ZIP NODES
 dot.node('START', 'START', shape='ellipse', fillcolor='black', fontcolor='white')
 dot.node('END', 'END', shape='ellipse', fillcolor='#77DD77')
+dot.node('SKIP1', 'SKIP', shape='ellipse', fillcolor='#77DD77')
+dot.node('SKIP2', 'SKIP', shape='ellipse', fillcolor='#77DD77')
 dot.node('ABORT', 'ABORT', shape='ellipse', fillcolor='#ff8f8f')
 dot.node('A', 'FetchDataJob [db]')
-dot.node('B', 'LoadLocalStoreJob', fillcolor='#ffefd5')
-dot.node('C', 'OpenDbJob', fillcolor='#B0E0E6')
-dot.node('D', 'ProcessDbMainJob', fillcolor='#B0E0E6')
-dot.node('E', 'ProcessDbIndexJob', fillcolor='#B0E0E6:#ffefd5')
-dot.node('M', 'FetchFileJob [file]')
+dot.node('B', 'LoadLocalStoreSigsJob', fillcolor='#ffefd5')
+dot.node('C', 'LoadLocalStoreJob', fillcolor='#ffefd5')
+dot.node('D', 'OpenDbJob', fillcolor='#B0E0E6')
+dot.node('E', 'MixStoreAndDbJob', fillcolor='#B0E0E6')
+dot.node('F', 'ProcessDbMainJob', fillcolor='#B0E0E6')
+dot.node('G', 'ProcessDbIndexJob', fillcolor='#B0E0E6:#ffefd5')
+dot.node('H', 'FetchFileJob [file]')
 
 # Example "legend" nodes
 dot.node('0', 'CPU', fillcolor='#B0E0E6')
@@ -32,21 +38,46 @@
 dot.edge('START', 'A', label='1:N')
 dot.edge('START', 'B')
 
-dot.edge('A', 'C', weight='10')
-dot.edge('C', 'D', weight='10', label=' db + store')
-dot.edge('D', 'E', weight='10', label=' if no zips')
+dot.edge('A', 'D', weight='10')
+dot.edge('D', 'C', weight='5', constraint='false', label=' N:1, if any db sig != store sig')
+dot.edge('D', 'E', weight='10', label=' db')
+dot.edge('E', 'F', weight='10', label=' db + store')
+dot.edge('F', 'G', weight='10', label=' if no zips')
+
+dot.edge('G', 'H', label=' 1:N')
+
+dot.edge('C', 'E', style='dotted', constraint='true', label='wait\nstore', dir='back')
+dot.edge('B', 'D', style='dotted', constraint='true', label='wait\nstore sig', dir='back')
+
+# Position SKIP1 at same rank as OpenDbJob (D)
+with dot.subgraph() as s:
+    s.attr(rank='same')
+    s.node('SKIP1')
+    s.node('D')
+    s.node('C')
 
-dot.edge('E', 'M', label=' 1:N')
+dot.edge('SKIP1', 'D', style='invis')
+dot.edge('D', 'C', style='invis')
 
-dot.edge('B', 'C', style='dotted', constraint='true', label='wait\nstore', dir='back')
+# Position E, SKIP2, and ABORT at same rank (left to right)
+with dot.subgraph() as s:
+    s.attr(rank='same')
+    s.node('E')
+    s.node('SKIP2')
+    s.node('ABORT')
 
-dot.edge('B', 'B', label=' retry', style='dashed', constraint='false')
+dot.edge('E', 'SKIP2', style='invis')
+dot.edge('SKIP2', 'ABORT', style='invis')
+
+dot.edge('C', 'C', label=' retry', style='dashed', constraint='false')
 dot.edge('A', 'A', label=' retry', style='dashed', constraint='false')
-dot.edge('C', 'A', label=' retry', style='dashed', constraint='true')
-dot.edge('M', 'M', label=' retry', style='dashed', constraint='false')
+dot.edge('D', 'A', label=' retry', style='dashed', constraint='true')
+dot.edge('H', 'H', label=' retry', style='dashed', constraint='false')
 
-dot.edge('M', 'END', weight='20', constraint='true')
-dot.edge('B', 'ABORT', weight='20', constraint='true', label=' if always fails', style='dashed')
+dot.edge('H', 'END', weight='20', constraint='true')
+dot.edge('C', 'ABORT', weight='20', constraint='true', label=' if always fails', style='dashed', color=BACKUP_COLOR, fontcolor=BACKUP_COLOR)
+dot.edge('D', 'SKIP1', weight='1', constraint='false', label=' if db sig == store sig')
+dot.edge('E', 'SKIP2', weight='1', constraint='false', label=' if db sig == store sig')
 
 dot.render('jobs_diagram', format='png', cleanup=True)
 
@@ -55,51 +86,51 @@
     c.attr(style='filled', color='#f5f5ff', penwidth='1.0')
     c.node('3', 'Zip Feature', style='filled', fillcolor='#8f8fff', fontcolor='white', penwidth='0')
 
-    c.node('O', 'WaitDbZipsJob', fillcolor='#B0E0E6')
-    c.node('F', 'ProcessZipIndexJob', fillcolor='#B0E0E6:#ffefd5')
-    c.node('G', 'FetchDataJob [zip summary]')
-    c.node('I', 'OpenZipSummaryJob', fillcolor='#B0E0E6')
+    c.node('I', 'WaitDbZipsJob', fillcolor='#B0E0E6')
+    c.node('J', 'ProcessZipIndexJob', fillcolor='#B0E0E6:#ffefd5')
+    c.node('K', 'FetchDataJob [zip summary]')
+    c.node('L', 'OpenZipSummaryJob', fillcolor='#B0E0E6')
     c.node('DOT0', '', shape='circle', fixedsize='true', width='0.01', fillcolor='#ff8f8f')
-    c.node('J', 'FetchDataJob [zip contents]')
-    c.node('L', 'OpenZipContentsJob', fillcolor='#ffefd5')
+    c.node('M', 'FetchDataJob [zip contents]')
+    c.node('N', 'OpenZipContentsJob', fillcolor='#ffefd5')
     c.node('DOT1', '', shape='circle', fixedsize='true', width='0.01', fillcolor='#ff8f8f')
     c.node('DOT2', '', shape='circle', fixedsize='true', width='0.01', fillcolor='#ff8f8f')
 #
 # EDGES
 #
-dot.edge('D', 'O', weight='10', label='if zips')
-dot.edge('O', 'E', weight='10', label='store w/ deselected\nzip indexes')
-dot.edge('D', 'G', label='1:N (missing zips)')
-
-dot.edge('F', 'J', label='if many file installs')
-dot.edge('I', 'F')
-dot.edge('G', 'I')
-dot.edge('J', 'L')
-dot.edge('L', 'M', label='1:N\n(invalid files)')
-dot.edge('F', 'DOT1', dir='none')
+dot.edge('F', 'I', weight='10', label='if zips')
+dot.edge('I', 'G', weight='10', label='store w/ deselected\nzip indexes')
+dot.edge('F', 'K', label='1:N (missing zips)')
+
+dot.edge('J', 'M', label='if many file installs')
+dot.edge('L', 'J')
+dot.edge('K', 'L')
+dot.edge('M', 'N')
+dot.edge('N', 'H', label='1:N\n(invalid files)')
+dot.edge('J', 'DOT1', dir='none')
 dot.edge('DOT1', 'DOT2', label='if just few\n file installs', dir='none')
-dot.edge('DOT2', 'M', label=' 1:N')
-#dot.edge('L', 'M', label='1:N')
+dot.edge('DOT2', 'H', label=' 1:N')
+#dot.edge('N', 'H', label='1:N')
 
-# “Wait” edges
-dot.edge('F', 'O', style='dotted', constraint='true', label='wait\n zip indexes', dir='back')
+# "Wait" edges
+dot.edge('J', 'I', style='dotted', constraint='true', label='wait\n zip indexes', dir='back')
 
-# Labeled “1:N” edges
-dot.edge('D', 'F', label='1:N (stored zips)', weight='5')
+# Labeled "1:N" edges
+dot.edge('F', 'J', label='1:N (stored zips)', weight='5')
 
 # Edges to END
-dot.edge('L', 'END', weight='20', constraint='true')
+dot.edge('N', 'END', weight='20', constraint='true')
 
-# “Retry” edges
-dot.edge('G', 'G', label=' retry', style='dashed', constraint='false')
-dot.edge('I', 'G', label=' retry', style='dashed', constraint='false')
-dot.edge('J', 'J', label=' retry', style='dashed', constraint='false')
-dot.edge('L', 'J', label=' retry', style='dashed', constraint='false')
+# "Retry" edges
+dot.edge('K', 'K', label=' retry', style='dashed', constraint='false')
+dot.edge('L', 'K', label=' retry', style='dashed', constraint='false')
+dot.edge('M', 'M', label=' retry', style='dashed', constraint='false')
+dot.edge('N', 'M', label=' retry', style='dashed', constraint='false')
 
 # "Backup" edges
-dot.edge('G', 'DOT0', style='dashed', constraint='true', dir='none')
-dot.edge('I', 'DOT0', style='dashed', constraint='true', dir='none')
-dot.edge('DOT0', 'F', label='backup:\nstored summary', style='dashed', constraint='false')
+dot.edge('K', 'DOT0', style='dashed', constraint='true', dir='none', color=BACKUP_COLOR)
+dot.edge('L', 'DOT0', style='dashed', constraint='true', dir='none', color=BACKUP_COLOR)
+dot.edge('DOT0', 'J', label='backup:\nstored summary', style='dashed', constraint='false', color=BACKUP_COLOR, fontcolor=BACKUP_COLOR)
 #dot.edge('H', 'I', label='b', style='dashed', constraint='false', dir='none')
 #dot.edge('I', 'F', label='b', style='dashed', constraint='false')
 #dot.edge('J', 'E', label='b', style='dashed', constraint='false')

docs/jobs_diagram_with_zip_feature.png

@@ -25,13 +25,41 @@ LATEST_BIN_PATH="/media/fat/Scripts/.config/downloader/downloader_bin"
 CACERT_PEM_0="/etc/ssl/certs/cacert.pem"
 CACERT_PEM_1="/media/fat/Scripts/.config/downloader/cacert.pem"
 
-if (( $(date +%Y) < 2000 )) ; then
-    NTP_SERVER="0.pool.ntp.org"
-    echo "Syncing date and time with $NTP_SERVER"
-    echo
-    if ntpdate -s -b -u $NTP_SERVER ; then
+# NTP SETUP
+if (( 10#$(date +%Y) < 2000 )) ; then
+    NTP_SERVERS=(
+        "time.apple.com"
+        "time.amazonaws.cn"
+        "ntp.ntsc.ac.cn"
+        "cn.pool.ntp.org"
+        "ntp.aliyun.com"
+        "ntp.tencent.com"
+        "ntp.rt.ru"
+    )
+    NTP_CONF="/etc/ntp.conf"
+    for server in "${NTP_SERVERS[@]}"; do
+        if ! grep -qF "${server}" "${NTP_CONF}"; then
+            echo "server $server iburst" >> "${NTP_CONF}"
+        fi
+    done
+    NTP_PID="/var/run/ntpd.pid"
+    start-stop-daemon -K -p "${NTP_PID}"
+    rm -f "${NTP_PID}"
+    start-stop-daemon -S -q -p "${NTP_PID}" -x "/usr/sbin/ntpd" -- -g -p "${NTP_PID}"
+    connected=0
+    for ((i=1; i<=10; i++)); do
+        if ntpq -c "rv 0" 2>&1 | grep -qiE "connection refused|sync_unspec" ; then
+            printf "."
+            sleep 3
+        else
+            connected=1
+            break
+        fi
+    done
+    printf "\n"
+    if (( connected )); then
         echo "Date and time is:"
-        echo "$(date)"
+        date
         echo
     elif [[ "${CURL_SSL:-}" != "--insecure" ]] ; then
         echo "Unable to sync."
@@ -40,6 +68,7 @@ if (( $(date +%Y) < 2000 )) ; then
     fi
 fi
 
+# CERTS SETUP
 if [ -s "${CACERT_PEM_1}" ] ; then
     export SSL_CERT_FILE="${CACERT_PEM_1}"
 elif [ -s "${CACERT_PEM_0}" ] ; then
@@ -101,6 +130,7 @@ elif [[ "${CURL_SSL:-}" != "--insecure" ]] ; then
     esac
 fi
 
+# LAUNCHER
 download_file() {
     local DOWNLOAD_PATH="${1}"
     local DOWNLOAD_URL="${2}"

docs/jobs_flow_diagram.py

@@ -21,6 +21,6 @@ set -euo pipefail
 
 your_http_proxy_url="http://your-proxy-server-url-goes-here.com"
 
-rm -f /media/fat/Scripts/.config/downloader/downloader_bin /media/fat/Scripts/.config/downloader/downloader_latest.zip
+rm -f /media/fat/Scripts/.config/downloader_bin /media/fat/Scripts/.config/downloader_latest.zip
 export http_proxy="${http_proxy:-${your_http_proxy_url}}"
 /media/fat/Scripts/downloader.sh