Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

warn when requesting decompression with multiple threads #4335

Merged
merged 2 commits into from
Mar 11, 2025
+32 −12
Merged

warn when requesting decompression with multiple threads #4335

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c583c2c
warn when requesting decompression with multiple threads
Cyan4973 Mar 11, 2025
19541b4
fix minor unused variable warning
Cyan4973 Mar 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 8 additions & 12 deletions programs/zstdcli.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -822,15 +822,6 @@ static unsigned init_nbWorkers(void) {
CLEAN_RETURN(1); \
} } }

#define NEXT_INT32(_vari32) { \
const char* __nb; \
NEXT_FIELD(__nb); \
_vari32 = (int)readU32FromChar(&__nb); \
if(*__nb != 0) { \
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
} \
}

#define NEXT_UINT32(_varu32) { \
const char* __nb; \
NEXT_FIELD(__nb); \
Expand Down Expand Up @@ -887,7 +878,8 @@ int main(int argCount, const char* argv[])
removeSrcFile = 0,
cLevel = init_cLevel(),
ultra = 0,
cLevelLast = MINCLEVEL - 1; /* for benchmark range */
cLevelLast = MINCLEVEL - 1, /* for benchmark range */
setThreads_non1 = 0;
unsigned nbWorkers = init_nbWorkers();
ZSTD_ParamSwitch_e mmapDict = ZSTD_ps_auto;
ZSTD_ParamSwitch_e useRowMatchFinder = ZSTD_ps_auto;
Expand Down Expand Up @@ -1090,7 +1082,7 @@ int main(int argCount, const char* argv[])
continue;
}
#endif
if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); continue; }
if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); setThreads_non1 = (nbWorkers != 1); continue; }
if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; }
Expand Down Expand Up @@ -1297,6 +1289,7 @@ int main(int argCount, const char* argv[])
case 'T':
argument++;
nbWorkers = readU32FromChar(&argument);
setThreads_non1 = (nbWorkers != 1);
break;

/* Dictionary Selection level */
Expand Down Expand Up @@ -1341,6 +1334,9 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(3, WELCOME_MESSAGE);

#ifdef ZSTD_MULTITHREAD
if ((operation==zom_decompress) && (setThreads_non1)) {
DISPLAYLEVEL(2, "Warning : decompression does not support multi-threading\n");
}
if ((nbWorkers==NBWORKERS_AUTOCPU) && (!singleThread)) {
/* automatically set # workers based on # of reported cpu cores */
if (defaultLogicalCores) {
Expand All @@ -1354,7 +1350,7 @@ int main(int argCount, const char* argv[])
if (operation == zom_compress)
DISPLAYLEVEL(4, "Compressing with %u worker threads \n", nbWorkers);
#else
(void)singleThread; (void)nbWorkers; (void)defaultLogicalCores;
(void)singleThread; (void)nbWorkers; (void)defaultLogicalCores; (void)setThreads_non1;
#endif

g_utilDisplayLevel = g_displayLevel;
Expand Down
10 changes: 10 additions & 0 deletions tests/cli-tests/compression/multi-threaded.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,13 @@ zstd -T0 -f file -q ; zstd -t file.zst
zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst
zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst
zstd -T0 --jobsize=1M -f file -q ; zstd -t file.zst

# multi-thread decompression warning test
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T2 -d file.zst -o file4
# setting multi-thread via environment variable does not trigger decompression warning
zstd -T0 -f file -q ; zstd -t file.zst; ZSTD_NBTHREADS=0 zstd -df file.zst -o file3
zstd -T0 -f file -q ; zstd -t file.zst; ZSTD_NBTHREADS=2 zstd -df file.zst -o file4
# setting nbThreads==1 does not trigger decompression warning
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T1 -df file.zst -o file3
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T2 -T1 -df file.zst -o file4
14 changes: 14 additions & 0 deletions tests/cli-tests/compression/multi-threaded.sh.stderr.exact
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,17 @@ file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
Warning : decompression does not support multi-threading
file.zst : 65537 bytes
file.zst : 65537 bytes
Warning : decompression does not support multi-threading
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes