debug FreeBSD

more traces to understand the issue

Author: Cyan4973

lib/zstd.h

@@ -111,7 +111,7 @@ extern "C" {
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    5
-#define ZSTD_VERSION_RELEASE  7
+#define ZSTD_VERSION_RELEASE  8
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 
 /*! ZSTD_versionNumber() :

programs/zstdcli.c

@@ -686,6 +686,12 @@ static void printVersion(void)
             DISPLAYOUT("lz4 version %s\n", FIO_lz4Version());
             DISPLAYOUT("lzma version %s\n", FIO_lzmaVersion());
 
+        #ifdef ZSTD_MULTITHREAD
+            DISPLAYOUT("supports Multithreading \n");
+        #else
+            DISPLAYOUT("single-thread operations only \n");
+        #endif
+
             /* posix support */
         #ifdef _POSIX_C_SOURCE
             DISPLAYOUT("_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
@@ -779,6 +785,11 @@ static int init_cLevel(void) {
 static unsigned init_nbWorkers(void) {
 #ifdef ZSTD_MULTITHREAD
     const char* const env = getenv(ENV_NBWORKERS);
+    if (env==NULL) {
+        DISPLAYLEVEL(1, "Failed Reading environment variable %s \n", ENV_NBWORKERS);
+    } else {
+        DISPLAYLEVEL(1, "Reading environment variable %s : %s \n", ENV_NBWORKERS, env);
+    }
     if (env != NULL) {
         const char* ptr = env;
         if ((*ptr>='0') && (*ptr<='9')) {
@@ -793,6 +804,7 @@ static unsigned init_nbWorkers(void) {
         DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBWORKERS, env);
     }
 
+    DISPLAYLEVEL(1, "returning default nbThreads = %i \n", ZSTDCLI_NBTHREADS_DEFAULT);
     return ZSTDCLI_NBTHREADS_DEFAULT;
 #else
     return 1;
@@ -1336,7 +1348,7 @@ int main(int argCount, const char* argv[])
 
 #ifdef ZSTD_MULTITHREAD
     if ((nbWorkers==NBWORKERS_AUTOCPU) && (!singleThread)) {
-        /* automatically set # workers based on # of reported cpus */
+        /* automatically set # workers based on # of reported cpu cores */
         if (defaultLogicalCores) {
             nbWorkers = (unsigned)UTIL_countLogicalCores();
             DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers);
@@ -1345,8 +1357,7 @@ int main(int argCount, const char* argv[])
             DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
         }
     }
-    if (operation != zom_bench)
-        DISPLAYLEVEL(4, "Compressing with %u worker threads \n", nbWorkers);
+    DISPLAYLEVEL(1, "Compressing with %u worker threads \n", nbWorkers);
 #else
     (void)singleThread; (void)nbWorkers; (void)defaultLogicalCores;
 #endif

tests/playTests.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 set -e # exit immediately on error
-# set -x # print commands before execution (debug)
+set -x # print commands before execution (debug)
 
 unset ZSTD_CLEVEL
 unset ZSTD_NBTHREADS
@@ -1563,17 +1563,40 @@ then
     println "\n===>  zstdmt environment variable tests "
     echo "multifoo" >> mt_tmp
     ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting
-    ZSTD_NBTHREADS=''  zstd -f mt_tmp # empty env var, warn and revert to default setting
-    ZSTD_NBTHREADS=-   zstd -f mt_tmp # malformed env var, warn and revert to default setting
-    ZSTD_NBTHREADS=a   zstd -f mt_tmp # malformed env var, warn and revert to default setting
-    ZSTD_NBTHREADS=+a  zstd -f mt_tmp # malformed env var, warn and revert to default setting
+    ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting
+    ZSTD_NBTHREADS=-  zstd -f mt_tmp # malformed env var, warn and revert to default setting
+    ZSTD_NBTHREADS=a  zstd -f mt_tmp # malformed env var, warn and revert to default setting
+    ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting
     ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting
     ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting=
     ZSTD_NBTHREADS=2  zstd -f mt_tmp # correct usage
     ZSTD_NBTHREADS=1  zstd -f mt_tmp # correct usage: single worker
-    ZSTD_NBTHREADS=4 zstd -f mt_tmp -vv 2>&1 | grep "4 worker threads" # check message
+
+
+    # start of problematic test on FreeBSD; developping for testing and traces
+    zstd -vvvv --version
+
+    echo $0 # try to get shell name (note: not guaranteed)
+
+    zstd -T4 -f mt_tmp -vv # for comparison
+
+    ZSTD_NBTHREADS=4 zstd -f mt_tmp -vv
+
+    env ZSTD_NBTHREADS=4 zstd -f mt_tmp -vv
+
+    ZSTD_NBTHREADS=4 zstd -f mt_tmp -vv > mt_tmp_nbt4 2>&1
+
+    ls -las mt_tmp_nbt4
+
+    cat mt_tmp_nbt4
+
+    cat mt_tmp_nbt4 | $GREP "worker"
+
+    # The test below typically fails
+    ZSTD_NBTHREADS=4 zstd -f mt_tmp -vv 2>&1 | $GREP "4 worker threads" # check message
+
     zstd -tq mt_tmp.zst
-    ZSTD_NBTHREADS=0 zstd -f mt_tmp -vv 2>&1 | grep "core(s) detected" # check core count autodetection is triggered
+    ZSTD_NBTHREADS=0 zstd -f mt_tmp -vv 2>&1 | $GREP "core(s) detected" # check core count autodetection is triggered
     zstd -tq mt_tmp.zst
     # temporary envvar changes in the above tests would actually persist in macos /bin/sh
     unset ZSTD_NBTHREADS