Vdb 6027

CHANGES.md

@@ -1,6 +1,35 @@
+----
+                          PUBLIC DOMAIN NOTICE
+             National Center for Biotechnology Information
+
+This software/database is a "United States Government Work" under the
+terms of the United States Copyright Act.  It was written as part of
+the author's official duties as a United States Government employee and
+thus cannot be copyrighted.  This software/database is freely available
+to the public for use. The National Library of Medicine and the U.S.
+Government have not placed any restriction on its use or reproduction.
+
+Although all reasonable efforts have been taken to ensure the accuracy
+and reliability of the software and data, the NLM and the U.S.
+Government do not and cannot warrant the performance or results that
+may be obtained by using this software or data. The NLM and the U.S.
+Government disclaim all warranties, express or implied, including
+warranties of performance, merchantability or fitness for any particular
+purpose.
+
+Please cite the author in any work or product based on this material.
+
+----
 # NCBI External Developer Release:
 
 
+## NCBI VDB 3.2.1
+**March 18, 2025**
+
+  **fasterq-dump, kns, sratools, vdb, vfs**: fixed failure of fasterq-dump when accessing run via HTTP  
+  **kdb**: fixed memory leak  
+
+
 ## NCBI VDB 3.2.0
 **January 14, 2025**
 

build/.gitignore

@@ -1,4 +1,4 @@
 ld.linux.exe_cmd.sh
 Makefile.config*
 COMP.*
-
+a.out

build/Makefile.env

@@ -82,6 +82,8 @@ endif
 
 unexport TOP SRCDIR
 
+export OS_DISTRIBUTOR
+
 # CPP comes from Makefile.config.$(OS_ARCH)
 # TODO: simplify Makefile.config.$(OS_ARCH) generated by ./configure - switch from using LD to a new variable
 CMAKEFLAGS = \

build/Makefile.vers

@@ -23,4 +23,4 @@
 # ===========================================================================
 
 # NCBI-VDB and library version
-VERSION = 3.2.1
+VERSION = 3.2.2

build/common.cmake

@@ -39,6 +39,13 @@ function( MSVS_DLLRuntime name )
     endif()
 endfunction()
 
+#
+# sanitizer settings
+#   address sanitizer: stop and fail the test if undefined behavior is detected
+set( asan_defs "-fsanitize=address" "-fsanitize=undefined" "-fno-sanitize-recover=undefined" )
+set( tsan_defs "-fsanitize=thread" )
+#
+
 function( GenerateStaticLibsWithDefs target_name sources compile_defs )
     add_library( ${target_name} STATIC ${sources} )
     if( NOT "" STREQUAL "${compile_defs}" )
@@ -54,15 +61,13 @@ function( GenerateStaticLibsWithDefs target_name sources compile_defs )
     endif()
 
     if( RUN_SANITIZER_TESTS )
-        set( asan_defs "-fsanitize=address" "-fsanitize=undefined" )
         add_library( "${target_name}-asan" STATIC ${sources} )
         if( NOT "" STREQUAL "${compile_defs}" )
             target_compile_definitions( "${target_name}-asan" PRIVATE ${compile_defs} )
         endif()
         target_compile_options( "${target_name}-asan" PUBLIC ${asan_defs} )
         target_link_options( "${target_name}-asan" PUBLIC ${asan_defs} )
 
-        set( tsan_defs "-fsanitize=thread" "-fsanitize=undefined" )
         add_library( "${target_name}-tsan" STATIC ${sources} )
         if( NOT "" STREQUAL "${compile_defs}" )
             target_compile_definitions( "${target_name}-tsan" PRIVATE ${compile_defs} )
@@ -198,9 +203,13 @@ endfunction()
 
 function( BuildExecutableForTest exe_name sources libraries )
 	add_executable( ${exe_name} ${sources} )
-        if(NOT _NCBIVDB_CFG_PACKAGING)
+    if( WIN32 )
+        target_link_options( ${exe_name} PRIVATE "/ENTRY:mainCRTStartup" )
+    endif()
+
+    if(NOT _NCBIVDB_CFG_PACKAGING)
 	    MSVS_StaticRuntime( ${exe_name} )
-        endif()
+    endif()
 	target_link_libraries( ${exe_name} ${libraries} )
 endfunction()
 
@@ -209,13 +218,11 @@ function( AddExecutableTest test_name sources libraries )
 	add_test( NAME ${test_name} COMMAND ${test_name} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} )
 
 	if( RUN_SANITIZER_TESTS )
-		set( asan_defs "-fsanitize=address" "-fsanitize=undefined" )
 		BuildExecutableForTest( "${test_name}_asan" "${sources}" "${libraries}" )
 		target_compile_options( "${test_name}_asan" PRIVATE ${asan_defs} )
 		target_link_options( "${test_name}_asan" PRIVATE ${asan_defs} )
 		add_test( NAME "${test_name}_asan" COMMAND "${test_name}_asan" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} )
 
-		set( tsan_defs "-fsanitize=thread" "-fsanitize=undefined" )
 		BuildExecutableForTest( "${test_name}_tsan" "${sources}" "${libraries}" )
 		target_compile_options( "${test_name}_tsan" PRIVATE ${tsan_defs} )
 		target_link_options( "${test_name}_tsan" PRIVATE ${tsan_defs} )

build/env.cmake

@@ -389,31 +389,51 @@ if ( SINGLE_CONFIG )
     )
 endif()
 
-if( NOT SINGLE_CONFIG )
-	if( RUN_SANITIZER_TESTS )
-		message( "RUN_SANITIZER_TESTS (${RUN_SANITIZER_TESTS}) cannot be turned on in a non single config mode - overriding to OFF" )
-	endif()
-	set( RUN_SANITIZER_TESTS OFF )
-endif()
+execute_process( COMMAND sh -c "${CMAKE_CXX_COMPILER} -fsanitize=address test.cpp && ./a.out"
+    RESULT_VARIABLE ASAN_WORKS
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/build  )
+execute_process( COMMAND sh -c "${CMAKE_CXX_COMPILER} -fsanitize=thread test.cpp && ./a.out"
+    RESULT_VARIABLE TSAN_WORKS
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/build  )
+
+if( ASAN_WORKS EQUAL 0 AND TSAN_WORKS EQUAL 0 )
+
+    if( NOT SINGLE_CONFIG AND RUN_SANITIZER_TESTS )
+        message( "RUN_SANITIZER_TESTS (${RUN_SANITIZER_TESTS}) cannot be turned on in a non single config mode - overriding to OFF" )
+        set( RUN_SANITIZER_TESTS OFF )
+    endif()
 
-if( RUN_SANITIZER_TESTS )
-	find_program(LSB_RELEASE_EXEC lsb_release)
-	execute_process(COMMAND ${LSB_RELEASE_EXEC} -is
-		OUTPUT_VARIABLE LSB_RELEASE_ID_SHORT
-		OUTPUT_STRIP_TRAILING_WHITESPACE
-	)
-	message("LSB_RELEASE_ID_SHORT: ${LSB_RELEASE_ID_SHORT}")
-	if( LSB_RELEASE_ID_SHORT STREQUAL "Ubuntu" )
-		message("Disabling sanitizer tests on Ubuntu...")
-		set( RUN_SANITIZER_TESTS OFF )
-	endif()
-endif()
+    if( RUN_SANITIZER_TESTS_OVERRIDE )
+        message("Overriding sanitizer tests due to RUN_SANITIZER_TESTS_OVERRIDE: ${RUN_SANITIZER_TESTS_OVERRIDE}")
+        set( RUN_SANITIZER_TESTS ON )
+    endif()
+
+    #
+    # TSAN-instrumented programs used to crash on starup with certain version of Ubuntu kernel. Seems to be not thecase anymore, so disabling this section.
+    #
+    # if( RUN_SANITIZER_TESTS )
+    #     find_program(LSB_RELEASE_EXEC lsb_release)
+    #     execute_process(COMMAND ${LSB_RELEASE_EXEC} -is
+    #         OUTPUT_VARIABLE LSB_RELEASE_ID_SHORT
+    #         OUTPUT_STRIP_TRAILING_WHITESPACE
+    #     )
+    #     message("LSB_RELEASE_ID_SHORT: ${LSB_RELEASE_ID_SHORT}")
+    #     if( LSB_RELEASE_ID_SHORT STREQUAL "Ubuntu" )
+    #         message("Disabling sanitizer tests on Ubuntu...")
+    #         set( RUN_SANITIZER_TESTS OFF )
+    #     endif()
+    # endif()
+
+else()
+
+    message("ASAN suport is not detected. Disabling sanitizer tests.")
+    set( RUN_SANITIZER_TESTS OFF )
+    set( RUN_SANITIZER_TESTS_OVERRIDE OFF )
 
-if( RUN_SANITIZER_TESTS_OVERRIDE )
-	message("Overriding sanitizer tests due to RUN_SANITIZER_TESTS_OVERRIDE: ${RUN_SANITIZER_TESTS_OVERRIDE}")
-	set( RUN_SANITIZER_TESTS ON )
 endif()
 message( "RUN_SANITIZER_TESTS: ${RUN_SANITIZER_TESTS}" )
+
+
 endif(NOT _NCBIVDB_CFG_PACKAGING)
 
 if( NOT HAVE_MBEDTLS_F )

build/san_test.cpp

@@ -0,0 +1 @@
+int main(){}

build/test.cpp

@@ -0,0 +1,6 @@
+#include <cstdlib>
+int main()
+{ 
+    //void* p = malloc(1); 
+    return 0;
+}

interfaces/cc/clang/arm64/atomic.h → interfaces/cc/clang/atomic.h

@@ -42,11 +42,9 @@ extern "C" {
 #if DFLT_ATOMIC_BITS == 32
 #define ATOMIC_NAME( suffix ) \
     atomic32_ ## suffix
-typedef int atomic_int;
 #else
 #define ATOMIC_NAME( suffix ) \
     atomic64_ ## suffix
-typedef long int atomic_int;
 #endif
 
 typedef struct ATOMIC_NAME ( t ) atomic_t;
@@ -108,14 +106,16 @@ struct atomic_ptr_t
 
 /* int atomic_read_ptr ( const atomic_ptr_t *v ); */
 #define atomic_read_ptr( v ) \
-    ( ( v ) -> ptr )
+    __atomic_load_n( & ( ( v ) -> ptr ), __ATOMIC_SEQ_CST )
+
+#define atomic_set_ptr( v, s ) \
+    __atomic_store_n( & ( ( v ) -> ptr ), s, __ATOMIC_SEQ_CST )
 
 static __inline__
-void *atomic_test_and_set_ptr ( atomic_ptr_t *v, void *s, void *t )
+void *atomic_test_and_set_ptr ( atomic_ptr_t *const v, void *const s, void *const t )
 {
-    void *rtn = v -> ptr;
-    if ( rtn == t )
-        v -> ptr = s;
+    void *rtn = t;
+    __atomic_compare_exchange_n(&v->ptr, &rtn, s, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
     return rtn;
 }
 
@@ -178,6 +178,7 @@ void *atomic_test_and_set_ptr ( atomic_ptr_t *v, void *s, void *t )
 
 #ifdef __cplusplus
 }
+
 #endif
 
 #endif /* _h_atomic_ */

interfaces/cc/clang/arm64/atomic32.h → interfaces/cc/clang/atomic32.h

@@ -39,64 +39,64 @@ extern "C" {
 typedef struct atomic32_t atomic32_t;
 struct atomic32_t
 {
-    volatile int counter;
+    int volatile counter;
 };
 
 /* int atomic32_read ( const atomic32_t *v ); */
 #define atomic32_read( v ) \
-    ( ( v ) -> counter )
+    __atomic_load_n(&((v)->counter), __ATOMIC_SEQ_CST)
 
 /* void atomic32_set ( atomic32_t *v, int i ); */
 #define atomic32_set( v, i ) \
-    ( ( void ) ( ( ( v ) -> counter ) = ( i ) ) )
+    __atomic_store_n(&((v)->counter), i, __ATOMIC_SEQ_CST)
 
 /* add to v -> counter and return the prior value */
-static __inline__ int atomic32_read_and_add ( atomic32_t *v, int i )
-{
-    return __sync_fetch_and_add( & v -> counter, i );
-}
+#define atomic32_read_and_add( v, i ) \
+    __atomic_fetch_add(&((v)->counter), i, __ATOMIC_SEQ_CST)
 
 /* if no read is needed, define the least expensive atomic add */
 #define atomic32_add( v, i ) \
-    atomic32_read_and_add ( v, i )
+    ((void)atomic32_read_and_add(v, i))
 
 /* add to v -> counter and return the result */
-static __inline__ int atomic32_add_and_read ( atomic32_t *v, int i )
+static __inline__ int atomic32_add_and_read ( atomic32_t *const v, int const i )
 {
-    return __sync_add_and_fetch( & v -> counter, i );
+    return __atomic_add_fetch(&((v)->counter), i, __ATOMIC_SEQ_CST);
 }
 
 /* just don't try to find out what the result was */
-static __inline__ void atomic32_inc ( atomic32_t *v )
+static __inline__ void atomic32_inc ( atomic32_t *const v )
 {
     atomic32_add( v, 1 );
 }
 
-static __inline__ void atomic32_dec ( atomic32_t *v )
+static __inline__ void atomic32_dec ( atomic32_t *const v )
 {
-    atomic32_add( v, -1 );
+    __atomic_fetch_sub(&((v)->counter), 1, __ATOMIC_SEQ_CST);
 }
 
 /* decrement by one and test result for 0 */
-static __inline__ int atomic32_dec_and_test ( atomic32_t *v )
+static __inline__ int atomic32_dec_and_test ( atomic32_t *const v )
 {
-    return __sync_add_and_fetch( & v -> counter, -1 ) == 0;
+    return __atomic_sub_fetch(&((v)->counter), 1, __ATOMIC_SEQ_CST) == 0;
 }
 
 /* when atomic32_dec_and_test uses predecrement, you want
    postincrement to this function. so it isn't very useful */
-static __inline__ int atomic32_inc_and_test ( atomic32_t *v )
+static __inline__ int atomic32_inc_and_test ( atomic32_t *const v )
 {
-    return __sync_add_and_fetch( & v -> counter, 1 ) == 0;
+    return atomic32_add_and_read(v, 1) == 0;
 }
 
 /* HERE's useful */
 #define atomic32_test_and_inc( v ) \
     ( atomic32_read_and_add ( v, 1 ) == 0 )
 
-static __inline__ int atomic32_test_and_set ( atomic32_t *v, int newval, int oldval )
+static __inline__ int atomic32_test_and_set ( atomic32_t *const v, int const newval, int const oldval )
 {
-    return __sync_val_compare_and_swap( & v -> counter, oldval, newval ); //NB: newval/oldval switched around
+    int expected = oldval;
+    __atomic_compare_exchange_n(&((v)->counter), &expected, newval, 1, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+    return expected;
 }
 
 /* conditional modifications */