check if tsan or asan log is not empty in functional tests

To help find the problems at CI and in general case not to mute them

Signed-off-by: roman khapov <[email protected]>

Author: rkhapov

docker/functional/bin/ody-stop

@@ -5,24 +5,51 @@ import time
 import signal
 import subprocess
 
+ASAN_LOG_PREFIX = '/asan-output.log'
+TSAN_LOG_PREFIX = '/tsan-output.log'
+
+
+def check_sanitizer_log_is_valid(pid, prefix):
+    filename = f'{prefix}.{pid}'
+    if not os.path.exists(filename) or os.path.getsize(filename) == 0:
+        return True
+
+    with open(filename, 'r') as asan_log:
+        print(f'Found non-empty {filename} log:')
+        print(asan_log.read())
+        return False
+
+
+def asan_log_is_valid(pid):
+    # ASAN_OPITONS was set to this log_prefix in docker-compose.yml
+    return check_sanitizer_log_is_valid(pid, ASAN_LOG_PREFIX)
+
+
+def tsan_log_is_valid(pid):
+    # TSAN_OPITONS was set to this log_prefix in docker-compose.yml
+    return check_sanitizer_log_is_valid(pid, TSAN_LOG_PREFIX)
+
 
 def get_odyssey_pids():
-    print(subprocess.check_output('ps aux | grep odyssey', shell=True).decode('utf-8'))
+    print(subprocess.check_output(
+        'ps aux | grep odyssey', shell=True).decode('utf-8'))
 
     try:
         return list(map(int, subprocess.check_output(['pgrep', 'odyssey']).split()))
     except subprocess.CalledProcessError:
         # non-zero exit code means that there is no odyssey pids
         return []
 
+
 def terminate_gracefully(pid, timeout):
     try:
         os.kill(pid, signal.SIGTERM)
     except ProcessLookupError:
         print(f'Process {pid} already finished or doesnt ever existed')
         return
 
-    print(f'Waiting {timeout} seconds to {pid} finish after SIGTERM...', end='')
+    print(f'Waiting {timeout} seconds to {
+          pid} finish after SIGTERM...', end='')
 
     start = time.time()
     finished = False
@@ -56,10 +83,15 @@ def main():
     for pid in pids:
         if not terminate_gracefully(pid, timeout):
             print(f'Process {pid} didnt finish within {timeout} seconds')
-            print(subprocess.check_output(['gdb', '-p', str(pid), '--batch', '-ex', 't a a bt', '-ex', 'source /gdb.py', 'mmcoro all bt']))
+            print(subprocess.check_output(
+                ['gdb', '-p', str(pid), '--batch', '-ex', 't a a bt', '-ex', 'source /gdb.py', 'mmcoro all bt']))
+            exit(1)
+
+        if not asan_log_is_valid(pid) or not tsan_log_is_valid(pid):
             exit(1)
 
     exit(0)
 
+
 if __name__ == "__main__":
     main()

docker/functional/docker-compose.yml

@@ -17,6 +17,8 @@ services:
     command: ["-k", "${ODYSSEY_FUNCTIONAL_TESTS_SELECTOR:-}"]
     environment:
       CMAKE_BUILD_TYPE: "${CMAKE_BUILD_TYPE:-Debug}"
+      ASAN_OPTIONS: "log_path=/asan-output.log fast_unwind_on_malloc=0"
+      TSAN_OPTIONS: "log_path=/tsan-output.log"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     ports: