check if tsan or asan log is not empty in functional tests

docker/functional/bin/ody-stop

@@ -5,6 +5,39 @@ import time
 import signal
 import subprocess
 import sys
+import re
+
+ASAN_LOG_PREFIX = '/asan-output.log'
+TSAN_LOG_PREFIX = '/tsan-output.log'
+
+
+def get_sanitizer_log(pid, prefix):
+    filename = f'{prefix}.{pid}'
+    if not os.path.exists(filename) or os.path.getsize(filename) == 0:
+        return b''
+
+    with open(filename, 'rb') as logfile:
+        print(f'Found non-empty {filename} log:')
+        data = logfile.read()
+        print(data.decode('utf-8'))
+        return data
+
+
+def asan_log_is_valid(pid):
+    # ASAN_OPITONS was set to this log_prefix in docker-compose.yml
+    logdata = get_sanitizer_log(pid, ASAN_LOG_PREFIX)
+    warning_count = len(re.findall(rb'==\d+==WARNING', logdata))
+    error_count = len(re.findall(rb'==\d+==ERROR', logdata))
+
+    print(f'Found {error_count} errors and {warning_count} warnings in log')
+
+    return error_count == 0 and warning_count <= 1
+
+
+def tsan_log_is_valid(pid):
+    # TSAN_OPITONS was set to this log_prefix in docker-compose.yml
+    logdata = get_sanitizer_log(pid, TSAN_LOG_PREFIX)
+    return len(logdata) == 0
 
 
 def get_odyssey_pids():
@@ -67,6 +100,9 @@ def main():
                   't a a bt', '-ex', 'source /gdb.py', '-ex', 'mmcoro all bt']).decode('utf-8'))
             exit(1)
 
+        if not asan_log_is_valid(pid) or not tsan_log_is_valid(pid):
+            exit(1)
+
     exit(0)
 
 

docker/functional/docker-compose.yml

@@ -3,7 +3,7 @@ services:
     extra_hosts:
       - "ip4-localhost:127.0.0.1"
     ulimits:
-      core:
+      core: # sudo sysctl -w kernel.core_pattern=/var/cores/core-%e-%p-%t
         soft: -1
         hard: -1
     privileged: true
@@ -22,6 +22,7 @@ services:
       TSAN_OPTIONS: "log_path=/tsan-output.log"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+      - /var/cores:/var/cores
     ports:
       - "2222:22"
       - "7777:7777"

docker/functional/tests/cascade/odyssey-gateway.conf

@@ -12,6 +12,7 @@ log_to_stdout no
 log_config yes
 log_session yes
 log_stats yes
+stats_interval 1
 
 coroutine_stack_size 24
 

docker/functional/tests/cascade/odyssey-root1.conf

@@ -12,6 +12,7 @@ log_to_stdout no
 log_config yes
 log_session yes
 log_stats yes
+stats_interval 1
 
 coroutine_stack_size 24
 

docker/functional/tests/cascade/odyssey-root2.conf

@@ -12,6 +12,7 @@ log_to_stdout no
 log_config yes
 log_session yes
 log_stats yes
+stats_interval 1
 
 coroutine_stack_size 24
 

docker/functional/tests/lagpolling/lag-conf.conf

@@ -12,10 +12,10 @@ log_syslog no
 log_syslog_ident "odyssey"
 log_syslog_facility "daemon"
 
-log_debug no
+log_debug yes
 log_config yes
 log_session yes
-log_query no
+log_query yes
 log_stats yes
 stats_interval 60
 log_file "/var/log/odyssey.log"

docker/stress/Dockerfile

@@ -45,7 +45,8 @@ RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install
     gdb \
     build-essential \
     strace ltrace \
-    lsof
+    lsof \
+    python3
 
 # Taken from - https://docs.docker.com/engine/examples/running_ssh_service/#environment-variables
 
@@ -69,6 +70,8 @@ WORKDIR /stress-test
 
 COPY ./third_party/machinarium/gdb/machinarium-gdb.py /gdb.py
 
+COPY ./docker/functional/bin/ody-stop /usr/bin/ody-stop
+
 COPY --from=odyssey-build /odyssey /odyssey
 COPY --from=odyssey-build /odyssey.conf /odyssey.conf
 

docker/stress/docker-compose.yml

@@ -48,6 +48,9 @@ services:
       target: "${ODYSSEY_STRESS_TEST_TARGET:-stress-entrypoint}"
       args:
         odyssey_build_type: "${ODYSSEY_STRESS_BUILD_TYPE:-build_release}"
+    environment:
+      ASAN_OPTIONS: "log_path=/asan-output.log fast_unwind_on_malloc=0"
+      TSAN_OPTIONS: "log_path=/tsan-output.log"
     networks:
       - od_stress_net
     depends_on:

docker/stress/entrypoint.sh

@@ -54,8 +54,4 @@ echo "[`date` entrypoint] tuser finished"
 ps aux | head -n 1
 ps aux | grep odyssey
 
-kill -s TERM $(pidof odyssey)
-sleep 3
-if ps aux | grep -q '[o]dyssey'; then
-  echo "Can't finish odyssey after sigterm"
-fi
+ody-stop

sources/grac_shutdown_worker.c

@@ -158,7 +158,6 @@ void od_grac_shutdown_worker(void *arg)
 	}
 
 	od_system_free(system);
-	od_global_destroy(global);
 
 	machine_msg_set_type(msg, OD_MSG_GRAC_SHUTDOWN_FINISHED);
 	machine_channel_write(channel, msg);

sources/rules.c

@@ -221,8 +221,6 @@ void od_rules_group_checker_run(void *arg)
 		rc = od_attach_extended(instance, "group_checker", router,
 					group_checker_client);
 		if (rc != OK_RESPONSE) {
-			/* 1 second soft interval */
-			machine_sleep(1000);
 			continue;
 		}
 
@@ -1174,6 +1172,9 @@ __attribute__((hot)) int od_rules_merge(od_rules_t *rules, od_rules_t *src,
 		od_list_init(&rule->link);
 		od_list_append(&rules->rules, &rule->link);
 #ifdef PAM_FOUND
+		if (rule->auth_pam_data) {
+			od_pam_auth_data_free(rule->auth_pam_data);
+		}
 		rule->auth_pam_data = od_pam_auth_data_create();
 #endif
 		count_new++;

sources/sighandler.c

@@ -255,5 +255,7 @@ void od_system_signal_handler(void *arg)
 
 	od_instance_free(instance);
 
+	od_global_destroy(od_global_get());
+
 	exit(0);
 }