margo_finalize hangs if the process was forked

[mdorier]

In the following program, the child process hangs on margo_finalize. It only happens if the instance uses a progress thread.

#include <time.h>
#include <sys/types.h>
#include <unistd.h>
#include <sys/wait.h>
#include <stdio.h>
#include <stdlib.h>
#include <margo.h>

int main(int argc, char** argv) {

    pid_t pid = getpid();
    printf("%d: before margo_init\n", pid);
    margo_instance_id mid = margo_init("na+sm", MARGO_CLIENT_MODE, true, 0);

    printf("%d: before fork\n", pid);
    pid_t p = fork();
    if(p < 0 ) {
      perror("fork fail");
      exit(1);
    }

    pid = getpid();
    printf("%d: before margo_finalize\n", pid);
    margo_finalize(mid);

    printf("%d: before wait\n", pid);
    if(p > 0) wait(NULL);

    printf("%d: all done\n", pid);
    return 0;
}

[mdorier]

Actually this Argobots-only version has the same problem:

#include <time.h>
#include <sys/types.h>
#include <unistd.h>
#include <sys/wait.h>
#include <stdio.h>
#include <stdlib.h>
#include <abt.h>

int main(int argc, char** argv) {

    ABT_pool pool;
    ABT_xstream xstream;

    pid_t pid = getpid();
    printf("%d: before ABT_init\n", pid);
    ABT_init(0, NULL);

    ABT_pool_create_basic(ABT_POOL_FIFO, ABT_POOL_ACCESS_MPMC, ABT_TRUE, &pool);
    ABT_xstream_create_basic(ABT_SCHED_BASIC, 1, &pool, ABT_SCHED_CONFIG_NULL, &xstream);

    printf("%d: before fork\n", pid);
    pid_t p = fork();
    if(p < 0 ) {
      perror("fork fail");
      exit(1);
    }

    pid = getpid();

    printf("%d: before ABT_xstream_join\n", pid);
    ABT_xstream_join(xstream);
    printf("%d: before ABT_xstream_free\n", pid);
    ABT_xstream_free(&xstream);

    printf("%d: before wait\n", pid);
    if(p > 0) wait(NULL);

    printf("%d: all done\n", pid);
    ABT_finalize();
    return 0;
}

It blocks at ABT_xstream_join. Turns out fork does NOT copy existing POSIX threads apart from the one calling fork. I did not know that. This is something we should be careful with.