Merge remote-tracking branch 'upstream/master' into release/7.5

Author: btovar

.github/workflows/integration.yml

@@ -28,7 +28,7 @@ jobs:
       CCTOOLS_DOCKER_GITHUB: ${{ matrix.os-name }}
     strategy: 
       matrix:
-        os-name: ['centos7', 'centos8', 'ubuntu20.04']
+        os-name: ['centos7', 'almalinux8', 'ubuntu20.04']
     steps:
       - name: checkout CCTools from branch head
         if: github.event_name != 'workflow_dispatch'

doc/manuals/install/index.md

@@ -91,7 +91,7 @@ a new `cctools-dev` environment via Conda:
 
 ```sh
 unset PYTHONPATH
-conda create -y -n cctools-dev -c conda-forge --strict-channel-priority python=3 gcc_linux-64 gxx_linux-64 gdb m4 perl swig make zlib libopenssl-static openssl conda-pack
+conda create -y -n cctools-dev -c conda-forge --strict-channel-priority python=3 gcc_linux-64 gxx_linux-64 gdb m4 perl swig make zlib libopenssl-static openssl conda-pack packaging cloudpickle
 conda activate cctools-dev
 ```
 

doc/manuals/taskvine/index.md

@@ -24,6 +24,13 @@ the cluster.
 
 <img src=images/architecture.svg>
 
+The TaskVine system is naturally robust.  While an application is running,
+workers may be added or removed as computing resources become available.
+Newly added workers will gradually accumulate data within the cluster.
+Removed (or failed) workers are handled gracefully, and tasks will be
+retried elsewhere as needed.  If a worker failure results in the loss
+of files, tasks will be re-executed as necessary to re-create them.
+
 TaskVine manager applications can be written in Python or C
 on Linux or OSX platforms.  Individual tasks can be simple
 Python functions, complex Unix applications, or serverless function
@@ -47,8 +54,7 @@ Tasks share a common set of options.  Each task can be labelled with the resourc
 (CPU cores, GPU devices, memory, disk space) that it needs to execute.  This allows each worker to pack the appropriate
 number of tasks.  For example, a worker running on a 64-core machine could run 32 dual-core tasks, 16 four-core tasks,
 or any other combination that adds up to 64 cores.  If you don't know the resources needed, you can enable
-Automatic 
-a resource monitor to track and report what each task uses.
+an automatic resource monitor to track and report what each task uses.
 
 TaskVine is easy to deploy on existing HPC and cloud facilities.
 The worker processes are self-contained executables, and TaskVine
@@ -219,6 +225,8 @@ visible within the manager application.
 used to capture the output of a task, and then serve as the input
 of a later task.  Temporary files exist only within the cluster
 for the duration of a workflow, and are deleted when no longer needed.
+If a temporary file is unexpectedly lost due to the crash or failure
+of a worker, then the task that created it will be re-executed.
 
 If it is necessary to unpack a file before it is used,
 use the `declare_untar` transformation to wrap the file definition.
@@ -724,6 +732,28 @@ that peer transfers are not permitted:
     ```
     vine_declare_file(m,"myfile.txt",VINE_CACHE|VINE_PEER_NOSHARE)
     ```
+Automatic sharing of files between workers, or peer transfers, are enabled by default
+in TaskVine. If communication between workers is not possible or not desired, peer transfers
+may be globally disabled:
+
+=== "Python"
+    ```python
+    m.disable_peer_transfers()
+    ```
+=== "C"
+    ```
+    vine_disable_peer_transfers(m);
+    ```
+If peer transfers have been disabled, they may be re-enabled accordingly:
+
+=== "Python"
+    ```python
+    m.enable_peer_transfers()
+    ```
+=== "C"
+    ```
+    vine_enable_peer_transfers(m);
+    ```
 
 ### MiniTasks
 

dttools/src/histogram.c

@@ -47,22 +47,14 @@ struct histogram *histogram_create(double bucket_size) {
 	return h;
 }
 
-void histogram_clear(struct histogram *h) {
-
-	uint64_t key;
-	struct box_count *box;
-
-	itable_firstkey(h->buckets);
-	while(itable_nextkey(h->buckets, &key, (void **) &box)) {
-		free(box);
-	}
+void histogram_clear(struct histogram *h)
+{
+	itable_clear(h->buckets,free);
 
 	h->total_count = 0;
 	h->max_value   = 0;
 	h->min_value   = 0;
 	h->mode        = 0;
-
-	itable_clear(h->buckets);
 }
 
 void histogram_delete(struct histogram *h) {

dttools/src/itable.c

@@ -50,14 +50,15 @@ struct itable *itable_create(int bucket_count)
 	return h;
 }
 
-void itable_clear(struct itable *h)
+void itable_clear( struct itable *h, void (*delete_func)( void *) )
 {
 	struct entry *e, *f;
 	int i;
 
 	for(i = 0; i < h->bucket_count; i++) {
 		e = h->buckets[i];
 		while(e) {
+			if(delete_func) delete_func(e->value);
 			f = e->next;
 			free(e);
 			e = f;
@@ -71,7 +72,7 @@ void itable_clear(struct itable *h)
 
 void itable_delete(struct itable *h)
 {
-	itable_clear(h);
+	itable_clear(h,0);
 	free(h->buckets);
 	free(h);
 }

dttools/src/itable.h

@@ -56,11 +56,11 @@ ITABLE_ITERATE(h,key,value) {
 struct itable *itable_create(int buckets);
 
 /** Remove all entries from an integer table.
-Note that this function will not delete all of the objects contained within the integer table.
 @param h The integer table to delete.
+@param delete_func If non-null, will be invoked on each object to delete it.
 */
 
-void itable_clear(struct itable *h);
+void itable_clear( struct itable *h, void (*delete_func)(void*) );
 
 /** Delete an integer table.
 Note that this function will not delete all of the objects contained within the integer table.

dttools/src/sort_dir.c

@@ -29,8 +29,11 @@ int sort_dir(const char *dirname, char ***list, int (*sort) (const char *a, cons
 			n += 1;
 		}
 		closedir(dir);
+	} else {
+		return 0;
 	}
 
+
 	if(sort) {
 		qsort(*list, n, sizeof(char *), (int (*)(const void *, const void *)) sort);
 	}

dttools/test/test_runner_common.sh

@@ -116,7 +116,7 @@ run_wq_worker()
 	return 0
 }
 
-run_ds_worker()
+run_taskvine_worker()
 {
 	local port_file=$1
 	shift

packaging/scripts/generate-images

@@ -136,9 +136,12 @@ my %preinstall_for;
 # list of shell commands to execute post installation.
 my %postinstall_for;
 
-$versions_of{centos} = [ qw{ 8 7 } ];
+$versions_of{centos} = [ qw{ 7 } ];
 $command_for{centos} = 'yum install -y';
 
+$versions_of{almalinux} = [ qw{ 8 9 } ];
+$command_for{almalinux} = 'dnf install -y';
+
 $versions_of{fedora} = [ qw{ 30  } ];
 $command_for{fedora} = 'dnf install -y';
 
@@ -171,7 +174,7 @@ gtar                     => 'tar',
 image_magick             => 'ImageMagick',
 libattr                  => 'libattr-devel',
 libffi                   => 'libffi',
-lsb_release              => 'redhat-lsb-core',
+lsb_release              => [],
 m4                       => 'm4',
 musl					 => [],             # from builder
 mysql                    => [],             # from builder
@@ -199,9 +202,11 @@ zlib                     => 'zlib-devel',
 #$package_for{centos}{6}{python3} = ['python34-devel', 'python34-setuptools'];  centos6 end-of-life
 $package_for{centos}{7}{python}   = ['python', 'python-devel', 'python-setuptools', 'python-tools'];
 $package_for{centos}{7}{python3}  = ['python36-devel', 'python36-setuptools', 'python36-pip'];
+$package_for{centos}{7}{lsb_release} = ['redhat-lsb-core'];
 
 $package_for{centos}{8}{python3}  = ['python39-devel', 'python39-setuptools', 'python39-pip'];
 $package_for{centos}{8}{swig}     = ['swig'];
+$package_for{centos}{8}{lsb_release} = ['redhat-lsb-core'];
 
 $extras_for{centos}{6} = ['libc-devel'];
 $extras_for{centos}{7} = ['libc-devel'];
@@ -218,15 +223,6 @@ $preinstall_for{centos}{7} = [
 	'yum -y install https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm'
 ];
 
-$preinstall_for{centos}{8} = [
-	"sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*",
-	"sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*",
-	'dnf install -y "dnf-command(config-manager)"',
-	'dnf config-manager -y --set-enabled powertools',  # for doxygen, and groff
-	'dnf install -y epel-release',
-	'dnf install -y https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm'
-];
-
 $postinstall_for{centos}{7} = [
 	@{$builder},
 	@{$fuse},
@@ -240,23 +236,45 @@ $postinstall_for{centos}{7} = [
 	'yum -y clean all',
 ];
 
-$postinstall_for{centos}{8} = [
+# end-of-life
+#$postinstall_for{centos}{6} = [
+#	'ln -sf /usr/bin/python3.4 /usr/bin/python3',
+#	'ln -sf /usr/bin/python3.4-config /usr/bin/python3-config'
+#];
+
+########## almalinux ##########
+$package_for{almalinux}{default}     = { %{$package_for{centos}{default}} };
+
+$preinstall_for{almalinux}{default} = [
+	'dnf install -y "dnf-command(config-manager)"',
+	'dnf install -y epel-release',
+	'dnf install -y https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm'
+];
+
+$preinstall_for{almalinux}{8} = [
+	'dnf config-manager -y --set-enabled powertools',  # for doxygen, and groff
+];
+
+$preinstall_for{almalinux}{9} = [
+	'dnf config-manager -y --set-enabled crb',  # for doxygen, and groff
+];
+
+$postinstall_for{almalinux}{default} = [
 	'ln -s /usr/bin/python3 /usr/bin/python',
 	@{$builder},
 	@{$fuse},
 	@{$e2fsprogs},
-	@{$mysql},
-	@{$uuid},
-	'yum -y install cvmfs-devel',
-	'yum -y clean all',
+	@{$mysql}, # does not compile in alma9
+	'dnf -y install cvmfs-devel',
+	'dnf -y clean all',
 ];
 
-# end-of-life
-#$postinstall_for{centos}{6} = [
-#	'ln -sf /usr/bin/python3.4 /usr/bin/python3',
-#	'ln -sf /usr/bin/python3.4-config /usr/bin/python3-config'
-#];
+$package_for{almalinux}{default}{python3}  = ['python3-devel', 'python3-setuptools', 'python3-pip'];
+$package_for{almalinux}{default}{swig}     = ['swig'];
+$extras_for{almalinux}{default} = ['glibc-devel', 'libuuid-devel', 'diffutils'];
 
+$package_for{almalinux}{9}{lsb_release} = []; # there is no redhat-lsb package for redhat9
+$package_for{almalinux}{9}{curl} = []; # curl already in base image and there is a conflict if we try to update it
 
 ########## FEDORA ##########
 $package_for{fedora}{default}     = { %{$package_for{centos}{default}} };

poncho/src/poncho/package_create.py

@@ -97,8 +97,24 @@ def _copy_run_in_env(env_dir):
         f.write('exec "${env_dir}"/bin/poncho_package_run -e ${env_dir} "$@"\n')
     os.chmod(f'{env_dir}/env/bin/run_in_env', 0o755)
 
+def pack_env_with_conda_dir(spec, output, ignore_editable_packages=False):
+    # remove trailing slash if present
+    spec = spec[:-1] if spec[-1] == '/' else spec 
+    try:
+        logger.info('packaging the environment...')
+        os.makedirs(f'{spec}/bin/', exist_ok=True)
+        os.makedirs(f'{spec}/env/bin/', exist_ok=True)
+        _copy_run_in_env(spec)
+        os.rename(f'{spec}/env/bin/poncho_package_run', f'{spec}/bin/poncho_package_run')
+        os.rename(f'{spec}/env/bin/run_in_env', f'{spec}/bin/run_in_env')
+        
+        conda_pack.pack(prefix=f'{spec}', output=str(output), force=True, ignore_missing_files=True, ignore_editable_packages=ignore_editable_packages)
+        logger.info('to activate environment run poncho_package_run -e {} <command>'.format(output))
+        return output
+    except Exception as e:
+        raise Exception(f"Error when packing a conda directory.\n{e}")
 
-def pack_env(spec, output, conda_executable=None, download_micromamba=None):
+def pack_env_with_spec(spec, output, conda_executable=None, download_micromamba=False, ignore_editable_packages=False):
     # record packages installed as editable from pip
     local_pip_pkgs = _find_local_pip()
 
@@ -138,13 +154,22 @@ def pack_env(spec, output, conda_executable=None, download_micromamba=None):
         # Bug breaks bundling common packages (e.g. python).
         # ignore_missing_files may be safe to remove in the future.
         # https://github.com/conda/conda-pack/issues/145
-        conda_pack.pack(prefix=f'{env_dir}/env', output=str(output), force=True, ignore_missing_files=True)
+        if ignore_editable_packages is not True:
+            ignore_editable_packages = False
+        conda_pack.pack(prefix=f'{env_dir}/env', output=str(output), force=True, ignore_missing_files=True, ignore_editable_packages=ignore_editable_packages)
 
         logger.info('to activate environment run poncho_package_run -e {} <command>'.format(output))
 
     return output
 
+def pack_env(spec, output, conda_executable=None, download_micromamba=False, ignore_editable_packages=False):
+    # pack a conda directory directly
+    if not os.path.isfile(spec) and spec != "-":
+        pack_env_with_conda_dir(spec, output, ignore_editable_packages)
 
+    # else if spec is a file or from stdin
+    else:
+        pack_env_with_spec(spec, output, conda_executable, download_micromamba, ignore_editable_packages)
 
 def _run_conda_command(environment, needs_confirmation, command, *args):
     all_args = [conda_exec] + command.split()
@@ -160,7 +185,6 @@ def _run_conda_command(environment, needs_confirmation, command, *args):
         print(e.output.decode())
         sys.exit(1)
 
-
 def _find_local_pip():
     edit_raw = subprocess.check_output([sys.executable, '-m' 'pip', 'list', '--editable']).decode()
 

poncho/src/poncho_package_create

@@ -2,12 +2,13 @@
 from poncho import package_create as create
 import argparse
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Create a packed environment from a spec.')
-    parser.add_argument('spec', help='Read in a spec file, or - for stdin.')
+    parser = argparse.ArgumentParser(description='Create a packed environment from a spec or a conda directory.')
+    parser.add_argument('spec', help='Read in a spec file, a conda directory, or - for stdin.')
     parser.add_argument('output', help='Write output from conda-pack to the given file.')
     parser.add_argument('--conda-executable', action='store_true', help='Path to conda executable to use. Default are, in this order: mamba, $CONDA_EXE, conda')
 
     parser.add_argument('--no-micromamba', action='store_true', help='Do not try no download micromamba if a conda executable is not found.')
+    parser.add_argument('--ignore-editable-packages', action='store_true', help='Skip checks for editable packages.')
 
     args = parser.parse_args()
-    create.pack_env(args.spec, args.output, args.conda_executable, not args.no_micromamba)
+    create.pack_env(args.spec, args.output, args.conda_executable, not args.no_micromamba, args.ignore_editable_packages)