Merge pull request #372 from CamStan/master

Merge dev branch into master for release 0.9.0

TEST_CHECKPATCH_ALLOW_FAILURE=yes

Author: adammoody

.github/ISSUE_TEMPLATE.md

@@ -3,7 +3,7 @@ Thank you for reporting an issue.
 
 *IMPORTANT* - Please search our issue tracker *before* making a new issue.
 If you cannot find a similar issue, then create a new issue.
-https://github.com/LLNL/UnifyCR/issues
+https://github.com/LLNL/UnifyFS/issues
 
 Please fill in as much of the template as possible.
 -->
@@ -15,7 +15,7 @@ Type                   | Version/Name
 Operating System       |
 OS Version             |
 Architecture           |
-UnifyCR Version        |
+UnifyFS Version        |
 
 ### Describe the problem you're observing
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -25,7 +25,7 @@
 ### Checklist:
 <!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
 <!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
-- [ ] My code follows the UnifyCR code style requirements.
+- [ ] My code follows the UnifyFS code style requirements.
 - [ ] I have updated the documentation accordingly.
 - [ ] I have read the **CONTRIBUTING** document.
 - [ ] I have added tests to cover my changes.

.gitignore

@@ -47,13 +47,13 @@ log
 *_unittest
 
 # project files
-server/src/unifycrd
-util/unifycr/src/unifycr
+server/src/unifyfsd
+util/unifyfs/src/unifyfs
 
 # generated package files
-client/unifycr-config
-client/unifycr.pc
-extras/unifycr.conf
+client/unifyfs-config
+client/unifyfs.pc
+extras/unifyfs.conf
 libtool
 m4/libtool.m4
 m4/ltoptions.m4
@@ -64,24 +64,13 @@ meta/src/Mlog2/.dirstamp
 t/sys/.dirstamp
 
 # test files
-client/unifycr-runtime-config.h
-examples/src/app-btio-gotcha
-examples/src/app-btio-static
-examples/src/app-mpiio-gotcha
-examples/src/app-mpiio-static
-examples/src/sysio-read-gotcha
-examples/src/sysio-read-static
-examples/src/app-tileio-gotcha
-examples/src/app-tileio-static
-examples/src/sysio-write-gotcha
-examples/src/sysio-write-static
-examples/src/sysio-writeread-gotcha
-examples/src/sysio-writeread-static
-examples/src/sysio-writeread2-gotcha
-examples/src/sysio-writeread2-static
-examples/src/app-hdf5-create-gotcha
-examples/src/app-hdf5-writeread-gotcha
+client/unifyfs-runtime-config.h
+examples/src/*-gotcha
+examples/src/*-posix
+examples/src/*-static
 t/sys/open.t
 t/test-results/
-t/unifycr_unmount.t
+t/unifyfs_unmount.t
 t/test_run_env.sh
+deps
+install

.gitlab-ci.yml

@@ -0,0 +1,132 @@
+stages:
+  - build
+  - test-unit
+  - test-integ
+
+cache:
+  paths:
+      - spack_ci/
+
+##### Templates #####
+
+.quartz-template: &quartz_template
+  tags:
+    - quartz
+    - shell
+  variables:
+    LLNL_SERVICE_USER: "unifysrv"
+  retry:
+    max: 1
+    when:
+      - unknown_failure
+      - stuck_or_timeout_failure
+
+.butte-template: &butte_template
+  tags:
+    - butte
+    - shell
+  variables:
+    LLNL_SERVICE_USER: "unifysrv"
+  retry:
+    max: 1
+    when:
+      - unknown_failure
+      - stuck_or_timeout_failure
+
+.build-template: &build_template
+  stage: build
+  script:
+    - ./autogen.sh
+    - mkdir -p unifyfs-build unifyfs-install && cd unifyfs-build
+    - ../configure --prefix=$CI_PROJECT_DIR/unifyfs-install --enable-fortran --disable-silent-rules
+    - make V=1
+    - make V=1 install
+  artifacts:
+    name: "${CI_JOB_NAME}-${CI_PIPELINE_ID}"
+    untracked: true
+    expire_in: 1 hour
+    paths:
+      - unifyfs-build/
+      - unifyfs-install/
+
+.unit-test-template: &unit_test_template
+  stage: test-unit
+  script:
+    - cd unifyfs-build/t && make check
+  after_script:
+    - rm -rf /tmp/unify* /tmp/tmp.* /tmp/mdhim* /tmp/na_sm
+
+.quartz-batch-variables:
+  variables: &quartz_batch_variables
+    LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -p pbatch -t $WALL_TIME"
+    LLNL_SERVICE_USER: "unifysrv"
+    CI_PROJDIR: "$CI_PROJECT_DIR"
+    UNIFYFS_INSTALL: "$CI_PROJECT_DIR/unifyfs-install"
+    CI_NPROCS: "$NPROCS"
+
+.butte-batch-variables:
+  variables: &butte_batch_variables
+    LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes $NNODES -q pbatch -W $WALL_TIME"
+    LLNL_SERVICE_USER: "unifysrv"
+    CI_PROJDIR: "$CI_PROJECT_DIR"
+    UNIFYFS_INSTALL: "$CI_PROJECT_DIR/unifyfs-install"
+    CI_NPROCS: "$NPROCS"
+
+##### Jobs #####
+
+before_script:
+  # HERE BE DRAGONS!: Since on HPC and running as user, Spack might already
+  # exist and can get complicated if we install it again.
+  #
+  # check for sourced spack || check for unsourced spack in $HOME/spack and
+  # source it || check for cached spack, clone if none, and source it
+  - which spack || ((cd $HOME/spack && git describe) && . $HOME/spack/share/spack/setup-env.sh) || (((cd spack_ci && git describe) || git clone https://github.com/CamStan/spack spack_ci) && . spack_ci/share/spack/setup-env.sh)
+  - SPACK_ARCH=$(spack arch)
+  - spack install leveldb && spack load leveldb arch=$SPACK_ARCH
+  - spack install [email protected] && spack load [email protected] arch=$SPACK_ARCH
+  - spack install flatcc && spack load flatcc arch=$SPACK_ARCH
+  - spack install margo^mercury+bmi~boostsys && spack load argobots arch=$SPACK_ARCH && spack load mercury arch=$SPACK_ARCH && spack load margo arch=$SPACK_ARCH
+
+build-quartz:
+  <<: *quartz_template
+  <<: *build_template
+
+build-butte:
+  <<: *butte_template
+  <<: *build_template
+
+unit-test-quartz:
+  <<: *quartz_template
+  <<: *unit_test_template
+  dependencies:
+    - build-quartz
+
+unit-test-butte:
+  <<: *butte_template
+  <<: *unit_test_template
+  dependencies:
+    - build-butte
+
+#integ-test-quartz:
+#  <<: *quartz_template
+#  stage: test-integ
+#  tags:
+#    - quartz
+#    - batch
+#  variables: *quartz_batch_variables
+#  script:
+#    - cd t/ci && prove -v RUN_CI_TESTS.sh
+#  dependencies:
+#    - build-quartz
+
+integ-test-butte:
+  <<: *butte_template
+  stage: test-integ
+  tags:
+    - butte
+    - batch
+  variables: *butte_batch_variables
+  script:
+    - cd t/ci && prove -v RUN_CI_TESTS.sh
+  dependencies:
+    - build-butte

.travis.yml

@@ -1,27 +1,61 @@
 language: c
 sudo: required
+dist: xenial
+
+addons:
+  apt:
+    update: true
+    packages:
+      - autoconf
+      - automake
+      - build-essential
+      - cmake
+      - gfortran
+      - libhdf5-openmpi-dev
+      - libopenmpi-dev
+      - libtool-bin
+      - m4
+      - openmpi-bin
 
 before_install:
   # The default environment variable $CC is known to interfere with
   # MPI projects.
   - test -n $CC && unset CC
-  - sudo apt-get -qq update
-  - sudo apt-get install --yes -qq build-essential autoconf libtool cmake
-  - sudo apt-get install --yes -qq libopenmpi-dev openmpi-bin
-  - sudo apt-get install --yes -qq libhdf5-openmpi-dev
   - (cd $HOME/spack; git describe) || git clone https://github.com/spack/spack $HOME/spack
+  # Create packages.yaml to prevent building dependencies that time out
+  - |
+    test -f $HOME/spack/etc/spack/packages.yaml || cat > $HOME/spack/etc/spack/packages.yaml << '  EOF'
+    packages:
+      autoconf:
+        buildable: False
+        paths:
+          [email protected]: /usr
+      automake:
+        buildable: False
+        paths:
+          [email protected]: /usr
+      cmake:
+        buildable: False
+        paths:
+            [email protected]: /usr/local/cmake-3.12.4
+      libtool:
+        buildable: False
+        paths:
+          [email protected]: /usr
+      m4:
+        buildable: False
+        paths:
+          [email protected]: /usr
+    EOF
+
+install:
   - $HOME/spack/bin/spack install environment-modules
-  - $HOME/spack/bin/spack install leveldb
-  - $HOME/spack/bin/spack install [email protected]
-  - $HOME/spack/bin/spack install flatcc
-  # insall margo with spack
-  - git config --global http.sslVerify false
-  - git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git $HOME/sds-repo.git
-  - $HOME/spack/bin/spack repo add $HOME/sds-repo.git
-  - $HOME/spack/bin/spack install margo
-  - git config --global http.sslVerify true
-  # prepare build environment
   - . $HOME/spack/share/spack/setup-env.sh
+  - spack install leveldb
+  - spack install [email protected]
+  - spack install flatcc
+  - spack install margo^mercury+bmi~boostsys
+  # prepare build environment
   - spack load environment-modules
   - source <(spack module tcl loads leveldb [email protected] flatcc mercury argobots margo)
   - eval $(./scripts/git_log_test_env.sh)
@@ -39,7 +73,7 @@ script:
   # Force git to update the shallow clone and include tags so git-describe works
   - git fetch --unshallow --tags
   - sh autogen.sh
-  - ./configure || cat config.log
+  - ./configure --enable-fortran || cat config.log
   - make -k && make distcheck
   - ./scripts/checkpatch.sh || test "$TEST_CHECKPATCH_ALLOW_FAILURE" = yes
 

LICENSE

@@ -7,8 +7,8 @@ Copyright 2017, UT-Battelle, LLC.
 LLNL-CODE-741539
 All rights reserved.
 
-This is the license for UnifyCR. For details, see
-https://github.com/LLNL/UnifyCR.
+This is the license for UnifyFS. For details, see
+https://github.com/LLNL/UnifyFS.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile.am

@@ -5,7 +5,7 @@ CONFIG = ordered
 #ACLOCAL_AMFLAGS = -I m4
 
 pkgconfigdir = @pkgconfigdir@
-pkgconfig_DATA = client/unifycr.pc
+pkgconfig_DATA = client/unifyfs.pc
 
 CLEANFILES =
 

README.md

@@ -1,4 +1,4 @@
-# UnifyCR: A Distributed Burst Buffer File System - 0.1.0
+# UnifyFS: A Distributed Burst Buffer File System - 0.1.0
 
 Node-local burst buffers are becoming an indispensable hardware resource on
 large-scale supercomputers to buffer the bursty I/O from scientific
@@ -8,26 +8,26 @@ across different batch jobs. In addition, burst buffers need to cope with a
 variety of challenging I/O patterns from data-intensive scientific
 applications.
 
-UnifyCR is a user-level burst buffer file system under active development.
-UnifyCR supports scalable and efficient aggregation of I/O bandwidth from burst
-buffers while having the same life cycle as a batch-submitted job. While UnifyCR
-is designed for N-N write/read, UnifyCR compliments its functionality with the
+UnifyFS is a user-level burst buffer file system under active development.
+UnifyFS supports scalable and efficient aggregation of I/O bandwidth from burst
+buffers while having the same life cycle as a batch-submitted job. While UnifyFS
+is designed for N-N write/read, UnifyFS compliments its functionality with the
 support for N-1 write/read. It efficiently accelerates scientific I/O based on
 scalable metadata indexing, co-located I/O delegation, and server-side read
 clustering and pipelining.
 
 ## Documentation
-Full UnifyCR documentation is contained [here](https://unifycr.readthedocs.io).
+Full UnifyFS documentation is contained [here](https://unifyfs.readthedocs.io).
 
-Use [Build & I/O Interception](http://unifycr.readthedocs.io/en/dev/build-intercept.html)
-for instructions on how to build and install UnifyCR.
+Use [Build & I/O Interception](http://unifyfs.readthedocs.io/en/dev/build-intercept.html)
+for instructions on how to build and install UnifyFS.
 
 ## Build Status
-The current status of the UnifyCR dev branch is:
+The current status of the UnifyFS dev branch is:
 
-[![Build Status](https://api.travis-ci.org/LLNL/UnifyCR.png?branch=dev)](https://travis-ci.org/LLNL/UnifyCR)
-[![Read the Docs](https://readthedocs.org/projects/unifycr/badge/?version=dev)](https://unifycr.readthedocs.io)
+[![Build Status](https://api.travis-ci.org/LLNL/UnifyFS.png?branch=dev)](https://travis-ci.org/LLNL/UnifyFS)
+[![Read the Docs](https://readthedocs.org/projects/unifyfs/badge/?version=dev)](https://unifyfs.readthedocs.io)
 
 ## Contribute and Develop
 Contributing guidelines can be found in our
-[documentation](https://unifycr.readthedocs.io/en/dev/style-guides.html).
+[documentation](https://unifyfs.readthedocs.io/en/dev/style-guides.html).