Release v0.92

New features include:
- Full support for convolutional and pooling layers
- GPU acceleration of local Elemental GEMM operations
- Improved network and data reader support
-- Alexnet
-- VGG
-- CIFAR-10
- Added a suite of regularizers, objective functions, and metrics, including:
-- Batch normaalization
-- Drop-out
-- L2
- Dramatically improves the performance of inter-model communication
- Added suite of image prepossessing routines

Merge branch 'develop' to create release v0.92.

Author: bvanessen

.travis.yml

@@ -0,0 +1,49 @@
+# This will run on Travis' 'new' container-based infrastructure
+sudo: false
+# The Trusty beta Build Environment
+dist: trusty
+
+language: cpp
+
+# Blacklist
+branches:
+  except:
+    - develop
+    - gh-pages
+
+# Environment variables (Set the gh-pages repo dir to html to match with doxygen)
+env:
+  global:
+    - GH_REPO_NAME: html
+    - DOXYFILE: $TRAVIS_BUILD_DIR/doc/Doxyfile
+    - GH_REPO_REF: github.com:LLNL/lbann.git
+    - DOXYCONFIG: $TRAVIS_BUILD_DIR/doc/Doxyfile.in
+    - SCRIPT_DIR: code_docs
+
+# Install dependencies
+addons:
+  apt:
+    packages:
+      - doxygen
+      - doxygen-doc
+      - doxygen-latex
+      - doxygen-gui
+      - graphviz
+
+# Build your code e.g. by calling make
+script:
+  - cp $DOXYCONFIG $DOXYFILE
+  - chmod +x config/config-doxygen.sh
+  - ./config/config-doxygen.sh
+
+# Generate and deploy documentation
+after_success:
+  - cd $TRAVIS_BUILD_DIR
+  - chmod +x config/publish-doxygen.sh
+  - ./config/publish-doxygen.sh
+
+before_install:
+  - openssl aes-256-cbc -K $encrypted_b9b32342f7d0_key -iv $encrypted_b9b32342f7d0_iv
+    -in config/travisci_rsa.enc -out config/travisci_rsa -d
+  - chmod 0600 config/travisci_rsa
+  - cp config/travisci_rsa ~/.ssh/id_rsa

CMakeLists.txt

@@ -3,18 +3,31 @@ cmake_minimum_required(VERSION 2.8.12)
 #set(LBANN_MAJOR_VERSION 0)
 #set(LBANN_MINOR_VERSION 91)
 
-# Get the git version so that we can embed it into the executable
-execute_process(COMMAND git describe --abbrev=7 --dirty --always --tags OUTPUT_VARIABLE GIT_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
-set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLBANN_VERSION=\\\"${GIT_VERSION}\\\"" )
+# Check to see if we are in a git repo
+execute_process(COMMAND git rev-parse --is-inside-work-tree OUTPUT_VARIABLE GIT_REPO OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+if(GIT_REPO)
+  # Get the git version so that we can embed it into the executable
+  execute_process(COMMAND git --git-dir ${LBANN_HOME}/.git describe --abbrev=7 --dirty --always --tags OUTPUT_VARIABLE GIT_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+  set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLBANN_VERSION=\\\"${GIT_VERSION}\\\"" )
+else()
+  if(LBANN_VER)
+    set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLBANN_VERSION=\\\"${LBANN_VER}\\\"" )
+  else()
+    set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLBANN_VERSION=\\\"v0.UKWN\\\"" )
+  endif()
+endif()
 
 ################################################################
 # Options
 ################################################################
 
-option(WITH_CUDA "Include Nvidia CUDA" ON)
-option(WITH_CUDNN "Include Nvidia cuDNN" ON)
-option(WITH_TBINF "Include Tensorboard interface" ON)
-option(VERBOSE "Verbose output" OFF)
+option(WITH_CUDA  "Include Nvidia CUDA"                    ON)
+option(WITH_CUDNN "Include Nvidia cuDNN"                   ON)
+option(WITH_CNPY  "Include cnpy"                           ON)
+option(WITH_TBINF "Include Tensorboard interface"          ON)
+option(WITH_VTUNE "Link the Intel VTune profiling library" OFF)
+option(VERBOSE    "Verbose output"                         OFF)
 if(NOT MAKE_NUM_PROCESSES)
   set(MAKE_NUM_PROCESSES 1)
 endif()
@@ -32,6 +45,30 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 # Initialize C++ flags
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -std=c++11")
 
+# Disable all optimization in debug for better viewing under debuggers (cmake already adds -g)
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -DLBANN_DEBUG")
+
+# Enable parallel random matrix generation
+if(NOT SEQ_INIT)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLBANN_PARALLEL_RANDOM_MATRICES")
+endif()
+
+# Special handling if we're compiling with Clang's address sanitizer
+execute_process(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE CXX_VERSION)
+if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+   if(NOT (CXX_VERSION VERSION_GREATER 3.5 OR CXX_VERSION VERSION_EQUAL 3.5))
+     message(FATAL_ERROR "LBANN & Elemental requires clang Version >= 3.5")
+   endif()
+   if(CMAKE_BUILD_TYPE MATCHES Debug)
+     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer -fsanitize-recover=address")
+     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fno-omit-frame-pointer -fsanitize-recover=address")
+   else()
+     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
+     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer")
+   endif()
+endif()
+
+
 # Set verbose output (should be 0 or 1)
 if(VERBOSE)
   set(VERBOSE 1)
@@ -63,6 +100,7 @@ set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
 # Add build and install directories to RPATH
+set(CMAKE_INSTALL_RPATH "${PROJECT_BINARY_DIR};${CMAKE_INSTALL_RPATH}")
 set(CMAKE_INSTALL_RPATH "${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR};${CMAKE_INSTALL_RPATH}")
 set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib;${CMAKE_INSTALL_RPATH}")
 set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR};${CMAKE_INSTALL_RPATH}")
@@ -74,22 +112,37 @@ set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR};${CMAKE
 # Required dependencies
 include(OpenMP)
 include(MPI)
-include(Elemental)
 include(OpenCV)
+include(protobuf)
+include(LbannProto)
 
 # Optional dependencies
 if(WITH_CUDA)
   include(CUDA)
 endif()
+
+if(ELEMENTAL_USE_CUBLAS)
+   set(EL_CUBLAS_FLAGS "-DEL_USE_CUBLAS -I${CUDA_INCLUDE_DIRS}")
+endif()
+
 if(CUDA_FOUND)
   include(CUB)
   if(WITH_CUDNN)
     include(cuDNN)
   endif()
 endif()
+
+include(Elemental)
+
 if(WITH_TBINF)
   include(TBinf)
 endif()
+if(WITH_VTUNE)
+  include(VTune)
+endif()
+if(WITH_CNPY)
+  include(cnpy)
+endif()
 include(Doxygen)
 
 ################################################################
@@ -107,7 +160,14 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel")
      message(FATAL_ERROR "LBANN & Elemental requires icpc Version >= 16.0")
    endif()
 elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-   message(FATAL_ERROR "Unsupported compiler: Clang")
+   if(NOT (CXX_VERSION VERSION_GREATER 3.5 OR CXX_VERSION VERSION_EQUAL 3.5))
+     message(FATAL_ERROR "LBANN & Elemental requires clang Version >= 3.5")
+   endif()
+   if(CMAKE_BUILD_TYPE MATCHES Debug)
+     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer -fsanitize-recover=address")
+   else()
+     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
+   endif()
 else()
    message(FATAL_ERROR "Unsupported compiler: Unknown compiler vendor")
 endif()
@@ -152,9 +212,15 @@ endif()
 if(LBANN_BUILT_CUB)
   add_dependencies(src project_CUB)
 endif()
+if(LBANN_BUILT_CNPY)
+  add_dependencies(src project_CNPY)
+endif()
 if(LBANN_HAS_TBINF)
   add_dependencies(src TBinf)
 endif()
+if(LBANN_BUILT_LBANN_PROTO)
+  add_dependencies(src LbannProto)
+endif()
 
 # Build LBANN library
 add_library(lbann ${LIBRARY_TYPE} $<TARGET_OBJECTS:src>)
@@ -163,15 +229,29 @@ add_library(lbann ${LIBRARY_TYPE} $<TARGET_OBJECTS:src>)
 target_link_libraries(lbann ${MPI_CXX_LIBRARIES})
 target_link_libraries(lbann ${Elemental_LIBRARIES})
 target_link_libraries(lbann ${OpenCV_LIBRARIES})
+target_link_libraries(lbann ${PROTOBUF_LIBRARIES})
+target_link_libraries(lbann ${CNPY_LIBRARIES})
+
+message("main CMakeFiles XXX LbannProto_LIBRARIES: ${LbannProto_LIBRARIES}")
+# target_link_libraries(lbann ${LbannProto_LIBRARIES})
+
 if(LBANN_HAS_CUDA)
   target_link_libraries(lbann ${CUDA_LIBRARIES})
+  target_link_libraries(lbann ${cuBLAS_LIBRARIES})
 endif()
 if(LBANN_HAS_CUDNN)
   target_link_libraries(lbann ${cuDNN_LIBRARIES})
 endif()
+if(LBANN_HAS_VTUNE)
+  target_link_libraries(lbann ${VTUNE_STATIC_LIB})
+  target_link_libraries(lbann dl)
+endif()
 if(LBANN_HAS_TBINF)
   target_link_libraries(lbann TBinf)
 endif()
+if(LBANN_HAS_LBANN_PROTO)
+  target_link_libraries(lbann LbannProto)
+endif()
 
 ################################################################
 # Install LBANN
@@ -222,6 +302,10 @@ message("  LBANN_HAS_ELEMENTAL:  ${LBANN_HAS_ELEMENTAL}")
 message("  LBANN_HAS_OPENCV:     ${LBANN_HAS_OPENCV}")
 message("  LBANN_HAS_CUDA:       ${LBANN_HAS_CUDA}")
 message("  LBANN_HAS_CUDNN:      ${LBANN_HAS_CUDNN}")
+message("  ELEMENTAL_USE_CUBLAS: ${ELEMENTAL_USE_CUBLAS}")
 message("  LBANN_HAS_PROTOBUF:   ${LBANN_HAS_PROTOBUF}")
+message("  LBANN_HAS_CNPY:       ${LBANN_HAS_CNPY}")
 message("  LBANN_HAS_TBINF:      ${LBANN_HAS_TBINF}")
+message("  LBANN_HAS_VTUNE:      ${LBANN_HAS_VTUNE}")
 message("  LBANN_HAS_DOXYGEN:    ${LBANN_HAS_DOXYGEN}")
+message("  LBANN_HAS_LBANN_PROTO:${LBANN_HAS_LBANN_PROTO}")

README.md

@@ -1,4 +1,25 @@
 # LBANN: Livermore Big Artificial Neural Network Toolkit
+
+## Building LBANN with Spack [for Users]
+
+   spack install lbann
+
+## Building LBANN with Spack [for Developers]
+
+### Using spack setup
+
+    http://spack.readthedocs.io/en/latest/packaging_guide.html?highlight=spack%20diy#build-system-configuration-support
+
+    cd lbann
+    spack setup lbann@local %[email protected] ^mvapich2
+    mkdir spack-build; cd spack-build
+    ../spconfig.py ..
+    make
+    make install
+
+### Using spack diy
+   spack diy lbann@local %[email protected]
+   spack diy lbann@local %[email protected]
 
 ## Using LBANN on LLNL LC clusters
 

build/.keep

@@ -9,6 +9,8 @@ if(CUDA_FOUND)
   # Include CUDA header files
   include_directories(${CUDA_INCLUDE_DIRS})
 
+  set(cuBLAS_LIBRARIES ${CUDA_TOOLKIT_ROOT_DIR}/lib64/${CMAKE_SHARED_LIBRARY_PREFIX}cublas${CMAKE_SHARED_LIBRARY_SUFFIX})
+
   # Add preprocessor flag for CUDA
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__LIB_CUDA")
 

cmake/CUDA.cmake

@@ -6,13 +6,22 @@ if(DOXYGEN_FOUND)
   # Status message
   message(STATUS "Found Doxygen (version ${DOXYGEN_VERSION}): ${DOXYGEN_EXECUTABLE}")
 
+  # Doxygen options
+  if(NOT DOXYGEN_OUTPUT_DIR)
+    set(DOXYGEN_OUTPUT_DIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_DOCDIR})
+  endif()
+
   # Configure Doxygen configuration file
+  set(DOXYFILE ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DOCDIR}/Doxyfile)
   configure_file(${PROJECT_SOURCE_DIR}/doc/Doxyfile.in
-                 ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DOCDIR}/Doxyfile @ONLY)
+                 ${DOXYFILE} @ONLY)
 
-  # Generate documentation with Doxygen
-  set(DOXYGEN_COMMAND ${DOXYGEN_EXECUTABLE} ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DOCDIR}/Doxyfile)
-  install(CODE "execute_process(COMMAND ${DOXYGEN_COMMAND} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DOCDIR})")
+  # Generate documentation
+  add_custom_target(doc
+    ${DOXYGEN_EXECUTABLE} ${DOXYFILE}
+    WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DOCDIR}
+    COMMENT "Generating API documentation with Doxygen" VERBATIM
+  )
 
   # LBANN has access to Doxygen
   set(LBANN_HAS_DOXYGEN TRUE)