Fix the conflicts happened in the merge of csr-interface-dev and develop

.gitlab-ci.yml

@@ -87,7 +87,7 @@ build_on_marianas:
   variables:
     SLURM_Q: "dl"
     MY_CLUSTER: "marianas"
-    TIMELIMIT: '1:00:00'
+    TIMELIMIT: '1:30:00'
     SLURM_ARGS: --gres=gpu:1 --exclusive
   <<: *pnnl_tags_definition
   <<: *pnnl_script_definition
@@ -107,6 +107,7 @@ build_on_newell:
     - if: '$CI_PROJECT_ROOT_NAMESPACE == "exasgd"'
 
 build_on_incline:
+  allow_failure: true
   stage: default_build
   variables:
     SLURM_Q: incline

BUILD.sh

@@ -118,7 +118,7 @@ if [ -z "$SLURM_SUBMIT_DIR" ]; then
 fi
 
 # Fail fast if we can't find NVBLAS_CONFIG_FILE since it's needed for all CUDA GPU builds
-if [[ ! $MY_CLUSTER =~ incline ]]; then
+if [[ ! $MY_CLUSTER =~ incline ]] && [[ ! $MY_CLUSTER =~ spock ]]; then
   if [[ ! -v NVBLAS_CONFIG_FILE ]] || [[ ! -f "$NVBLAS_CONFIG_FILE" ]]
   then
     echo "Please provide file 'nvblas.conf' or set variable to desired location."

CHANGELOG.md

@@ -1,6 +1,18 @@
 # Change Log
 All notable changes to HiOp will be documented in this file.
 
+## Version 0.5.4: Elastic mode, Krylov solvers, and misc bug fixes (March 2, 2022)
+New algorithmic features related to the NLP solver(s) and associated linear algebra KKT systems
+
+ - soft feasibility restoration
+ - Relaxer of equality constraints at the NLP formulation level
+ - Krylov interfaces and implementation for CG and BiCGStab (ready for device computations)
+ - protype of the condensed linear system and initial Krylov-based iterative refinement
+ - update of the Magma solver class for the latest Magma API
+ - elastic mode
+
+This release also includes several bug fixes.
+
 ## Version 0.5.3: xSDK compliance (Dec 3, 2021)
 
  xSDK compliance 

CMakeLists.txt

@@ -5,6 +5,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 set(CMAKE_CUDA_SEPARABLE_COMPILATION ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if (POLICY CMP0074)
   cmake_policy(SET CMP0074 NEW)
@@ -34,12 +35,14 @@ option(HIOP_BUILD_SHARED "Build shared library" OFF)
 option(HIOP_BUILD_STATIC "Build static library" ON)
 option(HIOP_SPARSE "Build with sparse linear algebra" ON)
 option(HIOP_USE_COINHSL "Build with sparse linear algebra" ON)
-option(HIOP_USE_STRUMPACK "Build with STRUMPACK backend for sparse linear algebra" OFF)
 option(HIOP_USE_PARDISO "Build with PARDISO backend for sparse linear algebra" OFF)
+option(HIOP_USE_STRUMPACK "Build with STRUMPACK backend for sparse linear algebra" OFF)
 option(HIOP_WITH_VALGRIND_TESTS "Run valgrind on certain integration tests" OFF)
 option(HIOP_BUILD_DOCUMENTATION "Build HiOp documentation via Doxygen" ON)
 
 set(HIOP_CTEST_LAUNCH_COMMAND "" CACHE STRING "Manually override launch commands used when running CTest")
+set(HIOP_CTEST_OUTPUT_DIR ${PROJECT_BINARY_DIR} CACHE PATH "Directory where CTest outputs are saved")
+mark_as_advanced(HIOP_CTEST_OUTPUT_DIR) # Hide this var as the default will be overriden rarely
 set(HIOP_EXTRA_LINK_FLAGS "" CACHE STRING "Manually pass extra flags to linker")
 
 # Ensure at least one library type is being built!
@@ -81,6 +84,9 @@ cmake_dependent_option(
 cmake_dependent_option(
   HIOP_USE_MAGMA "Use Magma linear algebra" ON "HIOP_USE_GPU" OFF
 )
+cmake_dependent_option(
+  HIOP_USE_CUSOLVER "Build with cuSolver linear algebra" ON "HIOP_USE_CUDA" OFF
+)
 
 add_library(hiop_tpl INTERFACE)
 add_library(hiop_options INTERFACE)
@@ -181,9 +187,9 @@ target_link_libraries(hiop_tpl INTERFACE ${LAPACK_LIBRARIES})
 message(STATUS "Using LAPACK libraries: ${LAPACK_LIBRARIES}")
 
 if(HIOP_USE_GPU)
+  include(CheckLanguage)
   set(HIOP_MAGMA_DIR CACHE PATH "Path to Magma directory")
   if(HIOP_USE_CUDA)
-    include(CheckLanguage)
     enable_language(CUDA)
     check_language(CUDA)
 
@@ -229,6 +235,14 @@ if(HIOP_USE_RAJA)
   target_link_libraries(hiop_tpl INTERFACE umpire RAJA)
 endif()
 
+# KLU needed with cuSolver. TODO: make this an optional dependency
+if (HIOP_USE_CUSOLVER)
+  set(HIOP_KLU_DIR CACHE PATH "Path to KLU directory")
+  include(FindKLU)
+  target_link_libraries(hiop_tpl INTERFACE KLU)
+endif()
+
+
 if(HIOP_WITH_KRON_REDUCTION)
   set(HIOP_UMFPACK_DIR CACHE PATH "Path to UMFPACK directory")
   include(FindUMFPACK)
@@ -432,28 +446,28 @@ if (HIOP_WITH_MAKETEST)
 
   add_test(NAME NlpMixedDenseSparse4_1 COMMAND ${RUNCMD} bash -c "$<TARGET_FILE:nlpMDS_ex4.exe> 400 100 0 -selfcheck \
     | ${STRIP_TABLE_CMD} \
-    | tee ${PROJECT_BINARY_DIR}/mds4_1.out")
+    | tee ${HIOP_CTEST_OUTPUT_DIR}/mds4_1.out")
   add_test(NAME NlpMixedDenseSparse4_2 COMMAND ${RUNCMD} bash -c "$<TARGET_FILE:nlpMDS_ex4.exe> 400 100 1 -selfcheck \
     | ${STRIP_TABLE_CMD} \
-    | tee ${PROJECT_BINARY_DIR}/mds4_2.out")
+    | tee ${HIOP_CTEST_OUTPUT_DIR}/mds4_2.out")
 
   add_test(NAME NlpMixedDenseSparse4_3 COMMAND ${RUNCMD} "$<TARGET_FILE:nlpMDS_ex4.exe>" "400" "100" "0" "-empty_sp_row" "-selfcheck")
 
   if(HIOP_USE_RAJA)
     add_test(NAME NlpMixedDenseSparseRaja4_1 COMMAND ${RUNCMD} bash -c "$<TARGET_FILE:nlpMDS_ex4_raja.exe> 400 100 0 -selfcheck \
       | ${STRIP_TABLE_CMD} \
-      | tee ${PROJECT_BINARY_DIR}/mds4_raja_1.out")
+      | tee ${HIOP_CTEST_OUTPUT_DIR}/mds4_raja_1.out")
     add_test(NAME NlpMixedDenseSparseRaja4_2 COMMAND ${RUNCMD} bash -c "$<TARGET_FILE:nlpMDS_ex4_raja.exe> 400 100 1 -selfcheck \
       | ${STRIP_TABLE_CMD} \
-      | tee ${PROJECT_BINARY_DIR}/mds4_raja_2.out")
+      | tee ${HIOP_CTEST_OUTPUT_DIR}/mds4_raja_2.out")
     add_test(NAME NlpMixedDenseSparseRaja4_3 COMMAND ${RUNCMD} bash -c "$<TARGET_FILE:nlpMDS_ex4_raja.exe> 400 100 0 -empty_sp_row -selfcheck")
 
     if(HIOP_DEEPCHECKS)
       foreach(iter 1 2)
         add_test(
           NAME "CompareExample4_NumIterations_${iter}" 
           COMMAND bash -c "\
-          if [[ $(wc -l ${PROJECT_BINARY_DIR}/mds4_${iter}.out|cut -f1 -d' ') == $(wc -l ${PROJECT_BINARY_DIR}/mds4_raja_${iter}.out|cut -f1 -d' ') ]]
+          if [[ $(wc -l ${HIOP_CTEST_OUTPUT_DIR}/mds4_${iter}.out|cut -f1 -d' ') == $(wc -l ${HIOP_CTEST_OUTPUT_DIR}/mds4_raja_${iter}.out|cut -f1 -d' ') ]]
           then
           echo 'Output tables have the same number of iterations.'
           exit 0
@@ -464,7 +478,7 @@ if (HIOP_WITH_MAKETEST)
         add_test(
           NAME "CompareExample4_ElementWise_${iter}"
           COMMAND bash -c "\
-          join ${PROJECT_BINARY_DIR}/mds4_${iter}.out ${PROJECT_BINARY_DIR}/mds4_raja_${iter}.out \
+          join ${HIOP_CTEST_OUTPUT_DIR}/mds4_${iter}.out ${HIOP_CTEST_OUTPUT_DIR}/mds4_raja_${iter}.out \
           | ${PROJECT_SOURCE_DIR}/tests/testEx4CompareIterations.awk")
       endforeach()
     endif(HIOP_DEEPCHECKS)
@@ -475,11 +489,17 @@ if (HIOP_WITH_MAKETEST)
   if(HIOP_SPARSE)
     add_test(NAME NlpSparse6_1 COMMAND ${RUNCMD} "$<TARGET_FILE:nlpSparse_ex6.exe>" "500" "-selfcheck")
     add_test(NAME NlpSparse6_2 COMMAND ${RUNCMD} "$<TARGET_FILE:nlpSparse_ex6.exe>" "500" "-fr" "-selfcheck")
+    if(HIOP_USE_CUDA)
+      add_test(NAME NlpSparse6_3 COMMAND ${RUNCMD} "$<TARGET_FILE:nlpSparse_ex6.exe>" "500" "-cusolver" "-selfcheck")
+    endif(HIOP_USE_CUDA)
     if(HIOP_USE_PARDISO)
-      add_test(NAME NlpSparse6_3 COMMAND ${RUNCMD} "$<TARGET_FILE:nlpSparse_ex6.exe>" "500" "-pardiso" "-selfcheck")
+      add_test(NAME NlpSparse6_4 COMMAND ${RUNCMD} "$<TARGET_FILE:nlpSparse_ex6.exe>" "500" "-pardiso" "-selfcheck")
     endif(HIOP_USE_PARDISO)
     add_test(NAME NlpSparse7_1 COMMAND ${RUNCMD} "$<TARGET_FILE:nlpSparse_ex7.exe>" "500" "-selfcheck")
     add_test(NAME NlpSparse7_2 COMMAND ${RUNCMD} "$<TARGET_FILE:nlpSparse_ex7.exe>" "500" "-inertiafree" "-selfcheck")
+    if(HIOP_USE_CUDA)
+      add_test(NAME NlpSparse7_3 COMMAND ${RUNCMD} "$<TARGET_FILE:nlpSparse_ex7.exe>" "500" "-cusolver" "-selfcheck")
+    endif(HIOP_USE_CUDA)
   endif(HIOP_SPARSE)
 
   if(HIOP_USE_MPI)

cmake/FindKLU.cmake

@@ -0,0 +1,54 @@
+
+#[[
+
+Looks for `klu` library and header directory.
+
+Exports target `KLU` which links to umfpack.(so|a)
+and add include directories where klu.h was found.
+
+Users may set the following variables:
+
+- HIOP_KLU_DIR
+
+]]
+
+
+find_library(KLU_LIBRARY
+  NAMES
+  klu
+  PATHS
+  ${KLU_DIR} $ENV{KLU_DIR} ${HIOP_KLU_DIR}
+  ENV LD_LIBRARY_PATH ENV DYLD_LIBRARY_PATH
+  PATH_SUFFIXES
+  lib64 lib)
+
+if(KLU_LIBRARY)
+  get_filename_component(KLU_LIBRARY_DIR ${KLU_LIBRARY} DIRECTORY)
+endif()
+
+find_path(KLU_INCLUDE_DIR
+  NAMES
+  klu.h
+  PATHS
+  ${KLU_DIR} $ENV{KLU_DIR} ${HIOP_KLU_DIR} ${KLU_LIBRARY_DIR}/..
+  PATH_SUFFIXES
+  include
+  include/suitesparse
+  include/klu)
+
+if(KLU_LIBRARY)
+  message(STATUS "Found klu include: ${KLU_INCLUDE_DIR}")
+  message(STATUS "Found klu library: ${KLU_LIBRARY}")
+  add_library(KLU INTERFACE)
+  target_link_libraries(KLU INTERFACE ${KLU_LIBRARY})
+  target_include_directories(KLU INTERFACE ${KLU_INCLUDE_DIR})
+  get_filename_component(KLU_LIB_DIR ${KLU_LIBRARY} DIRECTORY)
+  set(CMAKE_INSTALL_RPATH "${KLU_LIB_DIR}")
+  install(TARGETS KLU EXPORT hiop-targets)
+else()
+  message(STATUS "KLU was not found.")
+endif()
+
+set(KLU_INCLUDE_DIR CACHE PATH "Path to klu.h")
+set(KLU_LIBRARY CACHE PATH "Path to klu library")
+

doc/src/hiop_examples.tex

@@ -0,0 +1,78 @@
+\documentclass[11pt,letterpaper]{article}
+
+\usepackage[utf8]{inputenc}
+
+\usepackage{amsmath}
+\usepackage{amsfonts}
+\usepackage{amssymb}
+\usepackage{algorithm}
+\usepackage{algcompatible}
+%\usepackage{comment}
+
+\usepackage{graphicx}
+\usepackage{xcolor}
+
+\usepackage{hyperref}
+\hypersetup{
+    colorlinks,%
+    citecolor=green,%
+    filecolor=black,%
+    linkcolor=red,%
+    urlcolor=blue
+}
+
+\bibliographystyle{unsrt}
+
+\title{$ $
+\\[-100pt] Solving Saddle Point Linear Systems}
+\author{Shaked Regev and Kasia \'{S}wirydowicz}
+
+\newcommand{\pmat}[1]{\begin{pmatrix}#1\end{pmatrix}}
+\newcommand{\bmat}[1]{\begin{bmatrix}#1\end{bmatrix}}
+\newcommand{\Red}[1]{\textcolor{red}#1}
+
+\begin{document}
+
+%\section{Example 1 --- A Dense Example}
+%\section{Example 2 --- A Dense Example}
+%\section{Example 3 --- A Dense Example}
+%\section{Example 4 --- A MDS Example}
+%\section{Example 5 --- A MDS Example}
+
+\section{Example 6 --- A Convex Sparse Example}
+\begin{align}
+&&&&\min_{x\in\mathbb{R}^n} & \hspace{0.5cm} \frac{a}{4}\sum_{i=1}^{n} (x_i-1)^4 &&&& \label{ex6:obj}\\
+&&&&\textnormal{s.t.} &\hspace{0.5cm}  4ax_1+2ax_2 = 10a &&&& \label{ex6:eq1}\\
+&&&&& \hspace{0.5cm} 5a \leq  2ax_1 + ax_3                 &&&& \label{ex6:ineq1}\\
+&&&&& \hspace{0.5cm}  a \leq  2ax_1       + 0.5ax_i \leq 2an &&&& \label{ex6:ineq2}\\
+&&&&& \hspace{0.5cm}  x_1~\textnormal{free}, &&&& \label{ex6:varb1}\\
+&&&&& \hspace{0.5cm}  0.0 \leq x_2, &&&& \label{ex6:varb2}\\
+&&&&& \hspace{0.5cm}  1.5 \leq x_3 \leq 10  &&&& \label{ex6:varb3}\\
+&&&&& \hspace{0.5cm}  0.5 \leq x_i, \ \forall i=4,...,n               &&& \label{ex6:varb4}
+\end{align}
+
+Here $n\geq3$ and $a>0$ are parameters that can be modified via Ex6 driver's command arguments. Their default values are $n=3$ and $a=1.0$.
+
+The analytical optimal solution is  $x_1=1.75$, $x_2=x_3=1.5$, and $x_i=1$ for $i\in \{4,5,\ldots,n\}$. The objective value is $0.110352$. The file nlpSparse\_ex6\_driver.cpp provides more details about how to use HiOp to solve instances of this example/test problem.
+
+\section{Example 7 --- A Nonconvex Sparse Example}
+\begin{align}
+    &&&&\min_{x\in\mathbb{R}^n} & \hspace{0.5cm} -\frac{a}{4}\sum_{i=1}^{n} (x_i-1)^4 + 0.5\sum_{i=1}^{n} x_i^2&&&& \label{ex7:obj}\\
+    &&&&\textnormal{s.t.} &\hspace{0.5cm}  4x_1+2x_2 = 10 &&&& \label{ex7:eq1}\\
+    &&&&& \hspace{0.5cm}  4x_1+2x_2 = 10 &&&& \label{ex7:defeq}\\
+    &&&&& \hspace{0.5cm}  5 \leq  2x_1 + x_3                 &&&& \label{ex7:ineq1}\\
+    &&&&& \hspace{0.5cm}          4x_1      + 2x_3   \leq 19 &&&& \label{ex7:defineq}\\
+    &&&&& \hspace{0.5cm}  1 \leq  2x_1      + 0.5x_i \leq 2n &&&& \label{ex7:ineq2}\\
+    &&&&& \hspace{0.5cm}  x_1~\textnormal{free}, &&&& \label{ex7:varb1}\\
+    &&&&& \hspace{0.5cm}  0.0 \leq x_2, &&&& \label{ex7:varb2}\\
+    &&&&& \hspace{0.5cm}  1.5 \leq x_3 \leq 10  &&&& \label{ex7:varb3}\\
+    &&&&& \hspace{0.5cm}  0.5 \leq x_i, \ \forall i=4,...,n    &&& \label{ex7:varb4}
+\end{align}
+
+Here $n\geq3$ and $a>0$ are parameters which can be tuned via Ex6 driver's command arguments. Their default values are  $n=3$ and $a=0.1$. Note that the equality constraints \eqref{ex7:eq1} and \eqref{ex7:defeq} are duplicate. This is done on purpose to make the constraint Jacobian matrix rank deficient and to stress test HiOp on this numerically difficult, nonconvex problem. The file nlpSparse\_ex7\_driver.cpp provides more details about how to use HiOp in various configurations to solve instances of this example/test problem.
+
+
+%\section{Example 8 --- A Primal Decomposition Example}
+%\section{Example 9 --- A Primal Decomposition Example}
+
+\end{document}