Merge pull request #262 from Robbybp/relocatable-libs

Make sure distributed libraries are relocatable

Author: ksbeattie

Diff for: doc/build-local.md

@@ -0,0 +1,36 @@
+# Building IDAES binaries locally
+
+For testing purposes, it may be useful to build the binaries locally, i.e. not
+using the Docker build scripts. This is also necessary for the MacOS builds
+(see the MacOS section in `build.md` for more information).
+
+You can build locally by running the following scripts:
+
+- `scripts/compile_solvers.sh`: Compile solvers (including `k_aug`, `dot_sens`,
+`petsc`, and all solver-associated libraries).
+- `scripts/compile_libs.sh`: Compile shared libraries for external functions.
+
+It is recommended to perform an "out-of-source" build. This can be done by
+first running `scripts/build_directory.sh _build` from the top level of this
+repository. This copies source code, makefiles, patch files, and build scripts
+into the `_build` directory, where you will actually run the build.
+Move into the build directory with `cd _build`.
+
+The build scripts were not designed for local use, so there are a few gotchas:
+
+- We look for patch files in a specific location relative to the original
+working directory. I.e. you must run `compile_solvers.sh` from *one level above*
+the scripts directory in order for patches (e.g. `ipopt.pc.patch`) to be
+correctly applied.
+- `ipopt.pc.patch` assumes that `coinmumps` and `coinhsl` are listed as
+dependencies in the `ipopt.pc` file. If either is not found (i.e. we are
+compiling without HSL), the patch application will fail.
+- By default, Petsc is assumed to be installed in hard-coded, OS-dependent
+location. If you are compiling locally, you have probably installed it
+in your own preferred location. To override the default, set the `PETSC_DIR`
+environment variable to the location where you have installed Petsc.
+- If you have your own system-installed HSL libraries that are discoverable
+by the linker (i.e. `LD_LIBRARY_PATH` is set), the COIN-OR solvers will likely
+be able to find and link against them. In this case, you will see a
+`HSL Present: NO` message (as there was no `coinhsl.zip` in the parent of the
+working directory), but will likely still build HSL-enabled solvers.

Diff for: doc/build.md

@@ -114,6 +114,26 @@ Test on clean VM for now, hopfully GitHub actions runners will be available soon
 
 There is a GitHub actions test.
 
+## Testing a non-default branch
+
+By default, the Docker driver scripts (`docker/build.sh` and `docker\build.ps1`)
+checkout the `main` branch of `https://github.com/idaes/idaes-ext.git` to use
+for the build process. To test a different branch, arguments can be provided
+to the Docker driver scripts. For example, to test the `ubuntu2204` build with
+a custom branch called `mybranch` on `user`'s fork, run
+```bash
+./build.sh ubuntu2204 https://github.com/user/idaes-ext.git mybranch
+```
+To test the Windows build, run
+```powershell
+.\build.ps1 windows --no-cache https://github.com/user/idaes-ext.git mybranch
+```
+Note that the second argument to `build.ps1` is interpreted as an argument
+to `docker build`, so to run with a custom branch and no such argument, run
+```powershell
+.\build.ps1 windows https://github.com/user/idaes-ext.git mybranch
+```
+
 ## Release Hashes
 
 Collect all the tar files for a release in the same directory.

Diff for: docker/build-extensions/build.ps1

@@ -1,8 +1,19 @@
 $flavor = $args[0]
 $buildarg_1 = $args[1]  # just use this to pass in --no-cache or some such
 
-$repo = "https://github.com/idaes/idaes-ext.git"
-$branch = "main"
+# The 3rd and 4th arguments provided will be interpreted as repo and branch.
+# (If you don't want to use buildarg_1, just pass in an empty string.)
+$repo = $args[2]
+$branch = $args[3]
+
+# If repo and branch are not provided, use default values
+IF ($repo -eq $null){
+  $repo = "https://github.com/idaes/idaes-ext.git"
+}
+IF ($branch -eq $null){
+  $branch = "main"
+}
+
 $mname = "x86_64"
 
 IF ($flavor -eq "windows"){
@@ -35,19 +46,13 @@ docker build --rm ${buildarg_1} --build-arg repo=${repo} --build-arg branch=${br
 Remove-Item extras -Recurse -Force -Confirm:$false
 docker run --name ${flavor}_build_tmp -dt ${flavor}_build_itmp:latest
 docker stop ${flavor}_build_tmp
-docker cp ${flavor}_build_tmp:${wdir}/idaes-ext/dist-lib/idaes-lib-${flavor}-${mname}.tar.gz .
+docker cp ${flavor}_build_tmp:${wdir}/idaes-ext/dist-functions/idaes-functions-${flavor}-${mname}.tar.gz .
 try{
-  docker cp ${flavor}_build_tmp:${wdir}/idaes-ext/dist-solvers/idaes-solvers-${flavor}-${mname}.tar.gz .
+  docker cp ${flavor}_build_tmp:${wdir}/idaes-ext/dist/idaes-solvers-${flavor}-${mname}.tar.gz .
 }
 catch{
   echo "Solvers were not built."
 }
-try{
-  docker cp ${flavor}_build_tmp:${wdir}/idaes-ext/dist-petsc/idaes-petsc-${flavor}-${mname}.tar.gz .
-}
-catch{
-  echo "PETSc was not built."
-}
 docker rm ${flavor}_build_tmp
 docker rmi ${flavor}_build_itmp
 cp *.tar.gz ../tarballs/

Diff for: docker/build-extensions/build.sh

@@ -1,8 +1,21 @@
 flavor=$1
 mname=$2
 
-repo="https://github.com/idaes/idaes-ext.git"
-branch="main"
+repo=$3
+branch=$4
+if [ ! "$repo" ]; then
+    repo="https://github.com/idaes/idaes-ext.git"
+fi
+if [ ! "$branch" ]; then
+    branch="main"
+fi
+
+echo "build.sh script arguments:
+    OS: $flavor
+    Arch: $mname
+    Repo: $repo
+    Branch: $branch
+"
 
 if [ "$flavor" = "windows" ]; then
   image="idaes-ext-windows-build:latest"
@@ -37,9 +50,8 @@ docker exec "$flavor"_"$mname"_build_tmp sh -c "cd ${wdir}/idaes-ext && bash scr
 docker exec "$flavor"_"$mname"_build_tmp sh -c "cd ${wdir}/idaes-ext && bash scripts/compile_libs.sh ${flavor}"
 docker stop "$flavor"_"$mname"_build_tmp
 
-docker cp "$flavor"_"$mname"_build_tmp:"$wdir"/idaes-ext/dist-lib/idaes-lib-"$flavor"-"$mname".tar.gz .
-docker cp "$flavor"_"$mname"_build_tmp:"$wdir"/idaes-ext/dist-solvers/idaes-solvers-"$flavor"-"$mname".tar.gz .
-docker cp "$flavor"_"$mname"_build_tmp:"$wdir"/idaes-ext/dist-petsc/idaes-petsc-"$flavor"-"$mname".tar.gz .
+docker cp "$flavor"_"$mname"_build_tmp:"$wdir"/idaes-ext/dist-functions/idaes-functions-"$flavor"-"$mname".tar.gz .
+docker cp "$flavor"_"$mname"_build_tmp:"$wdir"/idaes-ext/dist/idaes-solvers-"$flavor"-"$mname".tar.gz .
 
 docker rm "$flavor"_"$mname"_build_tmp
 

Diff for: scripts/compile_libs.sh

@@ -30,30 +30,32 @@ make
 cd $IDAES_EXT
 
 # Collect files
-rm -rf ./dist-lib
-mkdir dist-lib
-cd dist-lib
-cp ../src/dist/*.so ./
+rm -rf ./dist-functions
+mkdir dist-functions dist-functions/lib
+#cd dist-functions
+cp ./src/dist/*.so ./dist-functions/lib/
 if [ ${osname} = "windows" ]; then
-  mv functions.so functions.dll
-  mv general_helmholtz_external.so general_helmholtz_external.dll
-  mv cubic_roots.so cubic_roots.dll
+  mv dist-functions/lib/functions.so dist-functions/lib/functions.dll
+  mv dist-functions/lib/general_helmholtz_external.so dist-functions/lib/general_helmholtz_external.dll
+  mv dist-functions/lib/cubic_roots.so dist-functions/lib/cubic_roots.dll
 fi
 if [ ${osname} = "darwin" ]; then
-  mv functions.so functions.dylib
-  mv general_helmholtz_external.so general_helmholtz_external.dylib
-  mv cubic_roots.so cubic_roots.dylib
+  mv dist-functions/lib/functions.so dist-functions/lib/functions.dylib
+  mv dist-functions/lib/general_helmholtz_external.so dist-functions/lib/general_helmholtz_external.dylib
+  mv dist-functions/lib/cubic_roots.so dist-functions/lib/cubic_roots.dylib
 fi
-cp ../license.txt ./license_lib.txt
-cp ../version.txt ./version_lib.txt
-mkdir ./helm_data
-cp ../src/dist/param_data/*.json ./helm_data/
-cp ../src/dist/param_data/*.nl ./helm_data/
-cp ../src/dist/param_data/*.py ./helm_data/
-sed s/"(DATE)"/`date +%Y%m%d-%H%M`/g version_lib.txt > tmp
+cp ./license.txt ./dist-functions/license_lib.txt
+cp ./version.txt ./dist-functions/version_lib.txt
+mkdir ./dist-functions/lib/helm_data
+cp ./src/dist/param_data/*.json ./dist-functions/lib/helm_data/
+cp ./src/dist/param_data/*.nl ./dist-functions/lib/helm_data/
+cp ./src/dist/param_data/*.py ./dist-functions/lib/helm_data/
+sed s/"(DATE)"/`date +%Y%m%d-%H%M`/g dist-functions/version_lib.txt > tmp
 sed s/"(PLAT)"/${osname}-${MNAME}/g tmp > tmp2
-mv tmp2 version_lib.txt
+mv tmp2 dist-functions/version_lib.txt
 rm tmp
 
 # here you pack files
-tar -czvf idaes-lib-${osname}-${MNAME}.tar.gz *
+cd dist-functions
+tar -czvf idaes-functions-${osname}-${MNAME}.tar.gz *
+cd $IDAES_EXT