More conda build fixes

- Allow overriding libtiledbvcf path from command line
- Use -g and -O0 with debug option
- Don't overlink the arrow core library
- Bump the pyarrow minimum version to fix test errors
- Switch CI to ubuntu-18.04 image

Author: ihnorton

.azure-pipelines.yml

@@ -9,7 +9,7 @@ jobs:
   strategy:
     matrix:
       ubuntu_16:
-        imageName: 'ubuntu-16.04'
+        imageName: 'ubuntu-18.04'
         python.version: '3.x'
         CXX: g++
         BUILD_PYTHON_API: ON
@@ -25,4 +25,4 @@ jobs:
     inputs:
       versionSpec: '$(python.version)'
       architecture: 'x64'
-  - template: ci/azure-linux_mac.yml
+  - template: ci/azure-linux_mac.yml

apis/python/conda-env.yml

@@ -4,7 +4,7 @@ channels:
 dependencies:
   - python>=3.4
   - pybind11=2.3.0
-  - pyarrow=0.14.1
+  - pyarrow>=0.16.0
   - dask>=0.19.0
   - pip
   - pip:

apis/python/setup.py

@@ -39,13 +39,18 @@
 import subprocess
 import sys
 
+TILEDBVCF_DEBUG_BUILD = False
+LIBTILEDBVCF_PATH = None
+
 args = sys.argv[:]
 for arg in args:
   if arg.find('--debug') == 0:
       TILEDBVCF_DEBUG_BUILD = True
       sys.argv.remove(arg)
-  else:
-      TILEDBVCF_DEBUG_BUILD = False
+  if arg.find('--libtiledbvcf') == 0:
+      LIBTILEDBVCF_PATH = arg.split('=')[1]
+      sys.argv.remove(arg)
+
 
 class get_pybind_include(object):
     """Helper class to determine the pybind11 include path
@@ -81,6 +86,10 @@ class PathConfig(object):
 
 def find_libtiledbvcf():
     p = PathConfig()
+
+    if LIBTILEDBVCF_PATH:
+        p.native_lib_install_dirs = [LIBTILEDBVCF_PATH]
+
     libdirs = ['lib']
     libnames = ['libtiledbvcf.dylib', 'libtiledbvcf.so']
     for root in p.native_lib_install_dirs:
@@ -161,16 +170,17 @@ def find_or_build_libtiledbvcf(setuptools_cmd):
     tiledbvcf_ext.include_dirs += [inc_dir]
 
     # Copy native libs into the package dir so they can be found by package_data
-    native_libs = [os.path.join(lib_dir, f) for f in os.listdir(lib_dir)]
-    p = PathConfig()
-    package_data = []
-    for obj in native_libs:
-        shutil.copy(obj, p.pkg_src)
-        package_data.append(os.path.basename(obj))
+    if LIBTILEDBVCF_PATH is None:
+        native_libs = [os.path.join(lib_dir, f) for f in os.listdir(lib_dir)]
+        p = PathConfig()
+        package_data = []
+        for obj in native_libs:
+            shutil.copy(obj, p.pkg_src)
+            package_data.append(os.path.basename(obj))
 
-    # Install shared libraries inside the Python module via package_data.
-    print('Adding to package_data: {}'.format(package_data))
-    setuptools_cmd.distribution.package_data.update({'tiledbvcf': package_data})
+        # Install shared libraries inside the Python module via package_data.
+        print('Adding to package_data: {}'.format(package_data))
+        setuptools_cmd.distribution.package_data.update({'tiledbvcf': package_data})
 
 
 def get_ext_modules():
@@ -195,8 +205,11 @@ class BuildExtCmd(build_ext):
 
     def build_extensions(self):
         opts = ['-std=c++11', '-g']
-        if not TILEDBVCF_DEBUG_BUILD:
-            opts.append('-O2')
+        if TILEDBVCF_DEBUG_BUILD:
+            opts.extend(['-O0'])
+        else:
+            opts.extend(['-O2'])
+
         link_opts = []
         for ext in self.extensions:
             ext.extra_compile_args = opts
@@ -205,6 +218,8 @@ def build_extensions(self):
             import pyarrow
             ext.include_dirs.append(pyarrow.get_include())
             ext.libraries.extend(pyarrow.get_libraries())
+            # don't overlink the arrow core library
+            if 'arrow' in ext.libraries: ext.libraries.remove('arrow')
             ext.library_dirs.extend(pyarrow.get_library_dirs())
 
         find_or_build_libtiledbvcf(self)

ci/azure-linux_mac.yml

@@ -37,9 +37,6 @@ steps:
   displayName: 'Install dependencies'
 
 - bash: |
-    # DELETEME work-around for https://github.com/microsoft/azure-pipelines-image-generation/issues/969
-    sudo chown root.root /
-
     # azure bash does not treat intermediate failure as error
     # https://github.com/Microsoft/azure-pipelines-yaml/issues/135
     set -e pipefail
@@ -61,12 +58,12 @@ steps:
     make check
     ../test/run-cli-tests.sh . ../test/inputs
 
-    sudo make install-libtiledbvcf
+    make install-libtiledbvcf
 
   displayName: 'Build and test TileDB-VCF'
 
 - bash: |
-    set -e pipefail
+    set -ex pipefail
 
     if [[ "$BUILD_PYTHON_API" == "ON" ]]; then
       # Add conda to PATH
@@ -119,4 +116,4 @@ steps:
            cat $f
         done;
   condition: failed() # only run this job if the build step failed
-  displayName: "Print log files (failed build only)"
+  displayName: "Print log files (failed build only)"