From d621b2ceed23f4ecc945b00ffd6e859b70b89b50 Mon Sep 17 00:00:00 2001 From: Allard Hendriksen Date: Tue, 17 Sep 2019 14:43:58 +0200 Subject: Use conda compiler toolchain for conda builds For both libastra and astra-toolbox: 1) We do not use script_env to set CC/CXX anymore, since the compilers are installed by conda. 2) The build string is made useful by including either the python+numpy version or the cudatoolkit version that the package was built with. 3) Some clean-up of build.sh in buildenv/ For libastra: 1) The libastra.so is built with the conda C/C++ compiler toolchain. This has two benefits: 1) The rpath of libastra.so is set to $ORIGIN, which makes linking easier for dependent packages. 2) libastra.so is linkable against ancient versions of glibc. With old versions of memcpy. 2) The C/C++ compiler version is fixed to 5.4.0 3) In libastra/build.sh, we rename $CONDA_PREFIX to $PREFIX. Apparently, this is how it is supposed to be done. For me, $CONDA_PREFIX was suddenly undefined. Why this was not a problem before, is unclear to me. 4) The cudatoolkit runtime dependency is pinned with pin_compatible 5) The libastra conda package now provides headers and .pc file. This is useful for building C++ packages that depend on astra. 6) Remove some old code related to cudatoolkit<8.0. For astra-toolbox: 1) astra-toolbox uses the conda-provided compilers 2) The compilers are fixed to version 7.3 3) Add boost to host requirements of astra-toolbox Notes on testing: - The libastra build has been tested with all versions of cudatoolkit - The astra-toolbox build has been tested with all provided versions of python after building a single cudatoolkit version of libastra. How to test this branch: - It should work by just editing `python/conda/linux_release/buildenv/build.sh`. Set BRANCH=CI-use-conda-c-compiler-toolchain URL=https://github.com/ahendriksen/astra-toolbox and run release.sh from the `python/conda/linux_release` directory. --- .../astra-toolbox/conda_build_config_py27.yaml | 5 +++++ .../astra-toolbox/conda_build_config_py35.yaml | 4 ++++ .../astra-toolbox/conda_build_config_py36.yaml | 4 ++++ .../astra-toolbox/conda_build_config_py37.yaml | 4 ++++ python/conda/astra-toolbox/meta.yaml | 8 ++++--- python/conda/libastra/build.sh | 26 ++++++++-------------- python/conda/libastra/linux_build_config.yaml | 4 ++++ python/conda/libastra/meta.yaml | 13 ++++++----- python/conda/linux_release/buildenv/build.sh | 13 ++++++++--- 9 files changed, 52 insertions(+), 29 deletions(-) mode change 100644 => 100755 python/conda/libastra/build.sh mode change 100644 => 100755 python/conda/linux_release/buildenv/build.sh (limited to 'python') diff --git a/python/conda/astra-toolbox/conda_build_config_py27.yaml b/python/conda/astra-toolbox/conda_build_config_py27.yaml index f119b61..869d22e 100644 --- a/python/conda/astra-toolbox/conda_build_config_py27.yaml +++ b/python/conda/astra-toolbox/conda_build_config_py27.yaml @@ -8,3 +8,8 @@ numpy: - 1.14 - 1.15 - 1.16 + +c_compiler_version: + - 7.3 +cxx_compiler_version: + - 7.3 diff --git a/python/conda/astra-toolbox/conda_build_config_py35.yaml b/python/conda/astra-toolbox/conda_build_config_py35.yaml index 8dd963c..72d4cef 100644 --- a/python/conda/astra-toolbox/conda_build_config_py35.yaml +++ b/python/conda/astra-toolbox/conda_build_config_py35.yaml @@ -3,3 +3,7 @@ python: numpy: - 1.15 +c_compiler_version: + - 7.3 +cxx_compiler_version: + - 7.3 diff --git a/python/conda/astra-toolbox/conda_build_config_py36.yaml b/python/conda/astra-toolbox/conda_build_config_py36.yaml index f043435..996e22c 100644 --- a/python/conda/astra-toolbox/conda_build_config_py36.yaml +++ b/python/conda/astra-toolbox/conda_build_config_py36.yaml @@ -5,3 +5,7 @@ numpy: - 1.14 - 1.15 - 1.16 +c_compiler_version: + - 7.3 +cxx_compiler_version: + - 7.3 diff --git a/python/conda/astra-toolbox/conda_build_config_py37.yaml b/python/conda/astra-toolbox/conda_build_config_py37.yaml index ebbf44a..4282407 100644 --- a/python/conda/astra-toolbox/conda_build_config_py37.yaml +++ b/python/conda/astra-toolbox/conda_build_config_py37.yaml @@ -6,3 +6,7 @@ numpy: - 1.14 - 1.15 - 1.16 +c_compiler_version: + - 7.3 +cxx_compiler_version: + - 7.3 diff --git a/python/conda/astra-toolbox/meta.yaml b/python/conda/astra-toolbox/meta.yaml index 054751b..b3c036b 100644 --- a/python/conda/astra-toolbox/meta.yaml +++ b/python/conda/astra-toolbox/meta.yaml @@ -8,8 +8,7 @@ source: build: number: 0 - script_env: - - CC # [linux] + string: py_{{ python }}_numpy_{{ numpy }} test: imports: @@ -24,14 +23,17 @@ test: requirements: build: + - {{ compiler('c') }} # [win or linux] + - {{ compiler('cxx') }} # [linux] + host: - python - cython >=0.13 + - boost # [osx or linux] - nomkl # [not win] - numpy {{ numpy }} - scipy - six - libastra ==1.9.0.dev11 - run: - python - {{ pin_compatible('numpy', min_pin='x.x', max_pin='x.x') }} diff --git a/python/conda/libastra/build.sh b/python/conda/libastra/build.sh old mode 100644 new mode 100755 index d19c7af..4e48ba6 --- a/python/conda/libastra/build.sh +++ b/python/conda/libastra/build.sh @@ -18,35 +18,27 @@ cd $SRC_DIR/build/linux $SRC_DIR/build/linux/autogen.sh -# Add C++11 to compiler flags if nvcc supports it, mostly to work around a boost bug +# Add C++11 to compiler flags, mostly to work around a boost bug. +# Since we require cudatoolkit>=8.0 nvcc supports this. NVCC=$CUDA_ROOT/bin/nvcc -echo "int main(){return 0;}" > $CONDA_PREFIX/test.cu -$NVCC $CONDA_PREFIX/test.cu -ccbin $CC --std=c++11 -o $CONDA_PREFIX/test.out > /dev/null 2>&1 && EXTRA_NVCCFLAGS="--std=c++11" || true -rm -f $CONDA_PREFIX/test.out $CONDA_PREFIX/test.cu +EXTRA_NVCCFLAGS="--std=c++11" -$SRC_DIR/build/linux/configure --with-install-type=prefix --with-cuda=$CUDA_ROOT --prefix=$CONDA_PREFIX NVCCFLAGS="-ccbin $CC -I$CONDA_PREFIX/include $EXTRA_NVCCFLAGS" CC=$CC CXX=$CXX CPPFLAGS="-I$CONDA_PREFIX/include" + +$SRC_DIR/build/linux/configure --with-install-type=prefix --with-cuda=$CUDA_ROOT --prefix=$PREFIX NVCCFLAGS="-ccbin $CC -I$PREFIX/include $EXTRA_NVCCFLAGS" CC=$CC CXX=$CXX CPPFLAGS="-I$PREFIX/include" # Clean, because we may be re-using this source tree when building # multiple variants of this conda package. make clean -make -j $CPU_COUNT install-libraries +make -j $CPU_COUNT +make -j $CPU_COUNT install-dev test -d $CUDA_ROOT/lib64 && LIBPATH="$CUDA_ROOT/lib64" || LIBPATH="$CUDA_ROOT/lib" case `uname` in Darwin*) - cp -P $LIBPATH/libcudart.*.dylib $CONDA_PREFIX/lib - cp -P $LIBPATH/libcufft.*.dylib $CONDA_PREFIX/lib - ;; - Linux*) - if [ "$cudatoolkit" = "7.0" ]; then - # For some reason conda-build adds these symlinks automatically for - # cudatoolkit-5.5 and 6.0, but not 7.0. For 7.5 these symlinks are not - # necessary, and for 8.0 the cudatoolkit packages includes them. - ln -T -s libcudart.so.7.0.28 $CONDA_PREFIX/lib/libcudart.so.7.0 - ln -T -s libcufft.so.7.0.35 $CONDA_PREFIX/lib/libcufft.so.7.0 - fi + cp -P $LIBPATH/libcudart.*.dylib $PREFIX/lib + cp -P $LIBPATH/libcufft.*.dylib $PREFIX/lib ;; esac diff --git a/python/conda/libastra/linux_build_config.yaml b/python/conda/libastra/linux_build_config.yaml index 41db828..b415b4a 100644 --- a/python/conda/libastra/linux_build_config.yaml +++ b/python/conda/libastra/linux_build_config.yaml @@ -4,3 +4,7 @@ cudatoolkit: - 9.2 - 10.0 - 10.1 +c_compiler_version: # [linux or win] + - 5.4 # [linux or win] +cxx_compiler_version: # [linux or win] + - 5.4 # [linux or win] diff --git a/python/conda/libastra/meta.yaml b/python/conda/libastra/meta.yaml index 59c3a09..407929b 100644 --- a/python/conda/libastra/meta.yaml +++ b/python/conda/libastra/meta.yaml @@ -8,21 +8,22 @@ source: build: number: 0 - script_env: - - CC # [linux] - - CXX # [linux] + string: cuda_{{ cudatoolkit }} # [linux] requirements: build: - - {{compiler('c')}} # [win] + - {{ compiler('c') }} # [win or linux] + - {{ compiler('cxx') }} # [linux] - boost # [osx] - automake # [osx] - autoconf # [osx] - libtool # [osx] + host: + - boost # [osx or linux] - cudatoolkit {{ cudatoolkit }} # [linux] - run: - - cudatoolkit {{ cudatoolkit }} # [linux] + # See: https://github.com/conda-forge/conda-forge.github.io/issues/687#issuecomment-460095230 + - {{ pin_compatible('cudatoolkit', max_pin='x.x') }} # [linux] about: home: http://www.astra-toolbox.com diff --git a/python/conda/linux_release/buildenv/build.sh b/python/conda/linux_release/buildenv/build.sh old mode 100644 new mode 100755 index cc6dc31..158c1d5 --- a/python/conda/linux_release/buildenv/build.sh +++ b/python/conda/linux_release/buildenv/build.sh @@ -1,16 +1,23 @@ #!/bin/sh -export CC=gcc -export CXX=g++ +set -e + +BRANCH=master +URL=https://github.com/astra-toolbox/astra-toolbox + +echo "Cloning from ${URL}" +echo " branch: ${BRANCH}" cd /root -git clone --depth 1 --branch master https://github.com/astra-toolbox/astra-toolbox +git clone --depth 1 --branch ${BRANCH} ${URL} + [ $# -eq 0 ] || perl -pi -e "s/^(\s*version:\s*)[0-9a-z+\.']+$/\${1}'$1'/" astra-toolbox/python/conda/libastra/meta.yaml astra-toolbox/python/conda/astra-toolbox/meta.yaml [ $# -eq 0 ] || perl -pi -e "s/^(\s*number:\s*)[0-9]+$/\${1}$2/" astra-toolbox/python/conda/libastra/meta.yaml astra-toolbox/python/conda/astra-toolbox/meta.yaml [ $# -eq 0 ] || perl -pi -e "s/^(\s*-\s*libastra\s*==\s*)[0-9a-z+\.]+$/\${1}$1/" astra-toolbox/python/conda/astra-toolbox/meta.yaml conda-build -m astra-toolbox/python/conda/libastra/linux_build_config.yaml astra-toolbox/python/conda/libastra + for i in 27 35 36 37; do conda-build -m astra-toolbox/python/conda/astra-toolbox/conda_build_config_py$i.yaml astra-toolbox/python/conda/astra-toolbox done -- cgit v1.2.3