summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorWillem Jan Palenstijn <wjp@usecode.org>2020-11-05 16:59:26 +0100
committerWillem Jan Palenstijn <wjp@usecode.org>2020-11-13 12:09:13 +0100
commit74f822299c0292a77a0dfdf79a8cee4a2d4b6c1d (patch)
treef6750929c990f81a02f12660337ad4239b400d11
parent038d3e752e033600a9f5337eb2bed1a4681c635b (diff)
downloadastra-74f822299c0292a77a0dfdf79a8cee4a2d4b6c1d.tar.gz
astra-74f822299c0292a77a0dfdf79a8cee4a2d4b6c1d.tar.bz2
astra-74f822299c0292a77a0dfdf79a8cee4a2d4b6c1d.tar.xz
astra-74f822299c0292a77a0dfdf79a8cee4a2d4b6c1d.zip
Split conda/cuda build environment in two
-rw-r--r--python/conda/libastra/linux_archive_build_config.yaml10
-rw-r--r--python/conda/libastra/linux_build_config.yaml5
-rw-r--r--python/conda/linux_release/buildenv/Dockerfile56
-rw-r--r--python/conda/linux_release/buildenv/archive.Dockerfile60
-rwxr-xr-xpython/conda/linux_release/buildenv/build.sh9
-rw-r--r--python/conda/linux_release/release.sh10
6 files changed, 93 insertions, 57 deletions
diff --git a/python/conda/libastra/linux_archive_build_config.yaml b/python/conda/libastra/linux_archive_build_config.yaml
new file mode 100644
index 0000000..b415b4a
--- /dev/null
+++ b/python/conda/libastra/linux_archive_build_config.yaml
@@ -0,0 +1,10 @@
+cudatoolkit:
+ - 8.0
+ - 9.0
+ - 9.2
+ - 10.0
+ - 10.1
+c_compiler_version: # [linux or win]
+ - 5.4 # [linux or win]
+cxx_compiler_version: # [linux or win]
+ - 5.4 # [linux or win]
diff --git a/python/conda/libastra/linux_build_config.yaml b/python/conda/libastra/linux_build_config.yaml
index 49f0038..936879c 100644
--- a/python/conda/libastra/linux_build_config.yaml
+++ b/python/conda/libastra/linux_build_config.yaml
@@ -1,9 +1,4 @@
cudatoolkit:
- - 8.0
- - 9.0
- - 9.2
- - 10.0
- - 10.1
- 10.2
- 11.0
c_compiler_version: # [linux or win]
diff --git a/python/conda/linux_release/buildenv/Dockerfile b/python/conda/linux_release/buildenv/Dockerfile
index 7cb6443..5c891ce 100644
--- a/python/conda/linux_release/buildenv/Dockerfile
+++ b/python/conda/linux_release/buildenv/Dockerfile
@@ -3,6 +3,13 @@ ENV DEBIAN_FRONTEND noninteractive
#RUN echo 'deb http://archive.debian.org/debian/ wheezy main' > /etc/apt/sources.list && echo 'deb http://archive.debian.org/debian-security/ wheezy/updates main' >> /etc/apt/sources.list && apt-get -o Acquire::Check-Valid-Until=false update && apt-get install -y perl-modules build-essential autoconf libtool automake libboost-dev git libxml2 && rm -rf /var/lib/apt/lists/*
RUN apt-get update && apt-get install -y perl-modules build-essential autoconf libtool automake libboost-dev git libxml2 && rm -rf /var/lib/apt/lists/*
+ENV PATH /root/miniconda3/bin:$PATH
+COPY Miniconda3-py38_4.8.3-Linux-x86_64.sh /root/
+RUN /bin/bash /root/Miniconda3-py38_4.8.3-Linux-x86_64.sh -b && \
+ rm -f /root/Miniconda3*
+
+RUN conda install -y conda-build conda-verify
+
FROM BUILDBASE AS CUDA110
RUN touch /root/cuda110
COPY cuda_11.0.3_450.51.06_linux.run /root
@@ -16,58 +23,9 @@ COPY cuda_10.2.89_440.33.01_linux.run /root
RUN /bin/bash /root/cuda_10.2.89_440.33.01_linux.run --toolkit --silent --installpath=/usr/local/cuda-10.2 && \
rm -f /root/cuda_10.2.89_440.33.01_linux.run
-FROM BUILDBASE AS CUDA101
-RUN touch /root/cuda101
-COPY cuda_10.1.168_418.67_linux.run /root
-RUN /bin/bash /root/cuda_10.1.168_418.67_linux.run --toolkit --silent --installpath=/usr/local/cuda-10.1 && \
- rm -f /root/cuda_10.1.168_418.67_linux.run
-
-FROM BUILDBASE AS CUDA100
-RUN touch /root/cuda100
-COPY cuda_10.0.130_410.48_linux /root
-RUN /bin/bash /root/cuda_10.0.130_410.48_linux --toolkit --silent && \
- rm -f /root/cuda_10.0.130_410.48_linux
-
-FROM BUILDBASE AS CUDA92
-RUN touch /root/cuda92
-COPY cuda_9.2.148_396.37_linux /root
-RUN /bin/bash /root/cuda_9.2.148_396.37_linux --toolkit --silent && \
- rm -f /root/cuda_9.2.148_396.37_linux
-
-FROM BUILDBASE AS CUDA90
-RUN touch /root/cuda90
-COPY cuda_9.0.176_384.81_linux-run /root
-RUN /bin/bash /root/cuda_9.0.176_384.81_linux-run --toolkit --silent && \
- rm -f /root/cuda_9.0.176_384.81_linux-run
-
-FROM BUILDBASE AS CUDA80
-RUN touch /root/cuda80
-COPY cuda_8.0.61_375.26_linux-run /root
-RUN /bin/bash /root/cuda_8.0.61_375.26_linux-run --toolkit --silent && \
- rm -f /root/cuda_8.0.61_375.26_linux-run
-COPY cuda_8.0.61.2_linux-run /root
-RUN /bin/bash /root/cuda_8.0.61.2_linux-run --silent --accept-eula && \
- rm -f /root/cuda_8.0.61.2_linux-run
-
FROM BUILDBASE
RUN touch /root/cuda
COPY --from=CUDA110 /usr/local/cuda-11.0 /usr/local/cuda-11.0
COPY --from=CUDA102 /usr/local/cuda-10.2 /usr/local/cuda-10.2
-COPY --from=CUDA101 /usr/local/cuda-10.1 /usr/local/cuda-10.1
-COPY --from=CUDA100 /usr/local/cuda-10.0 /usr/local/cuda-10.0
-COPY --from=CUDA92 /usr/local/cuda-9.2 /usr/local/cuda-9.2
-COPY --from=CUDA90 /usr/local/cuda-9.0 /usr/local/cuda-9.0
-COPY --from=CUDA80 /usr/local/cuda-8.0 /usr/local/cuda-8.0
-ENV PATH /root/miniconda3/bin:$PATH
-COPY Miniconda3-py38_4.8.3-Linux-x86_64.sh /root/
-RUN /bin/bash /root/Miniconda3-py38_4.8.3-Linux-x86_64.sh -b && \
- rm -f /root/Miniconda3*
-
-RUN conda install -y conda-build conda-verify
-# cudatoolkit < 9.0 is not easily available anymore from recent versions of conda.
-# We have to enable the free channel (globally) to reenable old versions of cudatookit..
-# See: https://docs.conda.io/projects/conda/en/latest/user-guide/configuration/free-channel.html#troubleshooting
-# TODO: Remove when support for cudatoolkit 8.0 is dropped.
-RUN conda config --set restore_free_channel true
diff --git a/python/conda/linux_release/buildenv/archive.Dockerfile b/python/conda/linux_release/buildenv/archive.Dockerfile
new file mode 100644
index 0000000..b3de3ec
--- /dev/null
+++ b/python/conda/linux_release/buildenv/archive.Dockerfile
@@ -0,0 +1,60 @@
+FROM debian:8 AS BUILDBASE
+ENV DEBIAN_FRONTEND noninteractive
+#RUN echo 'deb http://archive.debian.org/debian/ wheezy main' > /etc/apt/sources.list && echo 'deb http://archive.debian.org/debian-security/ wheezy/updates main' >> /etc/apt/sources.list && apt-get -o Acquire::Check-Valid-Until=false update && apt-get install -y perl-modules build-essential autoconf libtool automake libboost-dev git libxml2 && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y perl-modules build-essential autoconf libtool automake libboost-dev git libxml2 && rm -rf /var/lib/apt/lists/*
+
+ENV PATH /root/miniconda3/bin:$PATH
+COPY Miniconda3-py38_4.8.3-Linux-x86_64.sh /root/
+RUN /bin/bash /root/Miniconda3-py38_4.8.3-Linux-x86_64.sh -b && \
+ rm -f /root/Miniconda3*
+
+RUN conda install -y conda-build conda-verify
+
+# cudatoolkit < 9.0 is not easily available anymore from recent versions of conda.
+# We have to enable the free channel (globally) to reenable old versions of cudatookit..
+# See: https://docs.conda.io/projects/conda/en/latest/user-guide/configuration/free-channel.html#troubleshooting
+# TODO: Remove when support for cudatoolkit 8.0 is dropped.
+RUN conda config --set restore_free_channel true
+
+FROM BUILDBASE AS CUDA101
+RUN touch /root/cuda101
+COPY cuda_10.1.243_418.87.00_linux.run /root
+RUN /bin/bash /root/cuda_10.1.243_418.87.00_linux.run --toolkit --silent --installpath=/usr/local/cuda-10.1 && \
+ rm -f /root/cuda_10.1.243_418.87.00_linux.run
+
+FROM BUILDBASE AS CUDA100
+RUN touch /root/cuda100
+COPY cuda_10.0.130_410.48_linux /root
+RUN /bin/bash /root/cuda_10.0.130_410.48_linux --toolkit --silent && \
+ rm -f /root/cuda_10.0.130_410.48_linux
+
+FROM BUILDBASE AS CUDA92
+RUN touch /root/cuda92
+COPY cuda_9.2.148_396.37_linux /root
+RUN /bin/bash /root/cuda_9.2.148_396.37_linux --toolkit --silent && \
+ rm -f /root/cuda_9.2.148_396.37_linux
+
+FROM BUILDBASE AS CUDA90
+RUN touch /root/cuda90
+COPY cuda_9.0.176_384.81_linux-run /root
+RUN /bin/bash /root/cuda_9.0.176_384.81_linux-run --toolkit --silent && \
+ rm -f /root/cuda_9.0.176_384.81_linux-run
+
+FROM BUILDBASE AS CUDA80
+RUN touch /root/cuda80
+COPY cuda_8.0.61_375.26_linux-run /root
+RUN /bin/bash /root/cuda_8.0.61_375.26_linux-run --toolkit --silent && \
+ rm -f /root/cuda_8.0.61_375.26_linux-run
+COPY cuda_8.0.61.2_linux-run /root
+RUN /bin/bash /root/cuda_8.0.61.2_linux-run --silent --accept-eula && \
+ rm -f /root/cuda_8.0.61.2_linux-run
+
+FROM BUILDBASE
+RUN touch /root/cuda
+COPY --from=CUDA101 /usr/local/cuda-10.1 /usr/local/cuda-10.1
+COPY --from=CUDA100 /usr/local/cuda-10.0 /usr/local/cuda-10.0
+COPY --from=CUDA92 /usr/local/cuda-9.2 /usr/local/cuda-9.2
+COPY --from=CUDA90 /usr/local/cuda-9.0 /usr/local/cuda-9.0
+COPY --from=CUDA80 /usr/local/cuda-8.0 /usr/local/cuda-8.0
+
+
diff --git a/python/conda/linux_release/buildenv/build.sh b/python/conda/linux_release/buildenv/build.sh
index 04e4066..1b183b5 100755
--- a/python/conda/linux_release/buildenv/build.sh
+++ b/python/conda/linux_release/buildenv/build.sh
@@ -16,9 +16,14 @@ git clone --depth 1 --branch ${BRANCH} ${URL}
[ $# -eq 0 ] || perl -pi -e "s/^(\s*string:.+_)[0-9]+/\${1}$2/" astra-toolbox/python/conda/libastra/meta.yaml
[ $# -eq 0 ] || perl -pi -e "s/^(\s*-\s*libastra\s*==\s*)[0-9a-z+\.]+$/\${1}$1/" astra-toolbox/python/conda/astra-toolbox/meta.yaml
+if [ x$3 = xarchive ]; then
+ CONF=linux_archive_build_config.yaml
+else
+ CONF=linux_build_config.yaml
+fi
-conda-build -m astra-toolbox/python/conda/libastra/linux_build_config.yaml astra-toolbox/python/conda/libastra
+conda-build -m astra-toolbox/python/conda/libastra/${CONF} astra-toolbox/python/conda/libastra
-conda-build -m astra-toolbox/python/conda/astra-toolbox/linux_build_config.yaml astra-toolbox/python/conda/astra-toolbox
+[ x$3 = xarchive ] || conda-build -m astra-toolbox/python/conda/astra-toolbox/linux_build_config.yaml astra-toolbox/python/conda/astra-toolbox
cp /root/miniconda3/conda-bld/linux-64/*astra* /out
diff --git a/python/conda/linux_release/release.sh b/python/conda/linux_release/release.sh
index bf9279c..8294b88 100644
--- a/python/conda/linux_release/release.sh
+++ b/python/conda/linux_release/release.sh
@@ -4,14 +4,22 @@ set -e
D=`mktemp -d`
-for F in https://repo.anaconda.com/miniconda/Miniconda3-py38_4.8.3-Linux-x86_64.sh https://developer.nvidia.com/compute/cuda/8.0/Prod2/local_installers/cuda_8.0.61_375.26_linux-run https://developer.nvidia.com/compute/cuda/8.0/Prod2/patches/2/cuda_8.0.61.2_linux-run https://developer.nvidia.com/compute/cuda/9.0/Prod/local_installers/cuda_9.0.176_384.81_linux-run https://developer.nvidia.com/compute/cuda/9.2/Prod2/local_installers/cuda_9.2.148_396.37_linux https://developer.nvidia.com/compute/cuda/10.0/Prod/local_installers/cuda_10.0.130_410.48_linux https://developer.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.168_418.67_linux.run http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_440.33.01_linux.run https://developer.download.nvidia.com/compute/cuda/11.0.3/local_installers/cuda_11.0.3_450.51.06_linux.run ; do
+for F in https://repo.anaconda.com/miniconda/Miniconda3-py38_4.8.3-Linux-x86_64.sh http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_440.33.01_linux.run https://developer.download.nvidia.com/compute/cuda/11.0.3/local_installers/cuda_11.0.3_450.51.06_linux.run ; do
[ -f buildenv/`basename $F` ] || (cd buildenv; wget $F )
done
+for F in https://repo.anaconda.com/miniconda/Miniconda3-py38_4.8.3-Linux-x86_64.sh https://developer.nvidia.com/compute/cuda/8.0/Prod2/local_installers/cuda_8.0.61_375.26_linux-run https://developer.nvidia.com/compute/cuda/8.0/Prod2/patches/2/cuda_8.0.61.2_linux-run https://developer.nvidia.com/compute/cuda/9.0/Prod/local_installers/cuda_9.0.176_384.81_linux-run https://developer.nvidia.com/compute/cuda/9.2/Prod2/local_installers/cuda_9.2.148_396.37_linux https://developer.nvidia.com/compute/cuda/10.0/Prod/local_installers/cuda_10.0.130_410.48_linux http://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run ; do
+ [ -f buildenv/`basename $F` ] || (cd buildenv; wget $F )
+done
+
+
+
docker build -t astra-build-env buildenv
+docker build -t astra-build-env-archive -f buildenv/archive.Dockerfile buildenv
cp buildenv/build.sh $D
+docker run -v $D:/out:z astra-build-env-archive /bin/bash /out/build.sh 1.9.9.dev5 0 archive
docker run -v $D:/out:z astra-build-env /bin/bash /out/build.sh 1.9.9.dev5 0
rm -f $D/build.sh