llvm.org GIT mirror llvm / 50f681d
[Dockerfiles] Split checkout and build scripts into separate files. Summary: This is a small refactoring to extract the svn checkout code from the build script used inside the docker image. This would give more flexibility if more than a single invocation of cmake is needed inside the docker image. User-facing interface (build_docker_image.sh) hasn't changed, only the internal scripts running inside the build container are affected. Reviewers: ioeric Reviewed By: ioeric Subscribers: mehdi_amini, llvm-commits Differential Revision: https://reviews.llvm.org/D45868 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@330412 91177308-0d34-0410-b5e6-96231b3b80d8 Ilya Biryukov 1 year, 5 months ago
6 changed file(s) with 247 addition(s) and 190 deletion(s). Raw diff Collapse all Expand all
1212 DOCKER_REPOSITORY=""
1313 DOCKER_TAG=""
1414 BUILDSCRIPT_ARGS=""
15 CHECKOUT_ARGS=""
16 CMAKE_ENABLED_PROJECTS=""
1517
1618 function show_usage() {
1719 cat << EOF
2426 -s|--source image source dir (i.e. debian8, nvidia-cuda, etc)
2527 -d|--docker-repository docker repository for the image
2628 -t|--docker-tag docker tag for the image
27 LLVM-specific:
29 Checkout arguments:
2830 -b|--branch svn branch to checkout, i.e. 'trunk',
2931 'branches/release_40'
3032 (default: 'trunk')
3941 Project 'llvm' is always included and ignored, if
4042 specified.
4143 Can be specified multiple times.
42 -i|--install-target name of a cmake install target to build and include in
43 the resulting archive. Can be specified multiple times.
4444 -c|--checksums name of a file, containing checksums of llvm checkout.
4545 Script will fail if checksums of the checkout do not
4646 match.
47 Build-specific:
48 -i|--install-target name of a cmake install target to build and include in
49 the resulting archive. Can be specified multiple times.
4750
4851 Required options: --source and --docker-repository, at least one
4952 --install-target.
7477
7578 CHECKSUMS_FILE=""
7679 SEEN_INSTALL_TARGET=0
80 SEEN_CMAKE_ARGS=0
7781 while [[ $# -gt 0 ]]; do
7882 case "$1" in
7983 -h|--help)
9599 DOCKER_TAG="$1"
96100 shift
97101 ;;
98 -i|--install-target|-r|--revision|-c|-cherrypick|-b|--branch|-p|--llvm-project)
99 if [ "$1" == "-i" ] || [ "$1" == "--install-target" ]; then
100 SEEN_INSTALL_TARGET=1
101 fi
102 -r|--revision|-c|-cherrypick|-b|--branch)
103 CHECKOUT_ARGS="$CHECKOUT_ARGS $1 $2"
104 shift 2
105 ;;
106 -i|--install-target)
107 SEEN_INSTALL_TARGET=1
102108 BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS $1 $2"
103109 shift 2
104110 ;;
111 -p|--llvm-project)
112 PROJ="$2"
113 if [ "$PROJ" == "cfe" ]; then
114 PROJ="clang"
115 fi
116
117 CHECKOUT_ARGS="$CHECKOUT_ARGS $1 $PROJ"
118 if [ "$PROJ" != "clang-tools-extra" ]; then
119 CMAKE_ENABLED_PROJECTS="$CMAKE_ENABLED_PROJECTS:$PROJ"
120 fi
121
122 shift 2
123 ;;
105124 -c|--checksums)
106125 shift
107126 CHECKSUMS_FILE="$1"
110129 --)
111130 shift
112131 BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -- $*"
132 SEEN_CMAKE_ARGS=1
113133 shift $#
114134 ;;
115135 *)
118138 ;;
119139 esac
120140 done
141
142
143 if [ "$CMAKE_ENABLED_PROJECTS" != "" ]; then
144 # Remove the leading ':' character.
145 CMAKE_ENABLED_PROJECTS="${CMAKE_ENABLED_PROJECTS:1}"
146
147 if [[ $SEEN_CMAKE_ARGS -eq 0 ]]; then
148 BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS --"
149 fi
150 BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -DLLVM_ENABLE_PROJECTS=$CMAKE_ENABLED_PROJECTS"
151 fi
121152
122153 command -v docker >/dev/null ||
123154 {
164195
165196 echo "Building ${DOCKER_REPOSITORY}${DOCKER_TAG} from $IMAGE_SOURCE"
166197 docker build -t "${DOCKER_REPOSITORY}${DOCKER_TAG}" \
198 --build-arg "checkout_args=$CHECKOUT_ARGS" \
167199 --build-arg "buildscript_args=$BUILDSCRIPT_ARGS" \
168200 -f "$BUILD_DIR/$IMAGE_SOURCE/Dockerfile" \
169201 "$BUILD_DIR"
4040
4141 ADD checksums /tmp/checksums
4242 ADD scripts /tmp/scripts
43 # Arguments passed to build_install_clang.sh.
43
44 # Checkout the source code.
45 ARG checkout_args
46 RUN /tmp/scripts/checkout.sh ${checkout_args}
47 # Run the build. Results of the build will be available at /tmp/clang-install/.
4448 ARG buildscript_args
45 # Run the build. Results of the build will be available at /tmp/clang-install/.
46 RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
49 RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args}
4750
4851
4952 # Stage 2. Produce a minimal release image with build results.
1818
1919 ADD checksums /tmp/checksums
2020 ADD scripts /tmp/scripts
21 # Arguments passed to build_install_clang.sh.
21
22 # Checkout the source code.
23 ARG checkout_args
24 RUN /tmp/scripts/checkout.sh ${checkout_args}
25 # Run the build. Results of the build will be available at /tmp/clang-install/.
2226 ARG buildscript_args
23 # Run the build. Results of the build will be available as /tmp/clang-install.
24 RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
27 RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args}
2528
2629
2730 # Stage 2. Produce a minimal release image with build results.
1616
1717 ADD checksums /tmp/checksums
1818 ADD scripts /tmp/scripts
19 # Arguments passed to build_install_clang.sh.
19
20 # Checkout the source code.
21 ARG checkout_args
22 RUN /tmp/scripts/checkout.sh ${checkout_args}
23 # Run the build. Results of the build will be available at /tmp/clang-install/.
2024 ARG buildscript_args
21 # Run the build. Results of the build will be available at /tmp/clang-install/.
22 RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
25 RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args}
2326
2427
2528 # Stage 2. Produce a minimal release image with build results.
1313 cat << EOF
1414 Usage: build_install_llvm.sh [options] -- [cmake-args]
1515
16 Checkout svn sources and run cmake with the specified arguments. Used
17 inside docker container.
16 Run cmake with the specified arguments. Used inside docker container.
1817 Passes additional -DCMAKE_INSTALL_PREFIX and puts the build results into
19 /tmp/clang-install/ directory.
18 the directory specified by --to option.
2019
2120 Available options:
2221 -h|--help show this help message
23 -b|--branch svn branch to checkout, i.e. 'trunk',
24 'branches/release_40'
25 (default: 'trunk')
26 -r|--revision svn revision to checkout
27 -c|--cherrypick revision to cherry-pick. Can be specified multiple times.
28 Cherry-picks are performed in the sorted order using the
29 following command:
30 'svn patch <(svn diff -c \$rev)'.
31 -p|--llvm-project name of an svn project to checkout. Will also add the
32 project to a list LLVM_ENABLE_PROJECTS, passed to CMake.
33 For clang, please use 'clang', not 'cfe'.
34 Project 'llvm' is always included and ignored, if
35 specified.
36 Can be specified multiple times.
3722 -i|--install-target name of a cmake install target to build and include in
3823 the resulting archive. Can be specified multiple times.
39 Required options: At least one --install-target.
24 --to destination directory where to install the targets.
25 Required options: --to, at least one --install-target.
4026
4127 All options after '--' are passed to CMake invocation.
4228 EOF
4329 }
4430
45 LLVM_SVN_REV=""
46 CHERRYPICKS=""
47 LLVM_BRANCH=""
4831 CMAKE_ARGS=""
4932 CMAKE_INSTALL_TARGETS=""
50 # We always checkout llvm
51 LLVM_PROJECTS="llvm"
52 CMAKE_LLVM_ENABLE_PROJECTS=""
53 CLANG_TOOLS_EXTRA_ENABLED=0
54
55 function contains_project() {
56 local TARGET_PROJ="$1"
57 local PROJ
58 for PROJ in $LLVM_PROJECTS; do
59 if [ "$PROJ" == "$TARGET_PROJ" ]; then
60 return 0
61 fi
62 done
63 return 1
64 }
65
66 function append_project() {
67 local PROJ="$1"
68
69 LLVM_PROJECTS="$LLVM_PROJECTS $PROJ"
70 if [ "$CMAKE_LLVM_ENABLE_PROJECTS" != "" ]; then
71 CMAKE_LLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS;$PROJ"
72 else
73 CMAKE_LLVM_ENABLE_PROJECTS="$PROJ"
74 fi
75 }
33 CLANG_INSTALL_DIR=""
7634
7735 while [[ $# -gt 0 ]]; do
7836 case "$1" in
79 -r|--revision)
80 shift
81 LLVM_SVN_REV="$1"
82 shift
83 ;;
84 -c|--cherrypick)
85 shift
86 CHERRYPICKS="$CHERRYPICKS $1"
87 shift
88 ;;
89 -b|--branch)
90 shift
91 LLVM_BRANCH="$1"
92 shift
93 ;;
94 -p|--llvm-project)
95 shift
96 PROJ="$1"
97 shift
98
99 if [ "$PROJ" == "cfe" ]; then
100 PROJ="clang"
101 fi
102
103 if [ "$PROJ" == "clang-tools-extra" ]; then
104 if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
105 echo "Project 'clang-tools-extra' is already enabled, ignoring extra occurrences."
106 else
107 CLANG_TOOLS_EXTRA_ENABLED=1
108 fi
109
110 continue
111 fi
112
113 if ! contains_project "$PROJ" ; then
114 append_project "$PROJ"
115 else
116 echo "Project '$PROJ' is already enabled, ignoring extra occurrences."
117 fi
118 ;;
11937 -i|--install-target)
12038 shift
12139 CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1"
40 shift
41 ;;
42 --to)
43 shift
44 CLANG_INSTALL_DIR="$1"
12245 shift
12346 ;;
12447 --)
14164 exit 1
14265 fi
14366
144 if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
145 if ! contains_project "clang"; then
146 echo "Project 'clang-tools-extra' was enabled without 'clang'."
147 echo "Adding 'clang' to a list of projects."
148
149 append_project "clang"
150 fi
67 if [ "$CLANG_INSTALL_DIR" == "" ]; then
68 echo "No install directory. Please specify the --to argument."
69 exit 1
15170 fi
15271
153 if [ "$LLVM_BRANCH" == "" ]; then
154 LLVM_BRANCH="trunk"
155 fi
72 CLANG_BUILD_DIR=/tmp/clang-build
15673
157 if [ "$LLVM_SVN_REV" != "" ]; then
158 SVN_REV_ARG="-r$LLVM_SVN_REV"
159 echo "Checking out svn revision r$LLVM_SVN_REV."
160 else
161 SVN_REV_ARG=""
162 echo "Checking out latest svn revision."
163 fi
74 mkdir -p "$CLANG_INSTALL_DIR"
16475
165 # Sort cherrypicks and remove duplicates.
166 CHERRYPICKS="$(echo "$CHERRYPICKS" | xargs -n1 | sort | uniq | xargs)"
167
168 function apply_cherrypicks() {
169 local CHECKOUT_DIR="$1"
170
171 [ "$CHERRYPICKS" == "" ] || echo "Applying cherrypicks"
172 pushd "$CHECKOUT_DIR"
173
174 # This function is always called on a sorted list of cherrypicks.
175 for CHERRY_REV in $CHERRYPICKS; do
176 echo "Cherry-picking r$CHERRY_REV into $CHECKOUT_DIR"
177
178 local PATCH_FILE="$(mktemp)"
179 svn diff -c $CHERRY_REV > "$PATCH_FILE"
180 svn patch "$PATCH_FILE"
181 rm "$PATCH_FILE"
182 done
183
184 popd
185 }
186
187 CLANG_BUILD_DIR=/tmp/clang-build
188 CLANG_INSTALL_DIR=/tmp/clang-install
189
190 mkdir "$CLANG_BUILD_DIR"
191
192 # Get the sources from svn.
193 echo "Checking out sources from svn"
194 mkdir "$CLANG_BUILD_DIR/src"
195 for LLVM_PROJECT in $LLVM_PROJECTS; do
196 if [ "$LLVM_PROJECT" == "clang" ]; then
197 SVN_PROJECT="cfe"
198 else
199 SVN_PROJECT="$LLVM_PROJECT"
200 fi
201
202 echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CLANG_BUILD_DIR/src/$LLVM_PROJECT"
203 svn co -q $SVN_REV_ARG \
204 "https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \
205 "$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
206
207 # We apply cherrypicks to all repositories regardless of whether the revision
208 # changes this repository or not. For repositories not affected by the
209 # cherrypick, applying the cherrypick is a no-op.
210 apply_cherrypicks "$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
211 done
212
213 if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
214 echo "Checking out https://llvm.org/svn/llvm-project/clang-tools-extra to $CLANG_BUILD_DIR/src/clang/tools/extra"
215 svn co -q $SVN_REV_ARG \
216 "https://llvm.org/svn/llvm-project/clang-tools-extra/$LLVM_BRANCH" \
217 "$CLANG_BUILD_DIR/src/clang/tools/extra"
218
219 apply_cherrypicks "$CLANG_BUILD_DIR/src/clang/tools/extra"
220 fi
221
222 CHECKSUMS_FILE="/tmp/checksums/checksums.txt"
223
224 if [ -f "$CHECKSUMS_FILE" ]; then
225 echo "Validating checksums for LLVM checkout..."
226 python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \
227 --partial --multi_dir "$CLANG_BUILD_DIR/src"
228 else
229 echo "Skipping checksumming checks..."
230 fi
231
232 mkdir "$CLANG_BUILD_DIR/build"
76 mkdir -p "$CLANG_BUILD_DIR/build"
23377 pushd "$CLANG_BUILD_DIR/build"
23478
23579 # Run the build as specified in the build arguments.
23680 echo "Running build"
23781 cmake -GNinja \
23882 -DCMAKE_INSTALL_PREFIX="$CLANG_INSTALL_DIR" \
239 -DLLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS" \
24083 $CMAKE_ARGS \
24184 "$CLANG_BUILD_DIR/src/llvm"
24285 ninja $CMAKE_INSTALL_TARGETS
24487 popd
24588
24689 # Cleanup.
247 rm -rf "$CLANG_BUILD_DIR"
90 rm -rf "$CLANG_BUILD_DIR/build"
24891
24992 echo "Done"
0 #!/usr/bin/env bash
1 #===- llvm/utils/docker/scripts/checkout.sh ---------------------===//
2 #
3 # The LLVM Compiler Infrastructure
4 #
5 # This file is distributed under the University of Illinois Open Source
6 # License. See LICENSE.TXT for details.
7 #
8 #===-----------------------------------------------------------------------===//
9
10 set -e
11
12 function show_usage() {
13 cat << EOF
14 Usage: checkout.sh [options]
15
16 Checkout svn sources into /tmp/clang-build/src. Used inside a docker container.
17
18 Available options:
19 -h|--help show this help message
20 -b|--branch svn branch to checkout, i.e. 'trunk',
21 'branches/release_40'
22 (default: 'trunk')
23 -r|--revision svn revision to checkout
24 -c|--cherrypick revision to cherry-pick. Can be specified multiple times.
25 Cherry-picks are performed in the sorted order using the
26 following command:
27 'svn patch <(svn diff -c \$rev)'.
28 -p|--llvm-project name of an svn project to checkout.
29 For clang, please use 'clang', not 'cfe'.
30 Project 'llvm' is always included and ignored, if
31 specified.
32 Can be specified multiple times.
33 EOF
34 }
35
36 LLVM_SVN_REV=""
37 CHERRYPICKS=""
38 LLVM_BRANCH=""
39 # We always checkout llvm
40 LLVM_PROJECTS="llvm"
41
42 function contains_project() {
43 local TARGET_PROJ="$1"
44 local PROJ
45 for PROJ in $LLVM_PROJECTS; do
46 if [ "$PROJ" == "$TARGET_PROJ" ]; then
47 return 0
48 fi
49 done
50 return 1
51 }
52
53 while [[ $# -gt 0 ]]; do
54 case "$1" in
55 -r|--revision)
56 shift
57 LLVM_SVN_REV="$1"
58 shift
59 ;;
60 -c|--cherrypick)
61 shift
62 CHERRYPICKS="$CHERRYPICKS $1"
63 shift
64 ;;
65 -b|--branch)
66 shift
67 LLVM_BRANCH="$1"
68 shift
69 ;;
70 -p|--llvm-project)
71 shift
72 PROJ="$1"
73 shift
74
75 if [ "$PROJ" == "cfe" ]; then
76 PROJ="clang"
77 fi
78
79 if ! contains_project "$PROJ" ; then
80 if [ "$PROJ" == "clang-tools-extra" ] && [ ! contains_project "clang" ]; then
81 echo "Project 'clang-tools-extra' specified before 'clang'. Adding 'clang' to a list of projects first."
82 LLVM_PROJECTS="$LLVM_PROJECTS clang"
83 fi
84 LLVM_PROJECTS="$LLVM_PROJECTS $PROJ"
85 else
86 echo "Project '$PROJ' is already enabled, ignoring extra occurrences."
87 fi
88 ;;
89 -h|--help)
90 show_usage
91 exit 0
92 ;;
93 *)
94 echo "Unknown option: $1"
95 exit 1
96 esac
97 done
98
99 if [ "$LLVM_BRANCH" == "" ]; then
100 LLVM_BRANCH="trunk"
101 fi
102
103 if [ "$LLVM_SVN_REV" != "" ]; then
104 SVN_REV_ARG="-r$LLVM_SVN_REV"
105 echo "Checking out svn revision r$LLVM_SVN_REV."
106 else
107 SVN_REV_ARG=""
108 echo "Checking out latest svn revision."
109 fi
110
111 # Sort cherrypicks and remove duplicates.
112 CHERRYPICKS="$(echo "$CHERRYPICKS" | xargs -n1 | sort | uniq | xargs)"
113
114 function apply_cherrypicks() {
115 local CHECKOUT_DIR="$1"
116
117 [ "$CHERRYPICKS" == "" ] || echo "Applying cherrypicks"
118 pushd "$CHECKOUT_DIR"
119
120 # This function is always called on a sorted list of cherrypicks.
121 for CHERRY_REV in $CHERRYPICKS; do
122 echo "Cherry-picking r$CHERRY_REV into $CHECKOUT_DIR"
123
124 local PATCH_FILE="$(mktemp)"
125 svn diff -c $CHERRY_REV > "$PATCH_FILE"
126 svn patch "$PATCH_FILE"
127 rm "$PATCH_FILE"
128 done
129
130 popd
131 }
132
133 CLANG_BUILD_DIR=/tmp/clang-build
134
135 # Get the sources from svn.
136 echo "Checking out sources from svn"
137 mkdir -p "$CLANG_BUILD_DIR/src"
138 for LLVM_PROJECT in $LLVM_PROJECTS; do
139 if [ "$LLVM_PROJECT" == "clang" ]; then
140 SVN_PROJECT="cfe"
141 else
142 SVN_PROJECT="$LLVM_PROJECT"
143 fi
144
145 if [ "$SVN_PROJECT" != "clang-tools-extra" ]; then
146 CHECKOUT_DIR="$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
147 else
148 CHECKOUT_DIR="$CLANG_BUILD_DIR/src/clang/tools/extra"
149 fi
150
151 echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CHECKOUT_DIR"
152 svn co -q $SVN_REV_ARG \
153 "https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \
154 "$CHECKOUT_DIR"
155
156 # We apply cherrypicks to all repositories regardless of whether the revision
157 # changes this repository or not. For repositories not affected by the
158 # cherrypick, applying the cherrypick is a no-op.
159 apply_cherrypicks "$CHECKOUT_DIR"
160 done
161
162 CHECKSUMS_FILE="/tmp/checksums/checksums.txt"
163
164 if [ -f "$CHECKSUMS_FILE" ]; then
165 echo "Validating checksums for LLVM checkout..."
166 python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \
167 --partial --multi_dir "$CLANG_BUILD_DIR/src"
168 else
169 echo "Skipping checksumming checks..."
170 fi
171
172 echo "Done"