llvm.org GIT mirror llvm / ae3278a
Added optional validation of svn sources to Dockerfiles. Summary: This commit also adds a script to compute sha256 hashes of llvm checkouts. Reviewers: klimek, mehdi_amini Reviewed By: klimek Subscribers: llvm-commits Differential Revision: https://reviews.llvm.org/D37099 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@313359 91177308-0d34-0410-b5e6-96231b3b80d8 Ilya Biryukov 2 years ago
7 changed file(s) with 335 addition(s) and 9 deletion(s). Raw diff Collapse all Expand all
3737 Can be specified multiple times.
3838 -i|--install-target name of a cmake install target to build and include in
3939 the resulting archive. Can be specified multiple times.
40 -c|--checksums name of a file, containing checksums of llvm checkout.
41 Script will fail if checksums of the checkout do not
42 match.
4043
4144 Required options: --source and --docker-repository, at least one
4245 --install-target.
6568 EOF
6669 }
6770
71 CHECKSUMS_FILE=""
6872 SEEN_INSTALL_TARGET=0
6973 while [[ $# -gt 0 ]]; do
7074 case "$1" in
9397 fi
9498 BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS $1 $2"
9599 shift 2
100 ;;
101 -c|--checksums)
102 shift
103 CHECKSUMS_FILE="$1"
104 shift
96105 ;;
97106 --)
98107 shift
140149 cp -r "$SOURCE_DIR/$IMAGE_SOURCE" "$BUILD_DIR/$IMAGE_SOURCE"
141150 cp -r "$SOURCE_DIR/scripts" "$BUILD_DIR/scripts"
142151
152 mkdir "$BUILD_DIR/checksums"
153 if [ "$CHECKSUMS_FILE" != "" ]; then
154 cp "$CHECKSUMS_FILE" "$BUILD_DIR/checksums/checksums.txt"
155 fi
156
143157 if [ "$DOCKER_TAG" != "" ]; then
144158 DOCKER_TAG=":$DOCKER_TAG"
145159 fi
1818 # Install compiler, python and subversion.
1919 RUN apt-get update && \
2020 apt-get install -y --no-install-recommends ca-certificates gnupg \
21 build-essential python2.7 wget subversion ninja-build && \
21 build-essential python wget subversion ninja-build && \
2222 rm -rf /var/lib/apt/lists/*
2323
2424 # Import public key required for verifying signature of cmake download.
3636 tar xzf cmake-3.7.2-Linux-x86_64.tar.gz -C /usr/local --strip-components=1 && \
3737 cd / && rm -rf /tmp/cmake-install
3838
39 ADD checksums /tmp/checksums
40 ADD scripts /tmp/scripts
41
3942 # Arguments passed to build_install_clang.sh.
4043 ARG buildscript_args
4144
4245 # Run the build. Results of the build will be available as /tmp/clang.tar.gz.
43 ADD scripts/build_install_llvm.sh /tmp
44 RUN /tmp/build_install_llvm.sh ${buildscript_args}
46 RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
1717 # FIXME: Install llvm/clang build dependencies. Including compiler to
1818 # build stage1, cmake, subversion, ninja, etc.
1919
20 # Arguments to pass to build_install_clang.sh.
20 ADD checksums /tmp/checksums
21 ADD scripts /tmp/scripts
22
23 # Arguments passed to build_install_clang.sh.
2124 ARG buildscript_args
2225
2326 # Run the build. Results of the build will be available as /tmp/clang.tar.gz.
24 ADD scripts/build_install_llvm.sh /tmp
25 RUN /tmp/build_install_llvm.sh ${buildscript_args}
27 RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
1616
1717 # Install llvm build dependencies.
1818 RUN apt-get update && \
19 apt-get install -y --no-install-recommends ca-certificates cmake python2.7 \
19 apt-get install -y --no-install-recommends ca-certificates cmake python \
2020 subversion ninja-build && \
2121 rm -rf /var/lib/apt/lists/*
2222
23 ADD checksums /tmp/checksums
24 ADD scripts /tmp/scripts
25
26 # Arguments passed to build_install_clang.sh.
27 ARG buildscript_args
28
2329 # Run the build. Results of the build will be available as /tmp/clang.tar.gz.
24 ADD scripts/build_install_llvm.sh /tmp
25 RUN /tmp/build_install_llvm.sh ${buildscript_args}
30 RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
180180 "$CLANG_BUILD_DIR/src/clang/tools/extra"
181181 fi
182182
183 CHECKSUMS_FILE="/tmp/checksums/checksums.txt"
184
185 if [ -f "$CHECKSUMS_FILE" ]; then
186 echo "Validating checksums for LLVM checkout..."
187 python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \
188 --partial --multi_dir "$CLANG_BUILD_DIR/src"
189 else
190 echo "Skipping checksumming checks..."
191 fi
192
183193 mkdir "$CLANG_BUILD_DIR/build"
184194 pushd "$CLANG_BUILD_DIR/build"
185195
0 #!/usr/bin/python
1 """ A small program to compute checksums of LLVM checkout.
2 """
3 from __future__ import absolute_import
4 from __future__ import division
5 from __future__ import print_function
6
7 import hashlib
8 import logging
9 import re
10 import sys
11 from argparse import ArgumentParser
12 from project_tree import *
13
14 SVN_DATES_REGEX = re.compile(r"\$(Date|LastChangedDate)[^\$]+\$")
15
16
17 def main():
18 parser = ArgumentParser()
19 parser.add_argument(
20 "-v", "--verbose", action="store_true", help="enable debug logging")
21 parser.add_argument(
22 "-c",
23 "--check",
24 metavar="reference_file",
25 help="read checksums from reference_file and " +
26 "check they match checksums of llvm_path.")
27 parser.add_argument(
28 "--partial",
29 action="store_true",
30 help="ignore projects from reference_file " +
31 "that are not checked out in llvm_path.")
32 parser.add_argument(
33 "--multi_dir",
34 action="store_true",
35 help="indicates llvm_path contains llvm, checked out " +
36 "into multiple directories, as opposed to a " +
37 "typical single source tree checkout.")
38 parser.add_argument("llvm_path")
39
40 args = parser.parse_args()
41 if args.check is not None:
42 with open(args.check, "r") as f:
43 reference_checksums = ReadLLVMChecksums(f)
44 else:
45 reference_checksums = None
46
47 if args.verbose:
48 logging.basicConfig(level=logging.DEBUG)
49
50 llvm_projects = CreateLLVMProjects(not args.multi_dir)
51 checksums = ComputeLLVMChecksums(args.llvm_path, llvm_projects)
52
53 if reference_checksums is None:
54 WriteLLVMChecksums(checksums, sys.stdout)
55 sys.exit(0)
56
57 if not ValidateChecksums(reference_checksums, checksums, args.partial):
58 sys.stdout.write("Checksums differ.\nNew checksums:\n")
59 WriteLLVMChecksums(checksums, sys.stdout)
60 sys.stdout.write("Reference checksums:\n")
61 WriteLLVMChecksums(reference_checksums, sys.stdout)
62 sys.exit(1)
63 else:
64 sys.stdout.write("Checksums match.")
65
66
67 def ComputeLLVMChecksums(root_path, projects):
68 """Compute checksums for LLVM sources checked out using svn.
69
70 Args:
71 root_path: a directory of llvm checkout.
72 projects: a list of LLVMProject instances, which describe checkout paths,
73 relative to root_path.
74
75 Returns:
76 A dict mapping from project name to project checksum.
77 """
78 hash_algo = hashlib.sha256
79
80 def collapse_svn_substitutions(contents):
81 # Replace svn substitutions for $Date$ and $LastChangedDate$.
82 # Unfortunately, these are locale-specific.
83 return SVN_DATES_REGEX.sub("$\1$", contents)
84
85 def read_and_collapse_svn_subsitutions(file_path):
86 with open(file_path, "rb") as f:
87 contents = f.read()
88 new_contents = collapse_svn_substitutions(contents)
89 if contents != new_contents:
90 logging.debug("Replaced svn keyword substitutions in %s", file_path)
91 logging.debug("\n\tBefore\n%s\n\tAfter\n%s", contents, new_contents)
92 return new_contents
93
94 project_checksums = dict()
95 # Hash each project.
96 for proj in projects:
97 project_root = os.path.join(root_path, proj.relpath)
98 if not os.path.exists(project_root):
99 logging.info("Folder %s doesn't exist, skipping project %s", proj.relpath,
100 proj.name)
101 continue
102
103 files = list()
104
105 def add_file_hash(file_path):
106 if os.path.islink(file_path) and not os.path.exists(file_path):
107 content = os.readlink(file_path)
108 else:
109 content = read_and_collapse_svn_subsitutions(file_path)
110 hasher = hash_algo()
111 hasher.update(content)
112 file_digest = hasher.hexdigest()
113 logging.debug("Checksum %s for file %s", file_digest, file_path)
114 files.append((file_path, file_digest))
115
116 logging.info("Computing checksum for %s", proj.name)
117 WalkProjectFiles(root_path, projects, proj, add_file_hash)
118
119 # Compute final checksum.
120 files.sort(key=lambda x: x[0])
121 hasher = hash_algo()
122 for file_path, file_digest in files:
123 file_path = os.path.relpath(file_path, project_root)
124 hasher.update(file_path)
125 hasher.update(file_digest)
126 project_checksums[proj.name] = hasher.hexdigest()
127 return project_checksums
128
129
130 def WriteLLVMChecksums(checksums, f):
131 """Writes checksums to a text file.
132
133 Args:
134 checksums: a dict mapping from project name to project checksum (result of
135 ComputeLLVMChecksums).
136 f: a file object to write into.
137 """
138
139 for proj in sorted(checksums.keys()):
140 f.write("{} {}\n".format(checksums[proj], proj))
141
142
143 def ReadLLVMChecksums(f):
144 """Reads checksums from a text file, produced by WriteLLVMChecksums.
145
146 Returns:
147 A dict, mapping from project name to project checksum.
148 """
149 checksums = {}
150 while True:
151 line = f.readline()
152 if line == "":
153 break
154 checksum, proj = line.split()
155 checksums[proj] = checksum
156 return checksums
157
158
159 def ValidateChecksums(reference_checksums,
160 new_checksums,
161 allow_missing_projects=False):
162 """Validates that reference_checksums and new_checksums match.
163
164 Args:
165 reference_checksums: a dict of reference checksums, mapping from a project
166 name to a project checksum.
167 new_checksums: a dict of checksums to be checked, mapping from a project
168 name to a project checksum.
169 allow_missing_projects:
170 When True, reference_checksums may contain more projects than
171 new_checksums. Projects missing from new_checksums are ignored.
172 When False, new_checksums and reference_checksums must contain checksums
173 for the same set of projects. If there is a project in
174 reference_checksums, missing from new_checksums, ValidateChecksums
175 will return False.
176
177 Returns:
178 True, if checksums match with regards to allow_missing_projects flag value.
179 False, otherwise.
180 """
181 if not allow_missing_projects:
182 if len(new_checksums) != len(reference_checksums):
183 return False
184
185 for proj, checksum in new_checksums.iteritems():
186 # We never computed a checksum for this project.
187 if proj not in reference_checksums:
188 return False
189 # Checksum did not match.
190 if reference_checksums[proj] != checksum:
191 return False
192
193 return True
194
195
196 if __name__ == "__main__":
197 main()
0 """Contains helper functions to compute checksums for LLVM checkouts.
1 """
2 from __future__ import absolute_import
3 from __future__ import division
4 from __future__ import print_function
5
6 import logging
7 import os
8 import os.path
9 import sys
10
11
12 class LLVMProject(object):
13 """An LLVM project with a descriptive name and a relative checkout path.
14 """
15
16 def __init__(self, name, relpath):
17 self.name = name
18 self.relpath = relpath
19
20 def is_subproject(self, other_project):
21 """ Check if self is checked out as a subdirectory of other_project.
22 """
23 return self.relpath.startswith(other_project.relpath)
24
25
26 def WalkProjectFiles(checkout_root, all_projects, project, visitor):
27 """ Walk over all files inside a project without recursing into subprojects, '.git' and '.svn' subfolders.
28
29 checkout_root: root of the LLVM checkout.
30 all_projects: projects in the LLVM checkout.
31 project: a project to walk the files of. Must be inside all_projects.
32 visitor: a function called on each visited file.
33 """
34 assert project in all_projects
35
36 ignored_paths = set()
37 for other_project in all_projects:
38 if other_project != project and other_project.is_subproject(project):
39 ignored_paths.add(os.path.join(checkout_root, other_project.relpath))
40
41 def raise_error(err):
42 raise err
43
44 project_root = os.path.join(checkout_root, project.relpath)
45 for root, dirs, files in os.walk(project_root, onerror=raise_error):
46 dirs[:] = [
47 d for d in dirs
48 if d != ".svn" and d != ".git" and
49 os.path.join(root, d) not in ignored_paths
50 ]
51 for f in files:
52 visitor(os.path.join(root, f))
53
54
55 def CreateLLVMProjects(single_tree_checkout):
56 """Returns a list of LLVMProject instances, describing relative paths of a typical LLVM checkout.
57
58 Args:
59 single_tree_checkout:
60 When True, relative paths for each project points to a typical single
61 source tree checkout.
62 When False, relative paths for each projects points to a separate
63 directory. However, clang-tools-extra is an exception, its relative path
64 will always be 'clang/tools/extra'.
65 """
66 # FIXME: cover all of llvm projects.
67
68 # Projects that reside inside 'projects/' in a single source tree checkout.
69 ORDINARY_PROJECTS = [
70 "compiler-rt", "dragonegg", "libcxx", "libcxxabi", "libunwind",
71 "parallel-libs", "test-suite"
72 ]
73 # Projects that reside inside 'tools/' in a single source tree checkout.
74 TOOLS_PROJECTS = ["clang", "lld", "lldb", "llgo"]
75
76 if single_tree_checkout:
77 projects = [LLVMProject("llvm", "")]
78 projects += [
79 LLVMProject(p, os.path.join("projects", p)) for p in ORDINARY_PROJECTS
80 ]
81 projects += [
82 LLVMProject(p, os.path.join("tools", p)) for p in TOOLS_PROJECTS
83 ]
84 projects.append(
85 LLVMProject("clang-tools-extra",
86 os.path.join("tools", "clang", "tools", "extra")))
87 else:
88 projects = [LLVMProject("llvm", "llvm")]
89 projects += [LLVMProject(p, p) for p in ORDINARY_PROJECTS]
90 projects += [LLVMProject(p, p) for p in TOOLS_PROJECTS]
91 projects.append(
92 LLVMProject("clang-tools-extra", os.path.join("clang", "tools",
93 "extra")))
94 return projects