Commit | Line | Data |
---|---|---|
de9b991b JR |
1 | import os |
2 | import shutil | |
3 | import git | |
4 | import subprocess | |
5 | import logging | |
cb87ff89 JR |
6 | import lttng_ivc.settings as Settings |
7 | ||
8 | from lttng_ivc.utils.utils import sha256_checksum | |
816f8cc1 | 9 | from lttng_ivc.utils.utils import find_dir, find_file |
de9b991b | 10 | |
fe7b987e | 11 | _logger = logging.getLogger('project') |
de9b991b JR |
12 | |
13 | class Project(object): | |
14 | ||
15 | def __init__(self, label, git_path, sha1, tmpdir): | |
16 | self.label = label | |
17 | self.git_path = git_path | |
18 | self.sha1 = sha1 | |
19 | ||
20 | """ Custom configure flags in the for of ['-x', 'arg']""" | |
21 | self.custom_configure_flags = [] | |
22 | ccache = shutil.which("ccache") | |
23 | if ccache is not None: | |
24 | self.custom_configure_flags.append("CC={} gcc".format(ccache)) | |
25 | self.custom_configure_flags.append("CXX={} g++".format(ccache)) | |
97d494b0 | 26 | self.custom_configure_flags.append("CFLAGS=-g -O0".format(ccache)) |
de9b991b JR |
27 | |
28 | """ A collection of Project dependencies """ | |
fe7b987e | 29 | self.dependencies = {} |
cb87ff89 | 30 | # used for project cache and pickle validation |
fe7b987e | 31 | self._immutable = False |
cb87ff89 | 32 | self._py_file_checksum = sha256_checksum(Settings.project_py_file_location) |
de9b991b JR |
33 | |
34 | # State | |
de9b991b JR |
35 | self.isBuilt = False |
36 | self.isConfigured = False | |
37 | self.isInstalled = False | |
ab63b97e | 38 | self.skip = False |
de9b991b | 39 | |
fe7b987e JR |
40 | self.basedir = tmpdir |
41 | self.log_path = os.path.join(tmpdir, "log") | |
42 | self.source_path = os.path.join(tmpdir, "source") | |
43 | self.installation_path = os.path.join(tmpdir, "install") | |
44 | ||
97d494b0 JR |
45 | if os.path.isdir(self.basedir): |
46 | # Perform cleanup since it should not happen | |
47 | shutil.rmtree(self.basedir) | |
48 | ||
fe7b987e | 49 | os.makedirs(self.log_path) |
de9b991b JR |
50 | os.makedirs(self.source_path) |
51 | os.makedirs(self.installation_path) | |
de9b991b JR |
52 | |
53 | self.special_env_variables = {} | |
54 | ||
55 | # Init the repo for work | |
56 | self.checkout() | |
57 | self.bootstrap() | |
58 | ||
18aedaf9 JR |
59 | def add_special_env_variable(self, key, value): |
60 | if key in self.special_env_variables: | |
61 | _logger.warning("{} Special var {} is already defined".format( | |
62 | self.label, key)) | |
63 | raise Exception("Multiple definition of a special environment variable") | |
64 | self.special_env_variables[key] = value | |
65 | ||
de9b991b | 66 | def get_cppflags(self): |
18aedaf9 JR |
67 | cppflags = ["-I{}/include".format(self.installation_path)] |
68 | for key, dep in self.dependencies.items(): | |
69 | cppflags.append(dep.get_cppflags()) | |
70 | ||
71 | return " ".join(cppflags) | |
de9b991b JR |
72 | |
73 | def get_ldflags(self): | |
18aedaf9 JR |
74 | ldflags = ["-L{}/lib".format(self.installation_path)] |
75 | for key, dep in self.dependencies.items(): | |
76 | ldflags.append(dep.get_ldflags()) | |
77 | return " ".join(ldflags) | |
de9b991b JR |
78 | |
79 | def get_ld_library_path(self): | |
18aedaf9 JR |
80 | library_path = ["{}/lib".format(self.installation_path)] |
81 | for key, dep in self.dependencies.items(): | |
82 | library_path.append(dep.get_ld_library_path()) | |
83 | return ":".join(library_path) | |
84 | ||
85 | def get_bin_path(self): | |
86 | bin_path = ["{}/bin".format(self.installation_path)] | |
87 | for key, dep in self.dependencies.items(): | |
88 | bin_path.append(dep.get_bin_path()) | |
89 | return ":".join(bin_path) | |
de9b991b JR |
90 | |
91 | def get_env(self): | |
92 | """Modify environment to reflect dependency""" | |
18aedaf9 JR |
93 | env_var = {"CPPFLAGS": (self.get_cppflags(), " "), |
94 | "LDFLAGS": (self.get_ldflags(), " "), | |
95 | "LD_LIBRARY_PATH": (self.get_ld_library_path(), ":"), | |
96 | } | |
de9b991b JR |
97 | |
98 | env = os.environ.copy() | |
99 | ||
100 | for var, value in self.special_env_variables.items(): | |
101 | if var in env: | |
de9b991b | 102 | # Raise for now since no special cases is known |
18aedaf9 JR |
103 | _logger.warning("{} Special var {} is already defined".format( |
104 | self.label, var)) | |
de9b991b JR |
105 | raise Exception("Multiple definition of a special environment variable") |
106 | else: | |
107 | env[var] = value | |
108 | ||
fe7b987e | 109 | for key, dep in self.dependencies.items(): |
de9b991b | 110 | # Extra space just in case |
de9b991b JR |
111 | for var, value in dep.special_env_variables.items(): |
112 | if var in env: | |
de9b991b | 113 | # Raise for now since no special cases is known |
18aedaf9 JR |
114 | _logger.warning("{} Special var {} is already defined".format( |
115 | self.label, var)) | |
de9b991b JR |
116 | raise Exception("Multiple definition of a special environment variable") |
117 | else: | |
118 | env[var] = value | |
119 | ||
18aedaf9 JR |
120 | for var, (value, delimiter) in env_var.items(): |
121 | tmp = [value] | |
122 | if var in env: | |
123 | tmp.append(env[var]) | |
124 | env[var] = delimiter.join(tmp) | |
125 | ||
de9b991b JR |
126 | return env |
127 | ||
128 | def autobuild(self): | |
129 | """ | |
130 | Perform the bootstrap, configuration, build and install the | |
131 | project. Build dependencies if not already built | |
132 | """ | |
fe7b987e JR |
133 | if (self.isConfigured and self.isBuilt and self.isInstalled): |
134 | return | |
135 | ||
136 | if self._immutable: | |
137 | raise Exception("Object is immutable. Illegal autobuild") | |
138 | ||
139 | for key, dep in self.dependencies.items(): | |
de9b991b JR |
140 | dep.autobuild() |
141 | ||
fe7b987e | 142 | if self.isConfigured ^ self.isBuilt ^ self.isInstalled: |
de9b991b JR |
143 | raise Exception("Project steps where manually triggered. Can't autobuild") |
144 | ||
97d494b0 JR |
145 | _logger.debug("{} Autobuild configure".format(self.label)) |
146 | try: | |
147 | self.configure() | |
148 | except subprocess.CalledProcessError as e: | |
149 | _logger.error("{} Configure failed. See {} for more details.".format(self.label, self.log_path)) | |
150 | raise e | |
151 | ||
152 | _logger.debug("{} Autobuild build".format(self.label)) | |
153 | try: | |
154 | self.build() | |
155 | except subprocess.CalledProcessError as e: | |
156 | _logger.error("{} Build failed. See {} for more details.".format(self.label, self.log_path)) | |
157 | raise e | |
158 | ||
159 | _logger.debug("{} Autobuild install".format(self.label)) | |
160 | try: | |
161 | self.install() | |
162 | except subprocess.CalledProcessError as e: | |
163 | _logger.error("{} Install failed. See {} for more details.".format(self.label, self.log_path)) | |
164 | raise e | |
de9b991b JR |
165 | |
166 | def checkout(self): | |
fe7b987e JR |
167 | if self._immutable: |
168 | raise Exception("Object is immutable. Illegal checkout") | |
169 | ||
de9b991b JR |
170 | repo = git.Repo.clone_from(self.git_path, self.source_path) |
171 | commit = repo.commit(self.sha1) | |
172 | repo.head.reference = commit | |
173 | assert repo.head.is_detached | |
174 | repo.head.reset(index=True, working_tree=True) | |
175 | ||
176 | def bootstrap(self): | |
177 | """ | |
178 | Bootstap the project. Raise subprocess.CalledProcessError on | |
179 | bootstrap error. | |
180 | """ | |
fe7b987e JR |
181 | if self._immutable: |
182 | raise Exception("Object is immutable. Illegal bootstrap") | |
183 | ||
184 | out = os.path.join(self.log_path, "bootstrap.out") | |
185 | err = os.path.join(self.log_path, "bootstrap.err") | |
186 | ||
de9b991b | 187 | os.chdir(self.source_path) |
fe7b987e JR |
188 | with open(out, 'w') as stdout, open(err, 'w') as stderr: |
189 | p = subprocess.run(['./bootstrap'], stdout=stdout, stderr=stderr) | |
de9b991b JR |
190 | p.check_returncode() |
191 | return p | |
192 | ||
193 | def configure(self): | |
194 | """ | |
195 | Configure the project. | |
196 | Raises subprocess.CalledProcessError on configure error | |
197 | """ | |
fe7b987e JR |
198 | if self._immutable: |
199 | raise Exception("Object is immutable. Illegal configure") | |
200 | ||
de9b991b | 201 | # Check that all our dependencies were actually installed |
fe7b987e | 202 | for key, dep in self.dependencies.items(): |
de9b991b JR |
203 | if not dep.isInstalled: |
204 | # TODO: Custom exception here Dependency Error | |
205 | raise Exception("Dependency project flagged as not installed") | |
206 | ||
fe7b987e JR |
207 | out = os.path.join(self.log_path, "configure.out") |
208 | err = os.path.join(self.log_path, "configure.err") | |
209 | ||
de9b991b JR |
210 | os.chdir(self.source_path) |
211 | args = ['./configure'] | |
212 | prefix = '--prefix={}'.format(self.installation_path) | |
213 | args.append(prefix) | |
214 | args.extend(self.custom_configure_flags) | |
215 | ||
216 | # TODO: log output and add INFO log point | |
fe7b987e JR |
217 | with open(out, 'w') as stdout, open(err, 'w') as stderr: |
218 | p = subprocess.run(args, env=self.get_env(), stdout=stdout, | |
219 | stderr=stderr) | |
de9b991b JR |
220 | p.check_returncode() |
221 | self.isConfigured = True | |
222 | return p | |
223 | ||
224 | def build(self): | |
225 | """ | |
226 | Build the project. Raise subprocess.CalledProcessError on build | |
227 | error. | |
228 | """ | |
fe7b987e JR |
229 | if self._immutable: |
230 | raise Exception("Object is immutable. Illegal build") | |
231 | ||
232 | out = os.path.join(self.log_path, "build.out") | |
233 | err = os.path.join(self.log_path, "build.err") | |
234 | ||
de9b991b JR |
235 | os.chdir(self.source_path) |
236 | args = ['make'] | |
237 | env = self.get_env() | |
de9b991b JR |
238 | |
239 | # Number of usable cpu | |
240 | # https://docs.python.org/3/library/os.html#os.cpu_count | |
241 | num_cpu = str(len(os.sched_getaffinity(0))) | |
242 | args.append('-j') | |
243 | args.append(num_cpu) | |
97d494b0 | 244 | args.append('V=1') |
de9b991b JR |
245 | |
246 | # TODO: log output and add INFO log point with args | |
fe7b987e JR |
247 | with open(out, 'w') as stdout, open(err, 'w') as stderr: |
248 | p = subprocess.run(args, env=env, stdout=stdout, | |
249 | stderr=stderr) | |
de9b991b JR |
250 | p.check_returncode() |
251 | self.isBuilt = True | |
252 | return p | |
253 | ||
254 | def install(self): | |
255 | """ | |
256 | Install the project. Raise subprocess.CalledProcessError on | |
257 | bootstrap error | |
258 | """ | |
fe7b987e JR |
259 | if self._immutable: |
260 | raise Exception("Object is immutable. Illegal install") | |
261 | ||
97d494b0 JR |
262 | out = os.path.join(self.log_path, "install.out") |
263 | err = os.path.join(self.log_path, "install.err") | |
fe7b987e | 264 | |
de9b991b JR |
265 | os.chdir(self.source_path) |
266 | args = ['make', 'install'] | |
267 | ||
268 | # TODO: log output and add INFO log point | |
fe7b987e JR |
269 | with open(out, 'w') as stdout, open(err, 'w') as stderr: |
270 | p = subprocess.run(args, env=self.get_env(), stdout=stdout, | |
271 | stderr=stderr) | |
de9b991b JR |
272 | p.check_returncode() |
273 | self.isInstalled = True | |
274 | return p | |
275 | ||
276 | def cleanup(self): | |
277 | if os.path.exists(self.source_path): | |
278 | shutil.rmtree(self.source_path) | |
279 | if os.path.exists(self.installation_path): | |
280 | shutil.rmtree(self.installation_path) | |
281 | ||
282 | ||
283 | class Lttng_modules(Project): | |
97d494b0 JR |
284 | def __init__(self, label, git_path, sha1, tmpdir): |
285 | super(Lttng_modules, self).__init__(label=label, git_path=git_path, | |
286 | sha1=sha1, tmpdir=tmpdir) | |
f0acf3f3 | 287 | self.add_special_env_variable("MODPROBE_OPTIONS","-d {}".format(self.installation_path)) |
97d494b0 | 288 | |
de9b991b JR |
289 | def bootstrap(self): |
290 | pass | |
291 | ||
292 | def configure(self): | |
293 | pass | |
294 | ||
295 | def install(self): | |
fe7b987e JR |
296 | if self._immutable: |
297 | raise Exception("Object is immutable. Illegal install") | |
de9b991b JR |
298 | os.chdir(self.source_path) |
299 | args = ['make', 'INSTALL_MOD_PATH={}'.format(self.installation_path), | |
300 | 'modules_install'] | |
301 | p = subprocess.run(args, env=self.get_env(), stdout=subprocess.PIPE, | |
302 | stderr=subprocess.PIPE) | |
303 | p.check_returncode() | |
304 | ||
305 | # Perform a local depmod | |
306 | args = ['depmod', '-b', self.installation_path] | |
307 | p = subprocess.run(args, env=self.get_env()) | |
308 | p.check_returncode() | |
309 | self.isInstalled = True | |
310 | ||
311 | ||
312 | class Lttng_ust(Project): | |
313 | def __init__(self, label, git_path, sha1, tmpdir): | |
314 | super(Lttng_ust, self).__init__(label=label, git_path=git_path, | |
315 | sha1=sha1, tmpdir=tmpdir) | |
316 | self.custom_configure_flags.extend(['--disable-man-pages']) | |
e5cbfcca JR |
317 | self.custom_configure_flags.extend(['--enable-python-agent']) |
318 | self.custom_configure_flags.extend(['--enable-java-agent-jul']) | |
319 | ||
74eb7096 JR |
320 | jul_path = os.path.join(self.installation_path, |
321 | "share/java/liblttng-ust-agent.jar") | |
322 | classpath = ":".join([jul_path, Settings.log4j_class_path, '.']) | |
323 | self.add_special_env_variable("CLASSPATH", classpath) | |
de9b991b | 324 | |
2094d672 JR |
325 | def install(self): |
326 | super(Lttng_ust, self).install() | |
327 | python_path = find_dir(self.installation_path, "lttngust") | |
328 | if python_path: | |
329 | # Fetch the parent of lttngust folder | |
330 | python_path = os.path.dirname(python_path) | |
331 | self.add_special_env_variable("PYTHONPATH", python_path) | |
332 | ||
333 | ||
de9b991b JR |
334 | |
335 | class Lttng_tools(Project): | |
18aedaf9 JR |
336 | def __init__(self, label, git_path, sha1, tmpdir): |
337 | super(Lttng_tools, self).__init__(label=label, git_path=git_path, | |
338 | sha1=sha1, tmpdir=tmpdir) | |
339 | self.add_special_env_variable("LTTNG_SESSION_CONFIG_XSD_PATH", | |
340 | os.path.join(self.installation_path, "share/xml/lttng/")) | |
de9b991b | 341 | |
816f8cc1 JR |
342 | # Find the mi xsd |
343 | for xsd in Settings.mi_xsd_file_name: | |
344 | mi = find_file(self.source_path, xsd) | |
345 | if mi: | |
346 | break | |
347 | if not mi: | |
348 | raise Exception("MI xsd not found") | |
349 | self.mi_xsd = mi | |
350 | ||
de9b991b JR |
351 | |
352 | class Babeltrace(Project): | |
353 | pass |