#!/usr/bin/env python3 # # SPDX-License-Identifier: MIT # # SPDX-FileCopyrightText: 2023 EfficiOS, Inc. # # Author: Olivier Dion # # Auto-generate lttng-ust tracepoints for OpenMPI. # # Require: python-clang (libclang) import argparse import re import os import subprocess from string import Template import clang.cindex COMMON_PREFIX = None IGNORE = None PROVIDER = None # LTTNG_UST_TP_ARGS is limited to 10 arguments. Since we introduce two # arguments of our own (thread-id and local-id), the maximum is 8. # # If a function has more arguments than this limit, all arguments -- at the # exception of the IDs -- will be passed through a data structure instead. MAX_TP_ARGS_COUNT = 8 class EnumValue: def __init__(self, ev): self.name = ev.spelling self.value = ev.enum_value class EnumType: def __init__(self, en, name=None): self.name = name or en.spelling self.values = [EnumValue(ev) for ev in en.get_children()] class Typedef: def __init__(self, spelling, value): self.spelling = spelling self.value = value class ArgumentType: integer_set = { clang.cindex.TypeKind.UCHAR, clang.cindex.TypeKind.USHORT, clang.cindex.TypeKind.UINT, clang.cindex.TypeKind.ULONG, clang.cindex.TypeKind.ULONGLONG, clang.cindex.TypeKind.SHORT, clang.cindex.TypeKind.INT, clang.cindex.TypeKind.LONG, clang.cindex.TypeKind.LONGLONG, } float_set = { clang.cindex.TypeKind.FLOAT, clang.cindex.TypeKind.DOUBLE, } address_set = { clang.cindex.TypeKind.POINTER, clang.cindex.TypeKind.INCOMPLETEARRAY, } def __init__(self, arg, name_prefix="", expr_prefix=""): self.type = arg.type self.arg = arg self.const = "" self.name_prefix = name_prefix self.expr_prefix = expr_prefix if self.kind() == clang.cindex.TypeKind.POINTER: if self.type.get_pointee().is_const_qualified(): self.const = "const " elif self.type.is_const_qualified(): self.const = "const " def name(self): return self.arg.spelling def type_name(self): if self.kind() == clang.cindex.TypeKind.INCOMPLETEARRAY: return self.const + re.sub(r"\[[0-9]*\]", "*", self.type.spelling) if self.kind() == clang.cindex.TypeKind.POINTER: return f"{self.const}void *" return self.const + self.type.spelling def kind(self): return self.type.get_canonical().kind def to_lttng_field(self): if self.name() == "reserved": return "" elif self.kind() in ArgumentType.address_set: return f"lttng_ust_field_integer_hex(uintptr_t, {self.name_prefix}{self.name()}, (uintptr_t){self.expr_prefix}{self.name()})" elif self.kind() in ArgumentType.integer_set: return f"lttng_ust_field_integer({self.type_name()}, {self.name_prefix}{self.name()}, {self.expr_prefix}{self.name()})" elif self.kind() in ArgumentType.float_set: return f"lttng_ust_field_float({self.type_name()}, {self.name_prefix}{self.name()}, {self.expr_prefix}{self.name()})" elif self.kind() == clang.cindex.TypeKind.ENUM: enum_name = self.type_name().removeprefix("enum ") return f"lttng_ust_field_enum({PROVIDER}, {enum_name}, int, {self.name_prefix}{self.name()}, {self.expr_prefix}{self.name()})" elif self.kind() == clang.cindex.TypeKind.RECORD: return [ ArgumentType(field, f"{self.name()}_", f"{self.expr_prefix}{self.name()}.").to_lttng_field() for field in self.type.get_canonical().get_fields() ] else: raise Exception("Unsupported kind: %s" % self.kind()) class FunctionType: struct_tpl = Template(""" $name { $fields }; """) def __init__(self, fn): self.name = fn.spelling self.args = [ArgumentType(arg) for arg in fn.get_arguments()] self.fn = fn def tp_args(self): if len(self.args) == 0: return "" elif len(self.args) > MAX_TP_ARGS_COUNT: return ",\n " + f"{self.arguments_struct_name()} *, lttng_args" else: return ",\n " + ",\n ".join([f"{arg.type_name()}, {arg.name()}" for arg in self.args]) def tp_fields(self): if len(self.args) == 0: return "" elif len(self.args) > MAX_TP_ARGS_COUNT: packed_args = [ArgumentType(arg.arg, "", "lttng_args->") for arg in self.args] return "\n ".join(flatten([arg.to_lttng_field() for arg in packed_args])) else: return "\n ".join(flatten([arg.to_lttng_field() for arg in self.args])) def get_return_type_name(self): return self.fn.type.get_result().spelling def ctor_params(self): if len(self.args) == 0: return "" elif len(self.args) > MAX_TP_ARGS_COUNT: return ", <tng_args" else: return ", " + ", ".join(arg.name() for arg in self.args) def arguments_struct_variable(self): if len(self.args) > MAX_TP_ARGS_COUNT: return "%s lttng_args = {%s};" % (self.arguments_struct_name(), ", ".join([arg.name() for arg in self.args])) else: return f"/* {self.arguments_struct_name()} lttng_args */" def arguments_struct_name(self): return f"struct lttng_arguments_of_{self.name}" def arguments_struct(self): if len(self.args) > MAX_TP_ARGS_COUNT: return self.struct_tpl.substitute(name=self.arguments_struct_name(), fields="\n ".join([ f"{arg.type_name()} {arg.name()};" for arg in self.args ])) else: return "" def flatten(lst): new_lst = [] for e in lst: if isinstance(e, list): for e in flatten(e): new_lst.append(e) else: new_lst.append(e) return new_lst def list_function_declarations(root): return [ child for child in root.get_children() if child.kind == clang.cindex.CursorKind.FUNCTION_DECL ] def list_enum_declarations(root): return [ child for child in root.get_children() if child.kind == clang.cindex.CursorKind.ENUM_DECL ] def list_typedef_enums(root): enums = [] for child in root.get_children(): if child.kind == clang.cindex.CursorKind.TYPEDEF_DECL: maybe_enum = child.underlying_typedef_type.get_declaration() if maybe_enum.kind == clang.cindex.CursorKind.ENUM_DECL: enums.append(Typedef(child.spelling, maybe_enum)) return enums def search_header_in(name, paths): for path in paths.split(":"): for dirpath, _, files in os.walk(path, followlinks=True): for file in files: if file == name: return os.path.join(dirpath, file) return None def search_c_header(name): return search_header_in(name, os.environ["C_INCLUDE_PATH"]) def search_cxx_header(name): return search_header_in(name, os.environ["CPLUS_INCLUDE_PATH"]) def get_system_include_paths(): clang_args = ["clang", "-v", "-c", "-xc", "/dev/null"] paths = [] with subprocess.Popen(clang_args, stderr=subprocess.PIPE) as proc: start_sys_search = False for line in proc.stderr: if start_sys_search: if line == "End of search list.\n": break paths.append("-isystem") paths.append(line.strip()) elif line == "#include <...> search starts here:\n": start_sys_search = True return paths def parse_header(header_file, includes, defines, required_c_headers, required_cxx_headers): args = get_system_include_paths() if includes: for inc in includes: args.append("-I") args.append(inc) if defines: for d in defines: args.append("-D") args.append(d) for header in required_c_headers: found = search_c_header(header) if found: args.append("-I") args.append(os.path.dirname(found)) for header in required_cxx_headers: found = search_cxx_header(header) if found: args.append("-I") args.append(os.path.dirname(found)) tu = clang.cindex.Index.create().parse(header_file, args=args) for d in tu.diagnostics: print(d) return tu.cursor def list_functions(root): return [ FunctionType(fn) for fn in list_function_declarations(root) if fn.spelling.startswith(COMMON_PREFIX) and fn.spelling not in IGNORE ] def list_enums(root): enums = [ EnumType(en) for en in list_enum_declarations(root) if en.spelling.startswith(COMMON_PREFIX) and en.spelling not in IGNORE ] typedef_enums = [ EnumType(typedef.value, typedef.spelling) for typedef in list_typedef_enums(root) if typedef.spelling.startswith(COMMON_PREFIX) and typedef.spelling not in IGNORE ] return enums + typedef_enums def generate_tracepoint_definitions(function_declarations, enum_declarations, api_file, output_defs, output_interface, header_guard): defs_tpl = Template("""/* Auto-generated file! */ #undef LTTNG_UST_TRACEPOINT_PROVIDER #define LTTNG_UST_TRACEPOINT_PROVIDER $provider #undef LTTNG_UST_TRACEPOINT_INCLUDE #define LTTNG_UST_TRACEPOINT_INCLUDE "$output_defs" #if !defined($header_guard) #include <$api_file> $pass_by_struct #endif #if !defined($header_guard) || defined(LTTNG_UST_TRACEPOINT_HEADER_MULTI_READ) #define $header_guard #include $enum_definitions $tracepoint_definitions #endif /* $header_guard */ #include """) interface_tpl = Template("""/* Auto-generated file! */ #ifndef ${header_guard}_IMPL #define ${header_guard}_IMPL #include "${output_defs}" #endif /* ${header_guard}_IMPL */ """) tp_tpl = Template(""" LTTNG_UST_TRACEPOINT_EVENT( $provider, enter_$name, LTTNG_UST_TP_ARGS( uint64_t, lttng_thread_id, uint64_t, lttng_local_id$tp_args ), LTTNG_UST_TP_FIELDS( lttng_ust_field_integer(uint64_t, lttng_thread_id, lttng_thread_id) lttng_ust_field_integer(uint64_t, lttng_local_id, lttng_local_id) $tp_fields ) ) """) tp_ret_tpl = Template(""" LTTNG_UST_TRACEPOINT_EVENT( $provider, exit_$name, LTTNG_UST_TP_ARGS( uint64_t, lttng_thread_id, uint64_t, lttng_local_id, int, lttng_has_ret, $ret_type, lttng_ret ), LTTNG_UST_TP_FIELDS( lttng_ust_field_integer(uint64_t, lttng_thread_id, lttng_thread_id) lttng_ust_field_integer(uint64_t, lttng_local_id, lttng_local_id) lttng_ust_field_integer(int, lttng_has_ret, lttng_has_ret) lttng_ust_field_integer($ret_type, lttng_ret, lttng_ret) ) ) """) tp_void_tpl = Template(""" LTTNG_UST_TRACEPOINT_EVENT( $provider, exit_$name, LTTNG_UST_TP_ARGS( uint64_t, lttng_thread_id, uint64_t, lttng_local_id, int, lttng_has_ret ), LTTNG_UST_TP_FIELDS( lttng_ust_field_integer(uint64_t, lttng_thread_id, lttng_thread_id) lttng_ust_field_integer(uint64_t, lttng_local_id, lttng_local_id) lttng_ust_field_integer(int, lttng_has_ret, lttng_has_ret) ) ) """) enum_tpl = Template(""" LTTNG_UST_TRACEPOINT_ENUM($provider, $name, LTTNG_UST_TP_ENUM_VALUES( $values ) ) """) with open(output_defs, "w") as output: definitions = [] for fn in function_declarations: ret_type = fn.get_return_type_name() definitions.append(tp_tpl.substitute(provider=PROVIDER, name=fn.name, tp_args=fn.tp_args(), tp_fields=fn.tp_fields())) if ret_type == "void": tpl = tp_void_tpl else: tpl = tp_ret_tpl definitions.append(tpl.substitute(provider=PROVIDER, name=fn.name, ret_type=ret_type)) tracepoint_definitions = "\n".join(definitions) enum_definitions = "\n".join([ enum_tpl.substitute(provider=PROVIDER, name=en.name, values="\n ".join([f'lttng_ust_field_enum_value("{ev.name}", {ev.value})' for ev in en.values])) for en in enum_declarations ]) output.write(defs_tpl.substitute(provider=PROVIDER, output_defs=output_defs, header_guard=header_guard, tracepoint_definitions=tracepoint_definitions, enum_definitions=enum_definitions, api_file=api_file, pass_by_struct="".join([fn.arguments_struct() for fn in function_declarations]))) with open(output_interface, "w") as output: output.write(interface_tpl.substitute(header_guard=header_guard, output_defs=output_defs,)) def generate_tracepoint_classes(function_declarations, api_file, output_path, header_guard, namespace): global_tpl = Template("""/* Auto-generated file! */ #include #include #include <$api_file> namespace $namespace { struct unique_id { uint64_t thread_id; uint64_t local_id; }; class id_generator { static std::atomic _thread_counter; uint64_t _thread_id; uint64_t _local_id; public: id_generator() { _thread_id = _thread_counter++; _local_id = 0; } unique_id next_id() { return { .thread_id = _thread_id, .local_id = _local_id++, }; } }; extern thread_local id_generator generator; template class base_api_object { protected: unique_id _id; int _has_ret; RetType _ret; public: void generate_id() { _id = generator.next_id(); } void mark_return(RetType ret) { _ret = ret; _has_ret = 1; } }; class base_api_object_void { protected: unique_id _id; int _has_ret; public: void generate_id() { _id = generator.next_id(); } void mark_return(void) { _has_ret = 1; } }; $classes }; """) cls_ret_tpl = Template(""" class api_object_$fn_name : public base_api_object<$ret_type> { public: api_object_$fn_name($ctor_type_params) { if (lttng_ust_tracepoint_enabled($provider, enter_$fn_name)) { generate_id(); $pass_by_struct lttng_ust_do_tracepoint($provider, enter_$fn_name, _id.thread_id, _id.local_id$ctor_params); } } ~api_object_$fn_name() { if (lttng_ust_tracepoint_enabled($provider, exit_$fn_name)) { lttng_ust_do_tracepoint($provider, exit_$fn_name, _id.thread_id, _id.local_id, _has_ret, _ret); } } }; """) cls_void_tpl = Template(""" class api_object_$fn_name : public base_api_object_void { public: api_object_$fn_name($ctor_type_params) { if (lttng_ust_tracepoint_enabled($provider, enter_$fn_name)) { generate_id(); $pass_by_struct lttng_ust_do_tracepoint($provider, enter_$fn_name, _id.thread_id, _id.local_id$ctor_params); } } ~api_object_$fn_name() { if (lttng_ust_tracepoint_enabled($provider, exit_$fn_name)) { lttng_ust_do_tracepoint($provider, exit_$fn_name, _id.thread_id, _id.local_id, _has_ret); } } }; """) with open(output_path, "w") as output: classes = [] for fn in function_declarations: ret_type = fn.get_return_type_name() if ret_type == "void": cls_tpl = cls_void_tpl else: cls_tpl = cls_ret_tpl classes.append(cls_tpl.substitute(provider=PROVIDER, fn_name=fn.name, pass_by_struct=fn.arguments_struct_variable(), ctor_type_params=", ".join([f"{arg.type_name()} {arg.name()}" for arg in fn.args]), ctor_params=fn.ctor_params(), ret_type=ret_type)) output.write(global_tpl.substitute(api_file=api_file, namespace=namespace, classes="".join(classes))) def generate_tracepoint_emulated_classes(function_declarations, api_file, output_path, header_guard, namespace): global_tpl = Template("""/* Auto-generated file! */ #include #include <$api_file> #define ${NAMESPACE}_CAT_PRIMITIVE(A, B) A##B #define ${NAMESPACE}_CAT(A, B) ${NAMESPACE}_CAT_PRIMITIVE(A, B) struct ${namespace}_unique_id { uint64_t thread_id; uint64_t local_id; }; struct ${namespace}_id_generator { uint64_t thread_id; uint64_t local_id; int initialized; }; extern uint64_t ${namespace}_id_generator_thread_counter; extern _Thread_local struct ${namespace}_id_generator ${namespace}_generator; #define ${namespace}_unlikely(x) __builtin_expect(!!(x), 0) static inline void ${namespace}_id_generator_next_id(struct ${namespace}_unique_id *id) { if (${namespace}_unlikely(!${namespace}_generator.initialized)) { ${namespace}_generator.thread_id = __atomic_fetch_add(&${namespace}_id_generator_thread_counter, 1, __ATOMIC_RELAXED); ${namespace}_generator.initialized = 1; } id->thread_id = ${namespace}_generator.thread_id; id->local_id = ${namespace}_generator.local_id++; } #define ${NAMESPACE}_API_OBJECT_NAME ${namespace}_api_object #define ${NAMESPACE}_MAKE_API_OBJECT(name, ...) \\ struct ${NAMESPACE}_CAT(${namespace}_api_state_, name) __attribute__((cleanup(${NAMESPACE}_CAT(exit_, name)))) \\ ${NAMESPACE}_API_OBJECT_NAME = { 0 }; \\ ${NAMESPACE}_CAT(enter_, name)(&${NAMESPACE}_API_OBJECT_NAME, ##__VA_ARGS__); \\ do { } while (0) #define ${NAMESPACE}_MARK_RETURN_API_OBJECT(code) \\ ({ \\ ${NAMESPACE}_API_OBJECT_NAME.ret = code; \\ ${NAMESPACE}_API_OBJECT_NAME.has_ret = 1; \\ }) ${classes} """) cls_tpl = Template(""" struct ${namespace}_api_state_${fn_name} { struct ${namespace}_unique_id id; int has_ret; $ret_type ret; }; static inline void enter_${fn_name}(${ctor_type_params}) { if (${namespace}_ust_tracepoint_enabled(${provider}, enter_${fn_name})) { ${namespace}_id_generator_next_id(<tng_state->id); ${pass_by_struct} ${namespace}_ust_do_tracepoint($provider, enter_${fn_name}, lttng_state->id.thread_id, lttng_state->id.local_id${ctor_params}); } } static inline void exit_${fn_name}(const struct ${namespace}_api_state_${fn_name} *lttng_state) { lttng_ust_tracepoint(${provider}, exit_${fn_name}, lttng_state->id.thread_id, lttng_state->id.local_id, lttng_state->has_ret, lttng_state->ret); } """) with open(output_path, "w") as output: output.write(global_tpl.substitute(api_file=api_file, namespace=namespace, NAMESPACE=namespace.upper(), classes="".join([ cls_tpl.substitute(provider=PROVIDER, fn_name=fn.name, pass_by_struct=fn.arguments_struct_variable(), ctor_params=fn.ctor_params(), ctor_type_params=", ".join([f"struct {namespace}_api_state_{fn.name} *lttng_state"] + [f"{arg.type_name()} {arg.name()}" for arg in fn.args]), namespace=namespace, NAMESPACE=namespace.upper(), ret_type=fn.get_return_type_name()) for fn in function_declarations ]))) def generate_tracepoint_implementations(guard, namespace, defs, impls): tpl = Template("""/* Auto-generated !*/ #ifdef ${guard} #define LTTNG_UST_TRACEPOINT_CREATE_PROBES #define LTTNG_UST_TRACEPOINT_DEFINE #include "${defs}" #endif /* ${guard} */ """) with open(impls, "w") as output: output.write(tpl.substitute(guard=guard, defs=defs)) def generate_tracepoint_states(states_guard, namespace, interface, classes, states, emulated_classes): if emulated_classes: body_tpl = Template(""" uint64_t ${namespace}_id_generator_thread_counter = 0; _Thread_local struct ${namespace}_id_generator ${namespace}_generator; """) else: body_tpl = Template(""" #include namespace ${namespace} { std::atomic id_generator::_thread_counter{0}; thread_local id_generator generator; }; """) tpl = Template("""/* Auto-generated! */ #ifdef ${states_guard} #include "${interface}" #include "${classes}" $body #endif """) with open(states, "w") as output: output.write(tpl.substitute(states_guard=states_guard, interface=interface, classes=classes, body=body_tpl.substitute(namespace=namespace))) def main(): global COMMON_PREFIX global IGNORE global PROVIDER if os.getenv("LTTNG_UST_MPI_CLANG_LIBRARY_FILE", None) is not None: clang.cindex.Config.set_library_file(os.getenv("LTTNG_UST_MPI_CLANG_LIBRARY_FILE")) parser = argparse.ArgumentParser(prog="lttng-ust-autogen-api", description="Generate LTTng classes and tracepoint definitions") parser.add_argument("api", help="Header file that has the API") parser.add_argument("defs", help="Path to tracepoint definitions") parser.add_argument("interface", help="Path to tracepoints interfaces") parser.add_argument("classes", help="Path to tracepoint classes") parser.add_argument("impl", help="Path to tracepoint implementations") parser.add_argument("states", help="Path to states") parser.add_argument("--provider", dest="provider", metavar="PROVIDER", default="noprovider", help="Tracepoints PROVIDER") parser.add_argument("--common-prefix", dest="common_prefix", metavar="PREFIX", default="", help="Common PREFIX of API functions (C namespace)") parser.add_argument("-I", action="append", metavar="DIR", dest="includes", help="Add DIR to list of directories to include") parser.add_argument("-D", action="append", metavar="DEFINITION", dest="defines", help="Add DEFINITION to list of definitions") parser.add_argument("--tp-guard", dest="tp_guard", metavar="GUARD", default="LTTNG_TRACEPOINT_DEF_H", help="Use GUARD as header guard for tracepoint definitions") parser.add_argument("--classes-guard", dest="classes_guard", metavar="GUARD", default="LTTNG_TRACEPOINT_CLASSES_HPP", help="Use GUARD as header guard for classes definitions") parser.add_argument("--impl-guard", dest="impl_guard", metavar="GUARD", default="ENABLE_LTTNG_TRACEPOINTS", help="Use GUARD around implementations") parser.add_argument("--states-guard", dest="states_guard", metavar="GUARD", default="ENABLE_LTTNG_TRACEPOINTS", help="Use GUARD around states") parser.add_argument("--emulated-classes", dest="emulated_classes", action="store_true", default=False, help="Emulate C++ classes") parser.add_argument("--namespace", dest="namespace", metavar="NAMESPACE", default="lttng", help="Generate classes in NAMESPACE") parser.add_argument("--ignore", dest="ignore", metavar="FUNCTION", action="append", default=[], help="Ignore FUNCTION") parser.add_argument("--c-header", dest="required_c_headers", metavar="HEADER", action="append", default=[], help="Search for HEADER in C_INCLUDE_PATH and add its directory to search path") parser.add_argument("--cxx-header", dest="required_cxx_headers", metavar="HEADER", action="append", default=[], help="Search for HEADER in CPLUS_INCLUDE_PATH add its directory to search path") args = parser.parse_args() PROVIDER = args.provider COMMON_PREFIX = args.common_prefix IGNORE = set(args.ignore) root = parse_header(args.api, args.includes, args.defines, args.required_c_headers, args.required_cxx_headers) function_declarations = list_functions(root) enum_declarations = list_enums(root) generate_tracepoint_definitions(function_declarations, enum_declarations, args.api, args.defs, args.interface, args.tp_guard) if args.emulated_classes: generate_tracepoint_emulated_classes(function_declarations, args.api, args.classes, args.classes_guard, args.namespace) else: generate_tracepoint_classes(function_declarations, args.api, args.classes, args.classes_guard, args.namespace) generate_tracepoint_implementations(args.impl_guard, args.namespace, args.interface, args.impl) generate_tracepoint_states(args.states_guard, args.namespace, args.interface, args.classes, args.states, args.emulated_classes) if __name__ == "__main__": main()