diff --git a/ctypeslib/clang2py.py b/ctypeslib/clang2py.py index ee674bd..70a6740 100755 --- a/ctypeslib/clang2py.py +++ b/ctypeslib/clang2py.py @@ -241,6 +241,11 @@ def windows_dlls(option, opt, value, parser): default=False, help="Parse object in sources files only. Ignore includes") + parser.add_argument("-X", "--force-exclude-includes", + action="store_true", + default=False, + help="Forcibly disable generation for all object outside sources files.") + parser.add_argument("--show-ids", dest="showIDs", help="Don't compute cursor IDs (very slow)", default=False) @@ -295,8 +300,12 @@ def windows_dlls(option, opt, value, parser): # Preload libraries # [Library(name, mode=RTLD_GLOBAL) for name in options.preload] - - translate_files(inputs.files, outputs.stream, cfg) + try: + translate_files(inputs.files, outputs.stream, cfg) + except: + # return non-zero exit status in case of an unhandled exception + traceback.print_exc() + sys.exit(1) return 0 diff --git a/ctypeslib/codegen/codegenerator.py b/ctypeslib/codegen/codegenerator.py index 0291f72..10bcba4 100644 --- a/ctypeslib/codegen/codegenerator.py +++ b/ctypeslib/codegen/codegenerator.py @@ -32,7 +32,10 @@ def __init__(self, output, cfg): self.stream = StringIO() self.imports = StringIO() self.cfg = cfg + self.generate_locations = cfg.generate_locations + self.exclude_location = cfg.exclude_location + self.force_exclude_location = cfg.force_exclude_location self.generate_comments = cfg.generate_comments self.generate_docstrings = cfg.generate_docstrings self.known_symbols = cfg.known_symbols or {} @@ -337,12 +340,27 @@ def Variable(self, tp): self.print_comment(tp) # 2021-02 give me a test case for this. it breaks all extern variables otherwise. - if tp.extern and self.find_library_with_func(tp): + # if tp.extern and self.find_library_with_func(tp): + if tp.extern: dll_library = self.find_library_with_func(tp) + is_stub = False + if not dll_library: + class LibraryStub: + _filepath = "FIXME_STUB" + _name = "FIXME_STUB" + dll_library = LibraryStub() + is_stub = True + self._generate(tp.typ) # calling convention does not matter for in_dll... - libname = self.get_sharedlib(dll_library, "cdecl") - print("%s = (%s).in_dll(%s, '%s')" % (tp.name, self.type_name(tp.typ), libname, tp.name), file=self.stream) + libname = self.get_sharedlib(dll_library, "cdecl", stub=is_stub) + #print("%s = (%s).in_dll(%s, '%s')" % (tp.name, self.type_name(tp.typ), libname, tp.name), file=self.stream) + decl = "{tp} = ctypes_in_dll({type_name}, {libname}, '{tp}')".format( + tp=tp.name, + type_name=self.type_name(tp.typ), + libname=libname, + ) + print(decl, file=self.stream) self.names.append(tp.name) # wtypes.h contains IID_IProcessInitControl, for example return @@ -612,8 +630,12 @@ def StructureHead(self, head, inline=False): log.debug("Head start for %s inline:%s", head.name, inline) for struct in head.struct.bases: self._generate(struct.get_head()) + # we MUST generate the structure body before inheritance happens + # or ctypes will tell us _field_ is final, cannot be changed + self._generate(struct.get_body(), inline) # add dependencies - self.more[struct] = True + #self.more[struct] = True + basenames = [self.type_name(b) for b in head.struct.bases] if basenames: # method_names = [m.name for m in head.struct.members if type(m) is typedesc.Method] @@ -891,10 +913,27 @@ def FundamentalType(self, _type): ######## + def _get_location(self, item): + location = item.location + if not location: + if isinstance(item, typedesc.StructureBody) or isinstance(item, typedesc.StructureHead): + location = item.struct.location + elif isinstance(item, typedesc.EnumValue): + location = item.enumeration.location + elif isinstance(item, typedesc.PointerType): + location = item.typ.location + + return location + def _generate(self, item, *args): """ wraps execution of specific methods.""" if item in self.done: return + if self.force_exclude_location: + location = self._get_location(item) + if location and not location[0].endswith(self.parser.tu.spelling): + return + # verbose output with location. if self.generate_locations and item.location: print("# %s:%d" % item.location, file=self.stream) @@ -924,20 +963,35 @@ def generate_all(self, items): def generate_items(self, items): # items = set(items) loops = 0 - while items: + self.more = collections.OrderedDict() + while True: loops += 1 - self.more = collections.OrderedDict() - self.generate_all(items) + #self.more = collections.OrderedDict() + items_to_gen = [] + for item in items: + if self.exclude_location: + if item not in self.more: + location = self._get_location(item) + if location and not location[0].endswith(self.parser.tu.spelling): + continue + items_to_gen.append(item) + self.generate_all(items_to_gen) # items |= self.more , but keeping ordering _s = set(items) [items.append(k) for k in self.more.keys() if k not in _s] # items -= self.done, but keep ordering + # more -= self.done _done = self.done.keys() for i in list(items): if i in _done: items.remove(i) + if i in self.more: + self.more.pop(i) + + if not self.more: + break return loops diff --git a/ctypeslib/codegen/config.py b/ctypeslib/codegen/config.py index b233417..25bcfe0 100644 --- a/ctypeslib/codegen/config.py +++ b/ctypeslib/codegen/config.py @@ -17,6 +17,10 @@ class CodegenConfig: generate_docstrings: bool = False # include source file location in comments generate_locations: bool = False + # on-demand include definitions outside of source file, only used definitions will be included + exclude_location: bool = False + # Forcibly exclude ALL definitions that located outside of source file + force_exclude_location: bool = False # do not include declaration defined outside of the source files filter_location: bool = True # dll to be loaded before all others (to resolve symbols) @@ -45,6 +49,8 @@ def parse_options(self, options): self.generate_comments = options.generate_comments self.generate_docstrings = options.generate_docstrings self.generate_locations = options.generate_locations + self.exclude_location = options.exclude_includes + self.force_exclude_location = options.force_exclude_includes self.filter_location = not options.generate_includes self.preloaded_dlls = options.preload # List exported symbols from libraries diff --git a/ctypeslib/codegen/cursorhandler.py b/ctypeslib/codegen/cursorhandler.py index 46fc5d0..ec9a2c9 100644 --- a/ctypeslib/codegen/cursorhandler.py +++ b/ctypeslib/codegen/cursorhandler.py @@ -96,6 +96,11 @@ def INIT_LIST_EXPR(self, cursor): # now fixed by TranslationUnit.PARSE_SKIP_FUNCTION_BODIES COMPOUND_STMT = ClangHandler._do_nothing + @log_entity + def NAMESPACE(self, cursor): + for child in cursor.get_children(): + self.parse_cursor(child) # FIXME, where is the starElement + ################################ # TYPE REFERENCES handlers @@ -630,6 +635,20 @@ def _record_decl(self, cursor, _output_type, num=None): # FIXME: lets ignore bases for now. # bases = attrs.get("bases", "").split() # that for cpp ? bases = [] # FIXME: support CXX + for c in cursor.get_children(): + if c.kind == CursorKind.CXX_BASE_SPECIFIER: + base_class_name = c.type.spelling + for n in ['struct_' + base_class_name, 'union_' + base_class_name]: + if self.is_registered(n): + bases.append(self.get_registered(n)) + break + else: + typedef_typ: typedesc.Typedef = self.get_registered(base_class_name) + while isinstance(typedef_typ, typedesc.Typedef): + typedef_typ = typedef_typ.typ + bases.append(typedef_typ) + + log.debug("got base class %s", c.displayname) size = cursor.type.get_size() align = cursor.type.get_align() if size == -2: # @@ -677,6 +696,14 @@ def _record_decl(self, cursor, _output_type, num=None): declared_instance = True else: obj = self.get_registered(name) + if cursor.is_definition(): + self.set_location(obj, cursor) + self.set_comment(obj, cursor) + else: + # Correctly handle multiple-time declaration in multiple header + # FIXME: test case like this: struct TypeA; struct TypeA; struct TypeA {int a;} + log.debug('cursor %s is not on a definition, and is declared multiple times', name) + return obj declared_instance = False # capture members declaration members = [] @@ -830,9 +857,13 @@ def _fixup_record(self, s): log.debug('FIXUP_STRUCT: no members') s.members = [] return - if s.size == 0: + if s.size == 0 or (s.size == 1 and len(s.members) == 0): log.debug('FIXUP_STRUCT: struct has size %d', s.size) return + if len(s.members) == 0 and len(s.bases) > 0: + log.debug('FIXUP_STRUCT: derived struct without new member') + return + # try to fix bitfields without padding first self._fixup_record_bitfields_type(s) # No need to lookup members in a global var. @@ -840,6 +871,11 @@ def _fixup_record(self, s): members = [] member = None offset = 0 + for b in s.bases: + offset += b.size * 8 + if s.size * 8 == offset: + log.debug('FIXUP_STRUCT: struct has size %d equals to base size', s.size) + return padding_nb = 0 member = None prev_member = None diff --git a/ctypeslib/codegen/handler.py b/ctypeslib/codegen/handler.py index de30fb3..5d8f3d7 100644 --- a/ctypeslib/codegen/handler.py +++ b/ctypeslib/codegen/handler.py @@ -127,8 +127,10 @@ def get_unique_name(self, cursor): return '' # covers most cases name = cursor.spelling + if cursor.kind == CursorKind.CXX_BASE_SPECIFIER: + name = cursor.type.spelling # if its a record decl or field decl and its type is unnamed - if cursor.spelling == '': + if name == '': # a unnamed object at the root TU if (cursor.semantic_parent and cursor.semantic_parent.kind == CursorKind.TRANSLATION_UNIT): @@ -144,11 +146,13 @@ def get_unique_name(self, cursor): #code.interact(local=locals()) return '' if cursor.kind in [CursorKind.STRUCT_DECL,CursorKind.UNION_DECL, - CursorKind.CLASS_DECL]: + CursorKind.CLASS_DECL, CursorKind.CXX_BASE_SPECIFIER]: names= {CursorKind.STRUCT_DECL: 'struct', CursorKind.UNION_DECL: 'union', - CursorKind.CLASS_DECL: 'class', - CursorKind.TYPE_REF: ''} + CursorKind.CLASS_DECL: 'struct', + CursorKind.TYPE_REF: '', + CursorKind.CXX_BASE_SPECIFIER: 'struct' + } name = '%s_%s'%(names[cursor.kind],name) log.debug('get_unique_name: name "%s"',name) return name diff --git a/ctypeslib/codegen/typehandler.py b/ctypeslib/codegen/typehandler.py index 2320a6a..41c0a1e 100644 --- a/ctypeslib/codegen/typehandler.py +++ b/ctypeslib/codegen/typehandler.py @@ -123,7 +123,13 @@ def POINTER(self, _cursor_type): # # we shortcut to canonical typedefs and to pointee canonical defs comment = None - _type = _cursor_type.get_pointee().get_canonical() + # _type = _cursor_type.get_pointee().get_canonical() + _type = _cursor_type.get_pointee() + if _type.get_canonical().kind == TypeKind.FUNCTIONPROTO: + # in python there's no pure function proto for ctypes, + # in code generator, functionproto will emit CFUNCTYPE, which is func ptr + # so pointer to function proto should actually be directly functionproto + return self.parse_cursor_type(_type) _p_type_name = self.get_unique_name(_type) # get pointer size size = _cursor_type.get_size() # not size of pointee diff --git a/ctypeslib/data/structure_type.tpl b/ctypeslib/data/structure_type.tpl index 5c08ffb..aa83eb4 100644 --- a/ctypeslib/data/structure_type.tpl +++ b/ctypeslib/data/structure_type.tpl @@ -49,6 +49,18 @@ class Structure(ctypes.Structure, AsDictMixin): args.update(kwds) super(Structure, self).__init__(**args) + def get_cfield(self, field): + fieldType = None + for fn, ftype in self._fields_: + if fn == field: + fieldType = ftype + break + else: + raise AttributeError('Field %s not found' % field) + + fieldAttr = getattr(self.__class__, field) + return fieldType.from_buffer(self, fieldAttr.offset) + @classmethod def _field_names_(cls): if hasattr(cls, '_fields_'): @@ -104,4 +116,10 @@ class Structure(ctypes.Structure, AsDictMixin): class Union(ctypes.Union, AsDictMixin): pass +def ctypes_in_dll(typ, dll, name): + try: + return typ.in_dll(dll, name) + except (ValueError, TypeError): + return None +