From 73925c396faefc48745d6547defe3bb7edd3df53 Mon Sep 17 00:00:00 2001 From: sebaszm Date: Mon, 8 Jun 2026 17:10:37 +0200 Subject: [PATCH 1/8] Sync bugfixes from 4.4.6 backport --- JsonGenerator/source/class_emitter.py | 36 ++-- .../source/documentation_generator.py | 19 +- JsonGenerator/source/header_loader.py | 16 +- JsonGenerator/source/json_loader.py | 11 +- JsonGenerator/source/rpc_emitter.py | 46 ++--- JsonGenerator/source/trackers.py | 17 +- ProxyStubGenerator/CppParser.py | 183 +++++++++--------- ProxyStubGenerator/StubGenerator.py | 19 +- ProxyStubGenerator/default.h | 3 + 9 files changed, 191 insertions(+), 159 deletions(-) diff --git a/JsonGenerator/source/class_emitter.py b/JsonGenerator/source/class_emitter.py index 187a0a3b..050edee0 100644 --- a/JsonGenerator/source/class_emitter.py +++ b/JsonGenerator/source/class_emitter.py @@ -227,6 +227,8 @@ def ProcessEnums(log, action=None): return count def EmitEnumRegs(log, root, emit, header_file, if_file): + log.Info("Emitting enum conversion tables... ") + def _EmitEnumRegistration(log, enum): name = enum.original_type if enum.original_type else (Scoped(root, enum) + enum.cpp_class) @@ -274,6 +276,7 @@ def EmitObjects(log, root, emit, if_file, additional_includes, emitCommon = Fals emittedItems = 0 def _EmitEnumConversionHandler(root, enum): + log.Info("Emitting enum conversion handler for '{}'".format(enum.cpp_class)) name = enum.original_type if enum.original_type else (Scoped(root, enum) + enum.cpp_class) emit.Line("ENUM_CONVERSION_HANDLER(%s)" % name) @@ -705,11 +708,12 @@ def _EmitNoPushWarnings(prologue = True): emit.Indent() emit.Line() - if emitCommon and trackers.enum_tracker.CommonObjects(): + if emitCommon and trackers.enum_tracker.common_objects: log.Info("Emitting common enums...") emittedPrologue = False - for obj in trackers.enum_tracker.CommonObjects(): - if obj.do_create and not obj.is_duplicate and not obj.included_from: + for obj in trackers.enum_tracker.common_objects: + if obj.do_create and not obj.included_from: + assert not obj.is_duplicate if not emittedPrologue: emit.Line("// Common enums") emit.Line("//") @@ -717,11 +721,12 @@ def _EmitNoPushWarnings(prologue = True): emittedPrologue = True _EmitEnum(obj) - if emitCommon and trackers.object_tracker.CommonObjects(): + if emitCommon and trackers.object_tracker.common_objects: log.Info("Emitting common classes...") emittedPrologue = False - for obj in trackers.object_tracker.CommonObjects(): + for obj in trackers.object_tracker.common_objects: if not obj.included_from: + assert not obj.is_duplicate if not emittedPrologue: emit.Line("// Common classes") emit.Line("//") @@ -751,16 +756,19 @@ def _EmitNoPushWarnings(prologue = True): emit.Line("} // namespace %s" % config.DATA_NAMESPACE) emit.Line() - emittedPrologue = False - - for obj in trackers.enum_tracker.objects: - if not obj.is_duplicate and not obj.included_from: - if not emittedPrologue: - emit.Line("// Enum conversion handlers") - emittedPrologue = True - _EmitEnumConversionHandler(root, obj) - emittedItems += 1 + log.Info("Emitting enum conversion handlers...") + if trackers.enum_tracker.objects: + emittedPrologue = False + for obj in trackers.enum_tracker.objects: + if not obj.is_duplicate and not obj.included_from: + assert not obj.is_duplicate + if not emittedPrologue: + emit.Line("// Enum conversion handlers") + emittedPrologue = True + + _EmitEnumConversionHandler(root, obj) + emittedItems += 1 emit.Line() emit.Line("}") diff --git a/JsonGenerator/source/documentation_generator.py b/JsonGenerator/source/documentation_generator.py index 63c67187..03b9f9d4 100644 --- a/JsonGenerator/source/documentation_generator.py +++ b/JsonGenerator/source/documentation_generator.py @@ -102,9 +102,9 @@ def ParamTable(name, object): def _TableObj(name, obj, parentName="", parent=None, prefix="", parentOptional=False): # determine if the attribute is optional - optional = parentOptional or (obj["optional"] if "optional" in obj else False) - deprecated = obj["deprecated"] if "deprecated" in obj else False - obsolete = obj["obsolete"] if "obsolete" in obj else False + optional = parentOptional or obj.get("optional") or obj.get("@optionaltype") + deprecated = obj.get("deprecated") + obsolete = obj.get("obsolete") restricted = obj.get("range") name = name.replace(' ','-') @@ -123,7 +123,7 @@ def _TableObj(name, obj, parentName="", parent=None, prefix="", parentOptional=F endmarker = obj.get("@endmarker") for i,e in enumerate(obj["ids"]): if e == endmarker: - break; + break enums.append(str(obj["enum"][i])) if "default" in obj and e == obj["default"]: default_enum = enums[-1] @@ -237,10 +237,15 @@ def _TableObj(name, obj, parentName="", parent=None, prefix="", parentOptional=F if obj.get("@async"): if row: row += "
" - obj_type = "string (async ID)" + elif obj.get("opaque"): + obj_type = "opaque object" + elif obj.get("encode"): + obj_type = "string (%s)" % obj.get("encode") + elif obj.get("time"): + obj_type = "string (%s time)" % obj.get("time").upper() else: - obj_type = "opaque object" if obj.get("opaque") else ("string (%s)" % obj.get("encode")) if obj.get("encode") else obj["type"] + obj_type = obj["type"] if obj.get("@lookup"): if row == "...": @@ -260,7 +265,7 @@ def _TableObj(name, obj, parentName="", parent=None, prefix="", parentOptional=F _TableObj(pname, props, parentName + "/" + name, obj, prefix, False) elif obj["type"] == "array": - _TableObj("", obj["items"], parentName + "/" + name, obj, (prefix + "[#]") if name else "", False) + _TableObj("", obj["items"], parentName + "/" + name, obj, (prefix + "[#]") if name else "", optional) _TableObj(name, object, "") MdBr() diff --git a/JsonGenerator/source/header_loader.py b/JsonGenerator/source/header_loader.py index 0790d4fd..faf80d6f 100644 --- a/JsonGenerator/source/header_loader.py +++ b/JsonGenerator/source/header_loader.py @@ -353,7 +353,7 @@ def handle_optional(var, var_type, properties, quiet=False): is_iterator = isinstance(var_type.type, CppParser.Class) and var_type.type.is_iterator if not var.meta.output: - if isinstance(var_type.type, (CppParser.Integer, CppParser.Int24, CppParser.Enum)): + if isinstance(var_type.type, (CppParser.Integer, CppParser.Int24, CppParser.Enum, CppParser.Bool)): if not var.meta.default and not quiet: log.WarnLine(var, "%s: default value is assumed to be (%s)0 (use @default with @optional)" % (var.name, var.type.TypeName())) elif not isinstance(var_type.type, (CppParser.String, CppParser.Vector, CppParser.Class)) or (var_type.IsPointer() and not is_iterator): @@ -935,9 +935,6 @@ def BuildParameters(obj, vars, rpc_format, is_property=False, is_index=False, te converted["@position"] = vars.index(var) - if not handle_optional(var, ResolveTypedef(var.type), converted, test): - required.append(var_name) - if converted["type"] == "string" and not var.type.IsReference() and not var.type.IsPointer() \ and not "enum" in converted and not "time" in converted: log.WarnLine(var, "'%s': passing input string by value (forgot &?)" % var.name) @@ -952,6 +949,8 @@ def BuildParameters(obj, vars, rpc_format, is_property=False, is_index=False, te raise CppParseError(var, "@extract not allowed on this parameter") properties = converted["properties"] + params["@originaltype"] = converted["@originaltype"] + required = converted["required"] extracted = var elif converted["type"] != "array": raise CppParseError(var, "@extract not supported for this type") @@ -962,6 +961,9 @@ def BuildParameters(obj, vars, rpc_format, is_property=False, is_index=False, te else: properties[var_name] = converted + if not handle_optional(var, ResolveTypedef(var.type), converted, test): + required.append(var_name) + params["properties"] = properties params["required"] = required @@ -1026,6 +1028,7 @@ def BuildResult(method, vars, is_property=False, test=False): raise CppParseError(var, "@extract not allowed on property parameters") properties = converted["properties"] + params["@originaltype"] = converted["@originaltype"] extracted = var elif converted["type"] != "array": raise CppParseError(var, "@extract not supported for this type") @@ -1446,7 +1449,7 @@ def MaybeWarning(): for e in interfaces: for m in e.obj.methods: for p in m.vars: - if isinstance(p.type.type, CppParser.Class) and p.type.type.is_event: + if isinstance(p.type.type, CppParser.Class) and p.type.type.is_event and p.type.type.full_name == f.obj.full_name: log.WarnLine(method, "%s: @index is not recommended here, declare in event registration method instead: %s()" % (method.vars[0].name, m.full_name)) return MaybeWarning() @@ -1457,7 +1460,7 @@ def MaybeWarning(): if obj["id"]: obj["id"]["name"] = method.vars[0].name - obj["id"]["@originalname"] = "designatorId_" if deprecated else "index_" + obj["id"]["@originalname"] = "index_" obj["id"]["@generated"] = True if deprecated: @@ -1604,6 +1607,7 @@ def MaybeWarning(): return schemas, [] def LoadInterface(file, log, all = False, include_paths = []): + try: schemas = [] includes = [] diff --git a/JsonGenerator/source/json_loader.py b/JsonGenerator/source/json_loader.py index ef76d423..2f98afbf 100644 --- a/JsonGenerator/source/json_loader.py +++ b/JsonGenerator/source/json_loader.py @@ -255,6 +255,8 @@ def cpp_native_type(self): def cpp_native_type_cv(self): if self.schema.get("@cv"): return self.schema.get("@cv") + " " + self.cpp_native_type + else: + return self.cpp_native_type @property def cpp_native_type_opt(self): @@ -293,8 +295,10 @@ def cpp_native_type_opt_v(self, toggle=False): @property def local_proto(self): - assert self.schema.get("@proto") - return self.cpp_native_type_proto.replace('@', self.local_name) + if "@proto" in self.schema: + return self.cpp_native_type_proto.replace('@', self.local_name) + else: + return "%s %s" % (self.cpp_native_type_opt_cv, self.local_name) @property def cpp_concrete_type(self): @@ -443,7 +447,6 @@ def AddRef(self, obj): def RefCount(self): return len(self.refs) - class JsonTime(JsonNative, JsonType): def __init__(self, name, parent, schema, time_type): JsonType.__init__(self, name, parent, schema) @@ -1071,7 +1074,7 @@ def _AddMethods(section, schema, ctor): _AddMethods("events", schema, lambda name, obj, method: JsonNotification(name, obj, method)) if not self.methods: - raise JsonParseError("no methods, properties or events defined in %s" % self.schema["@fullname"] if "@fullname" in self.schema else self.name) + raise JsonParseError("no methods, properties or events defined in %s" % (self.schema["@fullname"] if "@fullname" in self.schema else self.name)) @property def root(self): diff --git a/JsonGenerator/source/rpc_emitter.py b/JsonGenerator/source/rpc_emitter.py index 5824814a..04fe2187 100644 --- a/JsonGenerator/source/rpc_emitter.py +++ b/JsonGenerator/source/rpc_emitter.py @@ -49,7 +49,7 @@ def FromString(emit, param, restrictions=None, emit_restrictions=False): needs_move = False error_condition_emitted = False - converted = param.TempName("conv_") if has_conversion else param.original_name + converted = param.TempName("conv_") if has_conversion else param.local_name converted_result = param.TempName("convResult_") converted_enum = param.TempName("convEnum_") array_size = param.schema.get("@arraysize") @@ -78,12 +78,12 @@ def EmitLocals(): if not is_optional_type: EmitLocals() - default_conditions.extend("%s.empty() == true" % param.original_name) + default_conditions.extend("%s.empty() == true" % param.local_name) emit.If(default_conditions) if param.default_value: emit.Line("%s = %s;" % (opt_name, param.default_value)) - elif is_optional_type: + elif is_optional_type or is_legacy_optional: emit.Line("// no error, optional") else: emit.Line("%s = %s;" % (error_code.temp_name, CoreError("bad_request"))) @@ -101,12 +101,12 @@ def Decode(length_type, length): if encode == "base64": converted_length_param = param.TempName("convLength_") emit.Line("%s %s(%s);" % (length_type, converted_length_param, length)) - emit.Line("Core::FromString(%s, %s, %s);" % (param.original_name, converted, converted_length_param)) + emit.Line("Core::FromString(%s, %s, %s);" % (param.local_name, converted, converted_length_param)) emit.Line("const bool %s = (%s != 0);" % (converted_result, converted_length_param)) elif encode == "hex": - emit.Line("const bool %s = (Core::FromHexString(%s, %s, %s) != 0));" % (converted_result, param.original_name, converted, length)) + emit.Line("const bool %s = (Core::FromHexString(%s, %s, %s) != 0));" % (converted_result, param.local_name, converted, length)) elif encode == "mac": - emit.Line("const bool %s = (Core::FromHexString(%s, %s, %s, TCHAR(':')) != 0);" % (converted_result, param.original_name, converted, length)) + emit.Line("const bool %s = (Core::FromHexString(%s, %s, %s, TCHAR(':')) != 0);" % (converted_result, param.local_name, converted, length)) else: assert False, "bad encode method" @@ -119,10 +119,10 @@ def Decode(length_type, length): needs_move = True elif isinstance(param, (JsonInteger, JsonBoolean)): - emit.Line("const bool %s = Core::FromString(%s, %s);" % (converted_result, param.original_name, converted)) + emit.Line("const bool %s = Core::FromString(%s, %s);" % (converted_result, param.local_name, converted)) elif isinstance(param, JsonEnum): - emit.Line("Core::EnumerateType<%s> %s(%s.c_str());" % (param.cpp_native_type, converted_enum, param.original_name)) + emit.Line("Core::EnumerateType<%s> %s(%s.c_str());" % (param.cpp_native_type, converted_enum, param.local_name)) emit.Line("%s %s{%s.Value()};" % (param.cpp_native_type, converted, converted_enum)) emit.Line("const bool %s = %s.IsSet();" % (converted_result, converted_enum)) @@ -156,7 +156,7 @@ def Decode(length_type, length): emit.Endif(default_conditions) - if converted != param.original_name: + if converted != param.local_name: emit.Line() return converted, error_condition_emitted @@ -176,7 +176,6 @@ def trim(identifier): names['designator'] = "designator_" names['sendif'] = "sendIfMethod_" names["index"] = "index_" - names['designator_id'] = "designatorId_" prefix = ("%s." % names.module) @@ -275,7 +274,8 @@ def EmitParam(p, parent=None, override=None): optional_conditions = Restrictions(json=False, original_name=True) if IsObjectOptional(p) and not IsObjectOptionalOrOpaque(p): optional_conditions.check_set(p) - local_name += ".Value()" + if p.optional: + local_name += ".Value()" elif IsObjectOptionalOrOpaque(p): optional_conditions.check_not_null(p) @@ -490,11 +490,11 @@ def Emit(parameters): if event.sendif_type and event.sendif_deprecated: if event.is_status_listener and has_client: emit.Line("const size_t _dot = %s.find('.');" % (names.designator)) - emit.Line("const string %s = %s.substr(0, _dot);" % (names.designator_id, names.designator)) + emit.Line("const string %s = %s.substr(0, _dot);" % (names.index, names.designator)) check = ("%s.substr(_dot + 1)" % (names.designator)) cond.append("((%s.empty() == true) || (%s == %s))" % (names.client, names.client, check)) else: - emit.Line("const string %s = %s.substr(0, %s.find('.'));" % (names.designator_id, names.designator, names.designator)) + emit.Line("const string %s = %s.substr(0, %s.find('.'));" % (names.index, names.designator, names.designator)) converted, _ = FromString(emit, event.sendif_type, Restrictions(json=False, adjust=False), True) @@ -1271,9 +1271,9 @@ def _Invoke(method, conditional_invoke, sorted_vars, params, response, parent="" else: # fallback to @length alloca_param = size - conditions.check_set(maxlength_param[0]) - conditions.check_not_null(maxlength_param[0]) - if maxlength_param[0].size > 16: + conditions.check_set(length_param[0]) + conditions.check_not_null(length_param[0]) + if length_param[0].size > 16: conditions.extend("%s <= 0x100000" % size) emit.EnterBlock(conditions) @@ -1303,7 +1303,6 @@ def _Invoke(method, conditional_invoke, sorted_vars, params, response, parent="" emit.Line("%s %s;" % (param.original_type, vector)) emit.Line("auto %s = %s.Elements();" % (temp, parent + param.cpp_name)) - # Removed extra paranthesis from the below line leading to compilation failure emit.Line("while (%s.Next() == true) { %s.push_back(%s.Current()); }" % (temp, vector, temp)) if param.optional: @@ -1397,12 +1396,13 @@ def _Invoke(method, conditional_invoke, sorted_vars, params, response, parent="" emit.Line("else {") emit.Indent() emit.Line("%s = %s;" % (error_code.temp_name, CoreError("unknown_key"))) - for c in cleanup: - emit.Line("if (%s != nullptr) {" % c) - emit.Indent() - emit.Line("%s->Release();" % c) - emit.Unindent() - emit.Line("}") + emit.Unindent() + emit.Line("}") + + for c in cleanup: + emit.Line("if (%s != nullptr) {" % c) + emit.Indent() + emit.Line("%s->Release();" % c) emit.Unindent() emit.Line("}") diff --git a/JsonGenerator/source/trackers.py b/JsonGenerator/source/trackers.py index 6c380506..ba28e7d0 100644 --- a/JsonGenerator/source/trackers.py +++ b/JsonGenerator/source/trackers.py @@ -16,6 +16,7 @@ # limitations under the License. import config +import logger from json_loader import * log = None @@ -23,9 +24,12 @@ def SortByDependency(objects): sorted_objects = [] + unique_objects = {it.cpp_class: it for it in filter(lambda o: not o.is_duplicate, objects)}.values() + # This will order objects by their relations - for obj in sorted(objects, key=lambda x: x.cpp_class, reverse=False): + for obj in sorted(unique_objects, key=lambda x: x.cpp_class, reverse=False): found = filter(lambda sorted_obj: obj.cpp_class in map(lambda x: x.cpp_class, sorted_obj.objects), sorted_objects) + try: index = min(map(lambda x: sorted_objects.index(x), found)) movelist = filter(lambda x: x.cpp_class in map(lambda x: x.cpp_class, sorted_objects), obj.objects) @@ -33,7 +37,10 @@ def SortByDependency(objects): for m in movelist: if m in sorted_objects: - sorted_objects.insert(index, sorted_objects.pop(sorted_objects.index(m))) + old = sorted_objects.index(m) + if old > index: + sorted_objects.insert(index, sorted_objects.pop(old)) + except ValueError: sorted_objects.append(obj) @@ -218,7 +225,8 @@ def Remove(self, obj): def Reset(self): self.objects = [] - def CommonObjects(self): + @property + def common_objects(self): return SortByDependency(filter(lambda obj: obj.RefCount() > 1, self.objects)) @@ -286,7 +294,8 @@ def __Compare(lhs, rhs): return None - def CommonObjects(self): + @property + def common_objects(self): return SortByDependency(filter(lambda obj: ((obj.RefCount() > 1) or self._IsTopmost(obj)), self.objects)) def SetLogger(logger): diff --git a/ProxyStubGenerator/CppParser.py b/ProxyStubGenerator/CppParser.py index 409c3aaf..f92dc56a 100755 --- a/ProxyStubGenerator/CppParser.py +++ b/ProxyStubGenerator/CppParser.py @@ -405,6 +405,16 @@ def __Search(tree, found, T): assert len(found) == 1, "Too many references found, need scope for %s" % identifier selected = found[-1] + if selected: + # count references to instantiated template classes, we might not want to emit proxystubs for unused templates + if isinstance(selected, InstantiatedTemplateClass): + selected.AddRef() + elif isinstance(selected, Typedef): + resolved = selected.DataType() + if not isinstance(resolved, str): + if isinstance(resolved.type, InstantiatedTemplateClass): + resolved.type.AddRef() + return selected @@ -557,6 +567,8 @@ def __init__(self, parent_block, parent, string, valid_specifiers, tags_allowed= self.meta.decorators.append("encode:ip") elif tag == "ENCODEMAC": self.meta.decorators.append("encode:mac") + elif tag == "ENCODEARRAY": + self.meta.decorators.append("encode:array") elif tag == "OPTIONAL": self.meta.decorators.append("optional") elif tag == "EXTRACT": @@ -1681,6 +1693,10 @@ def __init__(self, parent_block, name, params, args): self.args = args self.resolvedArgs = [Identifier(parent_block, self, [x], []) for x in args] self.type = self.TypeName() + self.refs = 0 + + def AddRef(self): + self.refs += 1 def TypeName(self): return "%s<%s>" % (self.baseName.full_name, ", ".join([str("".join([str(x) for x in p.type]) if isinstance(p.type, list) else p.type) for p in self.resolvedArgs])) @@ -1855,10 +1871,10 @@ def __Tokenize(contents,log = None): inComment = 0 for token in tokens: if token.startswith("// @_file:"): - line = 1 + line = int(token[11:].split(',')[1]) eoltokens.append(token) else: - if not inComment: + if not inComment and token.strip() and not token.lstrip().startswith("//"): eoltokens.append("// @_line:" + str(line) + " ") if (len(eoltokens) > 1) and eoltokens[-2].endswith("\\"): @@ -1867,7 +1883,8 @@ def __Tokenize(contents,log = None): else: eoltokens.append(token) - line += 1 + if not token.startswith("// @_"): + line += 1 inComment += eoltokens[-1].count("/*") - eoltokens[-1].count("*/") @@ -1913,7 +1930,7 @@ def __Tokenize(contents,log = None): if skipmode: if "@_file" in token: skipmode = False - else: + elif token not in ['}', '{', ';'] and not token.startswith("// @_"): continue def __ParseParameterValue(string, tag, mandatory=True, append=True, relay=None): @@ -2092,6 +2109,8 @@ def _find(word, string): tagtokens.append("@ENCODEIP") elif _find("@encode:mac", token): tagtokens.append("@ENCODEMAC") + elif _find("@encode:array", token): + tagtokens.append("@ENCODEARRAY") if _find("@length", token): tagtokens.append(__ParseParameterValue(token, "@length")) @@ -2152,8 +2171,9 @@ def EndOfTag(string, start): if _find("@_file", token): idx = token.index("@_file:") + 7 - tagtokens.append("@FILE:" + token[idx:]) - current_file = token[idx:] + _current_file = token[idx:].split(',')[0] + tagtokens.append("@FILE:" + _current_file) + current_file = _current_file if _find("@_line", token): idx = token.index("@_line:") + 7 if len(tagtokens) and not isinstance(tagtokens[-1],list) and tagtokens[-1].startswith("@LINE:"): @@ -2206,6 +2226,7 @@ def CurrentLine(): # Builds a syntax tree (data structures only) of C++ source code def Parse(contents,log = None): + # Start in global namespace. global global_namespace global current_file @@ -2230,7 +2251,6 @@ def Parse(contents,log = None): current_line = int(token[6:].split()[0]) elif isinstance(token, str) and token.startswith("@FILE:"): current_file = token[6:] - tokens.append("@GLOBAL") line_numbers.append(current_line) files.append(current_file) else: @@ -2238,7 +2258,6 @@ def Parse(contents,log = None): line_numbers.append(current_line) files.append(current_file) - global_namespace = Namespace(None) current_block = [global_namespace] @@ -2260,6 +2279,7 @@ def Parse(contents,log = None): collapsed_next = False compliant_next = False iterator_next = False + wrap_next = False sourcelocation_next = False text_next = None in_typedef = False @@ -2349,32 +2369,6 @@ def Parse(contents,log = None): iterator_next = True tokens[i] = ";" i += 1 - elif tokens[i] == "@GLOBAL": - current_block = [global_namespace] - next_block = None - last_template_def = [] - min_index = 0 - omit_mode = False - omit_next = False - stub_next = False - json_next = False - object_next = False - autoobject_next = False - encode_enum_next = False - wrap_next = False - json_version = "" - prefix_next = False - prefix_string = "" - event_next = False - extended_next = False - collapsed_next = False - compliant_next = False - iterator_next = False - sourcelocation_next = False - text_next = None - in_typedef = False - tokens[i] = ";" - i += 1 # Swallow template definitions elif tokens[i] == "template" and tokens[i + 1] == '<': @@ -2877,77 +2871,74 @@ def Parse(contents,log = None): # ------------------------------------------------------------------------- - -def ReadFile(source_file, includePaths, quiet=False, initial="", omit=False): +def ReadFile(source_file, include_paths, parent_file="", index=0, inclusions=[], quiet=False, omit=False, use_includes=False): contents = "" - global current_file + file_path = None + + if source_file[0] == '@': + quiet = True + source_file = source_file[1:] + + if not parent_file: + parent_file = source_file + abs_source_file = os.path.abspath(source_file) - if abs_source_file not in initial: - try: - with open(source_file) as file: - file_content = file.read() - pos = 0 - while True: - idx = file_content.find("@stubgen:include", pos) - if idx == -1: - idx = file_content.find("@insert", pos) + if os.path.isfile(abs_source_file): + file_path = abs_source_file - if idx != -1: - pos = idx + 1 - line = file_content[idx:].split("\n", 1)[0] - match = re.search(r' \"(.+?)\"', line) + elif not os.path.isabs(source_file) and use_includes: + for path in include_paths: + abs_source_file = os.path.abspath(os.path.join(path, source_file)) + if os.path.isfile(abs_source_file): + file_path = abs_source_file + break - if match: - if match.group(1) != os.path.basename(os.path.realpath(source_file)): - tryPath = os.path.join(os.path.dirname(os.path.realpath(source_file)), match.group(1)) - - if os.path.isfile(tryPath): - prev = current_file - current_file = source_file - contents += ReadFile(tryPath, includePaths, False, contents, True) - current_file = prev - else: - raise LoaderError(source_file, "can't include '%s', file does not exist" % tryPath) - else: - raise LoaderError(source_file, "can't recursively include self") - else: - match = re.search(r' <(.+?)>', line) - - if match: - found = False - for ipath in includePaths: - tryPath = os.path.join(ipath, match.group(1)) - - if os.path.isfile(tryPath): - prev = current_file - current_file = source_file - contents += ReadFile(tryPath, includePaths, True, contents, True) - current_file = prev - found = True - if not found: - raise LoaderError(source_file, "can't find '%s' in any of the include paths" % match.group(1)) - else: - raise LoaderError(source_file, "syntax error at '%s'" % source_file) - else: - break + if file_path in inclusions: + pass - contents += "// @_file:%s\n" % abs_source_file + elif file_path: + inclusions.append(file_path) - if omit: - contents += "// @_omit_start\n" - contents += file_content - contents += "// @_omit_end\n" + with open(file_path) as file: + file_content = file.readlines() + for i,line in enumerate(file_content): + match = re.search(r'@(?:insert|stubgen:include)\s*"([^"]+)"', line) + if match: + included_file = os.path.join(os.path.dirname(os.path.realpath(file_path)), match.group(1)) + included_content = ReadFile(included_file, include_paths, file_path, i+1, inclusions, quiet=False, omit=True, use_includes=False) + file_content[i] = included_content else: - contents += file_content + match = re.search(r'@(?:insert|stubgen:include)\s*<([^>]+)>', line) + if match: + included_content = ReadFile(match.group(1), include_paths, file_path, i+1, inclusions, quiet=False, omit=True, use_includes=True) + file_content[i] = included_content + else: + match = re.search(r'@insert:weak\s*<([^>]+)>', line) + if match: + included_content = ReadFile(match.group(1), include_paths, file_path, i+1, inclusions, quiet=True, omit=True, use_includes=True) + file_content[i] = included_content + + contents = "// @_file:%s,1\n" % (file_path) - return contents + if omit: + contents += "// @_omit_start\n" - except FileNotFoundError: - if not quiet: - raise LoaderError(source_file, "failed to open file") + contents += "".join(file_content) - return "" + if omit: + contents += "// @_omit_end\n" + + if index: + contents += "// @_file:%s,%i\n" % (parent_file,index) + + elif not quiet: + if use_includes: + raise LoaderError(parent_file, "can't find '%s' in any of the include paths" % source_file) + else: + raise LoaderError(parent_file, "failed to open file %s" % source_file) + + return contents def Locate(block_name, tree=None): @@ -2978,10 +2969,10 @@ def ParseFile(source_file, includePaths = []): def ParseFiles(source_files, framework_namespace, includePaths = [], log = None): contents = "" + for source_file in source_files: if source_file: - quiet = (source_file[0] == "@") - contents += ReadFile((source_file[1:] if quiet else source_file), includePaths, quiet, "") + contents += ReadFile(source_file, includePaths, inclusions=[]) contents = contents.replace("__FRAMEWORK_NAMESPACE__", framework_namespace) return Parse(contents, log) diff --git a/ProxyStubGenerator/StubGenerator.py b/ProxyStubGenerator/StubGenerator.py index fdbe5304..6b84d38a 100755 --- a/ProxyStubGenerator/StubGenerator.py +++ b/ProxyStubGenerator/StubGenerator.py @@ -126,7 +126,7 @@ def __init__(self, obj, iid, file): self.file = file -# Looks for interface classes (ie. classes inheriting from Core::Unknown and specifying ID enum). +# Looks for interface classes (ie. classes inheriting from Core::IUnknown and specifying ID enum). def FindInterfaceClasses(tree, namespace): interfaces = [] omit_interface_used = False @@ -140,7 +140,7 @@ def __Traverse(tree, interface_namespace, faces): if c.omit: omit_interface_used = True - if not isinstance(c, CppParser.TemplateClass): + if not isinstance(c, CppParser.TemplateClass) and (not isinstance(c, CppParser.InstantiatedTemplateClass) or c.refs > 0): if (c.full_name.startswith(interface_namespace + "::")): if (c.is_iterator and ENABLE_ITERATOR_OPTIMIZATION) and not c.is_force_interface: log.Info("Skipping iterator %s" % c.name, source_file) @@ -1047,8 +1047,8 @@ def _FindLength(length_name, variable_name): elif self.identifier_type.IsPointerToConst() and self.is_output: raise TypenameError(identifier, "'%s': output parameter must not be const" % self.trace_proto) - if (not self.is_buffer and not self.is_array) and isinstance(self.kind, (CppParser.Integer, CppParser.BuiltinInteger)): - if not self.kind.fixed and not self.kind.char: + if (not self.is_buffer and not self.is_array) and isinstance(self.kind, CppParser.Integer): + if not self.kind.fixed and self.kind.size != "char": log.WarnLine(self.identifier, "'%s': integer is not fixed-width, use a stdint type" % self.trace_proto) if isinstance(self.kind, CppParser.Enum): @@ -2673,9 +2673,11 @@ def EmitRegistration(announce_list): emit.Line("//") emit.Line("// implements COM-RPC proxy stubs for:") + emitted_interfaces = [] for face in interfaces: if not face.obj.omit: emit.Line("// - %s" % Flatten(str(face.obj), ns)) + emitted_interfaces.append(face) emit.Line("//") @@ -2737,6 +2739,7 @@ def EmitRegistration(announce_list): emit.Line("// -----------------------------------------------------------------") emit.Line("// REGISTRATION") emit.Line("// -----------------------------------------------------------------") + emit.Line() EmitRegistration(announce_list) emit.IndentDec() @@ -2744,7 +2747,7 @@ def EmitRegistration(announce_list): emit.Line() emit.Line("}") - return interfaces, omit_interface_used + return emitted_interfaces, omit_interface_used # ------------------------------------------------------------------------- @@ -2911,6 +2914,7 @@ def EmitRegistration(announce_list): print(" @statuslistener - emits registration/unregistration status listeners for an event") print(" @bitmask - indicates that enumerator lists should be packed into into a bit mask") print(" @index - indicates that a parameter in a JSON-RPC property or notification is an index") + print(" @index:deprecated - indicates that a parameter to a notification is legacy index") print(" @opaque - indicates that a string parameter is an opaque JSON object") print(" @extract - indicates that that if only one element is present in the array it shall be taken out of it") print(" or, if the parameter is a struct, the object should be collapsed and its members spilled to method parameters") @@ -2982,6 +2986,8 @@ def EmitRegistration(announce_list): for source_file in interface_files: try: _extra_includes = [ os.path.join("@" + os.path.dirname(source_file), MODULE_FILE) ] + _extra_includes = [ os.path.join("@" + os.path.dirname(source_file), "Ids.h") ] + _extra_includes.extend(args.extra_includes) tree = Parse(source_file, FRAMEWORK_NAMESPACE, args.includePaths, @@ -3007,6 +3013,9 @@ def EmitRegistration(announce_list): new_faces += output if not new_faces: + if os.path.isfile(output_file): + os.remove(output_file) + if not some_omitted: raise NoInterfaceError else: diff --git a/ProxyStubGenerator/default.h b/ProxyStubGenerator/default.h index 702c6bf6..6b303d01 100644 --- a/ProxyStubGenerator/default.h +++ b/ProxyStubGenerator/default.h @@ -78,6 +78,7 @@ namespace __FRAMEWORK_NAMESPACE__ { } } + // for legacy compatibiltity only namespace PluginHost { class IShell; class ISubSystem; @@ -87,3 +88,5 @@ namespace __FRAMEWORK_NAMESPACE__ { }; } } + +// @insert:weak From 41c0bc1f410bff873980824c40be860428725870 Mon Sep 17 00:00:00 2001 From: sebaszm Date: Wed, 10 Jun 2026 01:07:16 +0200 Subject: [PATCH 2/8] address comments --- JsonGenerator/source/rpc_emitter.py | 2 +- ProxyStubGenerator/CppParser.py | 5 ++++- ProxyStubGenerator/StubGenerator.py | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/JsonGenerator/source/rpc_emitter.py b/JsonGenerator/source/rpc_emitter.py index 04fe2187..520bc27f 100644 --- a/JsonGenerator/source/rpc_emitter.py +++ b/JsonGenerator/source/rpc_emitter.py @@ -104,7 +104,7 @@ def Decode(length_type, length): emit.Line("Core::FromString(%s, %s, %s);" % (param.local_name, converted, converted_length_param)) emit.Line("const bool %s = (%s != 0);" % (converted_result, converted_length_param)) elif encode == "hex": - emit.Line("const bool %s = (Core::FromHexString(%s, %s, %s) != 0));" % (converted_result, param.local_name, converted, length)) + emit.Line("const bool %s = (Core::FromHexString(%s, %s, %s) != 0);" % (converted_result, param.local_name, converted, length)) elif encode == "mac": emit.Line("const bool %s = (Core::FromHexString(%s, %s, %s, TCHAR(':')) != 0);" % (converted_result, param.local_name, converted, length)) else: diff --git a/ProxyStubGenerator/CppParser.py b/ProxyStubGenerator/CppParser.py index f92dc56a..e58f4146 100755 --- a/ProxyStubGenerator/CppParser.py +++ b/ProxyStubGenerator/CppParser.py @@ -2871,10 +2871,13 @@ def Parse(contents,log = None): # ------------------------------------------------------------------------- -def ReadFile(source_file, include_paths, parent_file="", index=0, inclusions=[], quiet=False, omit=False, use_includes=False): +def ReadFile(source_file, include_paths, parent_file="", index=0, inclusions=None, quiet=False, omit=False, use_includes=False): contents = "" file_path = None + if inclusions is None: + inclusions = [] + if source_file[0] == '@': quiet = True source_file = source_file[1:] diff --git a/ProxyStubGenerator/StubGenerator.py b/ProxyStubGenerator/StubGenerator.py index 6b84d38a..49b68aad 100755 --- a/ProxyStubGenerator/StubGenerator.py +++ b/ProxyStubGenerator/StubGenerator.py @@ -2985,8 +2985,8 @@ def EmitRegistration(announce_list): for source_file in interface_files: try: - _extra_includes = [ os.path.join("@" + os.path.dirname(source_file), MODULE_FILE) ] - _extra_includes = [ os.path.join("@" + os.path.dirname(source_file), "Ids.h") ] + _extra_includes = [ os.path.join("@" + os.path.dirname(source_file), MODULE_FILE), + os.path.join("@" + os.path.dirname(source_file), "Ids.h") ] _extra_includes.extend(args.extra_includes) From 5b5ddd1afe2acc0aed60f3216bedb5326eefb892 Mon Sep 17 00:00:00 2001 From: sebaszm Date: Wed, 10 Jun 2026 01:07:51 +0200 Subject: [PATCH 3/8] fix missing class definition if event has same name as method --- JsonGenerator/source/json_loader.py | 19 +++++++++++++++++++ JsonGenerator/source/trackers.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/JsonGenerator/source/json_loader.py b/JsonGenerator/source/json_loader.py index 2f98afbf..1de34cf6 100644 --- a/JsonGenerator/source/json_loader.py +++ b/JsonGenerator/source/json_loader.py @@ -243,6 +243,10 @@ def cpp_type(self): # C++ type of the object (e.g. may be array) def short_cpp_type(self): return self.cpp_type.replace("%s::%s::" % (config.DATA_NAMESPACE, self.root.cpp_class), "") + @property + def typed_print_name(self): + return self.print_name + @property def cpp_class(self): # C++ class type of the object raise RuntimeError("Can't instantiate '%s'" % self.print_name) @@ -949,6 +953,10 @@ def cpp_name(self): # C++ name of the object return (_name[0].upper() + _name[1:]) + @property + def typed_print_name(self): + return "method " + super().print_name + def Headline(self): return "'%s'%s%s" % (self.json_name, (" - " + self.summary.split(".", 1)[0]) if self.summary else "", " (DEPRECATED)" if self.deprecated else " (OBSOLETE)" if self.obsolete else "") @@ -970,6 +978,10 @@ def __init__(self, name, parent, schema, included=None): log.Info("'%s': notification parameter '%s' refers to generated JSON objects" % (name, param.name)) break + @property + def typed_print_name(self): + return "event " + super().print_name + def _Check(self): pass @@ -980,6 +992,10 @@ def __init__(self, name, parent, notification, schema, included=None): self.notification.sendif_type = JsonItem("id", self, { "type": "string", "@originalname": "index_", "@generated": True}) self.notification.sendif_deprecated = False + def typed_print_name(self): + return "callback " + super().print_name + + class JsonProperty(JsonMethod): def __init__(self, name, parent, schema, included=None): self.readonly = "readonly" in schema and schema["readonly"] @@ -1007,6 +1023,9 @@ def __init__(self, name, parent, schema, included=None): else: self.index = None + def typed_print_name(self): + return "property " + super().print_name + class JsonRpcSchema(JsonType): def __init__(self, name, schema): diff --git a/JsonGenerator/source/trackers.py b/JsonGenerator/source/trackers.py index ba28e7d0..8896ee63 100644 --- a/JsonGenerator/source/trackers.py +++ b/JsonGenerator/source/trackers.py @@ -24,7 +24,7 @@ def SortByDependency(objects): sorted_objects = [] - unique_objects = {it.cpp_class: it for it in filter(lambda o: not o.is_duplicate, objects)}.values() + unique_objects = {it.typed_print_name: it for it in filter(lambda o: not o.is_duplicate, objects)}.values() # This will order objects by their relations for obj in sorted(unique_objects, key=lambda x: x.cpp_class, reverse=False): From b7e6452d40fc3ba7ce68a6794e969983daf3ddf0 Mon Sep 17 00:00:00 2001 From: sebaszm Date: Wed, 10 Jun 2026 01:09:56 +0200 Subject: [PATCH 4/8] address comments --- JsonGenerator/source/trackers.py | 1 - ProxyStubGenerator/default.h | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/JsonGenerator/source/trackers.py b/JsonGenerator/source/trackers.py index 8896ee63..ab851632 100644 --- a/JsonGenerator/source/trackers.py +++ b/JsonGenerator/source/trackers.py @@ -16,7 +16,6 @@ # limitations under the License. import config -import logger from json_loader import * log = None diff --git a/ProxyStubGenerator/default.h b/ProxyStubGenerator/default.h index 6b303d01..f686e6f0 100644 --- a/ProxyStubGenerator/default.h +++ b/ProxyStubGenerator/default.h @@ -78,7 +78,7 @@ namespace __FRAMEWORK_NAMESPACE__ { } } - // for legacy compatibiltity only + // for legacy compatibility only namespace PluginHost { class IShell; class ISubSystem; From de1d984f1695532f0f85e84c06253794841dfeda Mon Sep 17 00:00:00 2001 From: sebaszm Date: Wed, 10 Jun 2026 01:19:38 +0200 Subject: [PATCH 5/8] add property tag --- JsonGenerator/source/json_loader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/JsonGenerator/source/json_loader.py b/JsonGenerator/source/json_loader.py index 1de34cf6..928f4450 100644 --- a/JsonGenerator/source/json_loader.py +++ b/JsonGenerator/source/json_loader.py @@ -992,6 +992,7 @@ def __init__(self, name, parent, notification, schema, included=None): self.notification.sendif_type = JsonItem("id", self, { "type": "string", "@originalname": "index_", "@generated": True}) self.notification.sendif_deprecated = False + @property def typed_print_name(self): return "callback " + super().print_name @@ -1023,6 +1024,7 @@ def __init__(self, name, parent, schema, included=None): else: self.index = None + @property def typed_print_name(self): return "property " + super().print_name From 1e79995e703e607c0677d82d01729f1fd3b49d4d Mon Sep 17 00:00:00 2001 From: sebaszm Date: Wed, 10 Jun 2026 08:27:54 +0200 Subject: [PATCH 6/8] smaller diff --- ProxyStubGenerator/StubGenerator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ProxyStubGenerator/StubGenerator.py b/ProxyStubGenerator/StubGenerator.py index 49b68aad..01f48ade 100755 --- a/ProxyStubGenerator/StubGenerator.py +++ b/ProxyStubGenerator/StubGenerator.py @@ -2739,7 +2739,6 @@ def EmitRegistration(announce_list): emit.Line("// -----------------------------------------------------------------") emit.Line("// REGISTRATION") emit.Line("// -----------------------------------------------------------------") - emit.Line() EmitRegistration(announce_list) emit.IndentDec() From 6c80230d149cf221f2dd66644ad900c1c8f0eca3 Mon Sep 17 00:00:00 2001 From: sebaszm Date: Wed, 10 Jun 2026 10:36:45 +0200 Subject: [PATCH 7/8] correct line numbering --- ProxyStubGenerator/CppParser.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ProxyStubGenerator/CppParser.py b/ProxyStubGenerator/CppParser.py index e58f4146..3eb204a6 100755 --- a/ProxyStubGenerator/CppParser.py +++ b/ProxyStubGenerator/CppParser.py @@ -2251,7 +2251,6 @@ def Parse(contents,log = None): current_line = int(token[6:].split()[0]) elif isinstance(token, str) and token.startswith("@FILE:"): current_file = token[6:] - line_numbers.append(current_line) files.append(current_file) else: tokens.append(token) @@ -2909,17 +2908,17 @@ def ReadFile(source_file, include_paths, parent_file="", index=0, inclusions=Non match = re.search(r'@(?:insert|stubgen:include)\s*"([^"]+)"', line) if match: included_file = os.path.join(os.path.dirname(os.path.realpath(file_path)), match.group(1)) - included_content = ReadFile(included_file, include_paths, file_path, i+1, inclusions, quiet=False, omit=True, use_includes=False) + included_content = ReadFile(included_file, include_paths, file_path, (i + 2), inclusions, quiet=False, omit=True, use_includes=False) file_content[i] = included_content else: match = re.search(r'@(?:insert|stubgen:include)\s*<([^>]+)>', line) if match: - included_content = ReadFile(match.group(1), include_paths, file_path, i+1, inclusions, quiet=False, omit=True, use_includes=True) + included_content = ReadFile(match.group(1), include_paths, file_path, (i + 2), inclusions, quiet=False, omit=True, use_includes=True) file_content[i] = included_content else: match = re.search(r'@insert:weak\s*<([^>]+)>', line) if match: - included_content = ReadFile(match.group(1), include_paths, file_path, i+1, inclusions, quiet=True, omit=True, use_includes=True) + included_content = ReadFile(match.group(1), include_paths, file_path, (i + 2), inclusions, quiet=True, omit=True, use_includes=True) file_content[i] = included_content contents = "// @_file:%s,1\n" % (file_path) From 323d02d0513dd499bc38743afd94011a813f6ebb Mon Sep 17 00:00:00 2001 From: sebaszm Date: Fri, 12 Jun 2026 13:21:27 +0200 Subject: [PATCH 8/8] add trace for bad request --- JsonGenerator/source/rpc_emitter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/JsonGenerator/source/rpc_emitter.py b/JsonGenerator/source/rpc_emitter.py index 520bc27f..e94273f2 100644 --- a/JsonGenerator/source/rpc_emitter.py +++ b/JsonGenerator/source/rpc_emitter.py @@ -998,6 +998,7 @@ def _Invoke(method, conditional_invoke, sorted_vars, params, response, parent="" emit.Line() emit.Line("if (%s) {" % restrictions.join()) emit.Indent() + emit.Line('TRACE_GLOBAL(Trace::Error, (_T("Invalid parameters for JSON-RPC call: %%s.%%s"), %s, %s));' % (Tstring(names.namespace), Tstring(method.name))) emit.Line("%s = %s;" % (error_code.temp_name, CoreError("bad_request"))) emit.Unindent() emit.Line("}")