diff --git a/.gitignore b/.gitignore index dc55407..73d8e9c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ zig-out/ *.o extension_api.json docs/plans/ +.worktrees/ diff --git a/build.zig b/build.zig index 7a56d22..706698a 100644 --- a/build.zig +++ b/build.zig @@ -25,6 +25,11 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }).module("zigdown"); + const zig_xml = b.dependency("xml", .{ + .target = target, + .optimize = optimize, + }).module("xml"); + const mod = b.addModule("gdoc", .{ .root_source_file = b.path("src/root.zig"), .target = target, @@ -32,6 +37,7 @@ pub fn build(b: *std.Build) void { .{ .name = "bbcodez", .module = bbcodez }, .{ .name = "known-folders", .module = known_folders }, .{ .name = "zigdown", .module = zigdown }, + .{ .name = "xml", .module = zig_xml }, }, }); mod.addOptions("build_options", build_options); @@ -67,12 +73,14 @@ pub fn build(b: *std.Build) void { }); const run_mod_tests = b.addRunArtifact(mod_tests); + run_mod_tests.setEnvironmentVariable("GDOC_NO_XML", "1"); const exe_tests = b.addTest(.{ .root_module = exe.root_module, }); const run_exe_tests = b.addRunArtifact(exe_tests); + run_exe_tests.setEnvironmentVariable("GDOC_NO_XML", "1"); const test_step = b.step("test", "Run tests"); test_step.dependOn(&run_mod_tests.step); diff --git a/build.zig.zon b/build.zig.zon index 1c4347a..ef417b3 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -20,6 +20,10 @@ .url = "https://github.com/JacobCrabill/zigdown/archive/refs/tags/v1.2.0.tar.gz", .hash = "zigdown-1.2.0-M06JT7-lFQCrQRNGBipFjSt-qAvFSLgy8-f-D4VFAeOi", }, + .xml = .{ + .url = "git+https://github.com/ianprime0509/zig-xml#8874de5b846f4e3e806a062cd11a01b2bb90fc7a", + .hash = "xml-0.2.0-ZTbP3wE6AgCsnyZut_plzxi6WB2tzzh3kFRBOp3AL7n9", + }, }, .paths = .{ "build.zig", diff --git a/snapshots/class_with_tutorials.md b/snapshots/class_with_tutorials.md new file mode 100644 index 0000000..ccc6a1a --- /dev/null +++ b/snapshots/class_with_tutorials.md @@ -0,0 +1,12 @@ +# Sprite2D + +General-purpose sprite node. + +## Description + +A node that displays a 2D texture. + +## Tutorials + +- [Custom drawing in 2D](https://docs.godotengine.org/en/stable/tutorials/2d/custom_drawing_in_2d.html) +- [All 2D Demos](https://github.com/godotengine/godot-demo-projects/tree/master/2d) diff --git a/src/DocDatabase.zig b/src/DocDatabase.zig index 5257c34..ce27bf3 100644 --- a/src/DocDatabase.zig +++ b/src/DocDatabase.zig @@ -21,6 +21,11 @@ pub const EntryKind = enum { signal, }; +pub const Tutorial = struct { + title: []const u8, + url: []const u8, +}; + pub const Entry = struct { key: []const u8, name: []const u8, @@ -30,6 +35,7 @@ pub const Entry = struct { brief_description: ?[]const u8 = null, signature: ?[]const u8 = null, members: ?[]usize = null, + tutorials: ?[]const Tutorial = null, }; const RootState = enum { @@ -421,6 +427,15 @@ fn generateMarkdownForEntry(self: DocDatabase, allocator: Allocator, entry: Entr try writer.print("\n## Description\n\n{s}\n", .{desc}); } + if (entry.tutorials) |tutorials| { + if (tutorials.len > 0) { + try writer.writeAll("\n## Tutorials\n\n"); + for (tutorials) |tutorial| { + try writer.print("- [{s}]({s})\n", .{ tutorial.title, tutorial.url }); + } + } + } + if (entry.members) |member_indices| { try self.generateMemberListings(allocator, member_indices, writer); } @@ -1263,6 +1278,41 @@ test "generateMarkdownForSymbol for class with members" { try writer.flush(); } +test "generateMarkdownForSymbol for class with tutorials" { + const allocator = std.testing.allocator; + + var db = DocDatabase{ + .symbols = StringArrayHashMap(Entry).empty, + }; + defer db.symbols.deinit(allocator); + + const tutorials = [_]Tutorial{ + .{ .title = "Custom drawing in 2D", .url = "https://docs.godotengine.org/en/stable/tutorials/2d/custom_drawing_in_2d.html" }, + .{ .title = "All 2D Demos", .url = "https://github.com/godotengine/godot-demo-projects/tree/master/2d" }, + }; + + const entry = Entry{ + .key = "Sprite2D", + .name = "Sprite2D", + .kind = .class, + .brief_description = "General-purpose sprite node.", + .description = "A node that displays a 2D texture.", + .tutorials = &tutorials, + }; + try db.symbols.put(allocator, "Sprite2D", entry); + + // Write snapshot + var file = try std.fs.cwd().createFile("snapshots/class_with_tutorials.md", .{}); + defer file.close(); + + var buf: [4096]u8 = undefined; + var file_writer = file.writer(&buf); + const writer = &file_writer.interface; + + try db.generateMarkdownForSymbol(allocator, "Sprite2D", writer); + try writer.flush(); +} + const std = @import("std"); const ArenaAllocator = std.heap.ArenaAllocator; const Allocator = std.mem.Allocator; diff --git a/src/Spinner.zig b/src/Spinner.zig new file mode 100644 index 0000000..063b5fd --- /dev/null +++ b/src/Spinner.zig @@ -0,0 +1,42 @@ +const std = @import("std"); + +const Spinner = @This(); + +const frames = [_][]const u8{ "⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏" }; +const delay_ns = 80 * std.time.ns_per_ms; + +message: []const u8, +thread: ?std.Thread = null, +stop: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), + +pub fn start(self: *Spinner) void { + self.stop.store(false, .release); + self.thread = std.Thread.spawn(.{}, run, .{self}) catch return; +} + +pub fn finish(self: *Spinner) void { + if (self.thread == null) return; + + self.stop.store(true, .release); + self.thread.?.join(); + self.thread = null; + + const stderr = std.fs.File.stderr(); + var buf: [256]u8 = undefined; + var w = stderr.writer(&buf); + w.interface.writeAll("\r\x1b[2K") catch {}; + w.interface.flush() catch {}; +} + +fn run(self: *Spinner) void { + const stderr = std.fs.File.stderr(); + var buf: [256]u8 = undefined; + var w = stderr.writer(&buf); + var i: usize = 0; + while (!self.stop.load(.acquire)) { + w.interface.print("\r{s} {s}", .{ frames[i], self.message }) catch return; + w.interface.flush() catch return; + i = (i + 1) % frames.len; + std.Thread.sleep(delay_ns); + } +} diff --git a/src/XmlDocParser.zig b/src/XmlDocParser.zig new file mode 100644 index 0000000..155c953 --- /dev/null +++ b/src/XmlDocParser.zig @@ -0,0 +1,336 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const xml = @import("xml"); + +const docs_url = "https://docs.godotengine.org/en/stable"; + +pub const Tutorial = struct { + title: []const u8, + url: []const u8, +}; + +pub const MemberDoc = struct { + name: []const u8, + description: ?[]const u8 = null, +}; + +pub const ClassDoc = struct { + name: []const u8, + inherits: ?[]const u8 = null, + brief_description: ?[]const u8 = null, + description: ?[]const u8 = null, + tutorials: ?[]Tutorial = null, + methods: ?[]MemberDoc = null, + properties: ?[]MemberDoc = null, + signals: ?[]MemberDoc = null, + constants: ?[]MemberDoc = null, +}; + +pub const ParseError = error{ + MalformedXml, + UnexpectedElement, + MissingClassElement, + MissingNameAttribute, + OutOfMemory, + ReadFailed, +}; + +pub fn parseClassDoc(allocator: Allocator, xml_content: []const u8) ParseError!ClassDoc { + var static_reader: xml.Reader.Static = .init(allocator, xml_content, .{ + .namespace_aware = false, + }); + defer static_reader.deinit(); + const reader = &static_reader.interface; + + var doc: ClassDoc = .{ .name = "" }; + var tutorials: std.ArrayListUnmanaged(Tutorial) = .empty; + defer tutorials.deinit(allocator); + var methods: std.ArrayListUnmanaged(MemberDoc) = .empty; + defer methods.deinit(allocator); + var properties: std.ArrayListUnmanaged(MemberDoc) = .empty; + defer properties.deinit(allocator); + var signals: std.ArrayListUnmanaged(MemberDoc) = .empty; + defer signals.deinit(allocator); + var constants: std.ArrayListUnmanaged(MemberDoc) = .empty; + defer constants.deinit(allocator); + + var found_class = false; + + while (true) { + const node = reader.read() catch return ParseError.MalformedXml; + switch (node) { + .eof => break, + .xml_declaration => continue, + .element_start => { + const name = reader.elementName(); + if (std.mem.eql(u8, name, "class")) { + found_class = true; + doc.name = try getAttributeAlloc(allocator, reader, "name") orelse return ParseError.MissingNameAttribute; + doc.inherits = try getAttributeAlloc(allocator, reader, "inherits"); + } else if (std.mem.eql(u8, name, "brief_description")) { + doc.brief_description = try readTextContent(allocator, reader); + } else if (std.mem.eql(u8, name, "description") and found_class) { + doc.description = try readTextContent(allocator, reader); + } else if (std.mem.eql(u8, name, "link")) { + const title = try getAttributeAlloc(allocator, reader, "title") orelse try allocator.dupe(u8, ""); + const url_raw = try readTextContent(allocator, reader) orelse try allocator.dupe(u8, ""); + const url = try expandDocsUrl(allocator, url_raw); + if (url.ptr != url_raw.ptr) { + allocator.free(url_raw); + } + try tutorials.append(allocator,.{ .title = title, .url = url }); + } else if (std.mem.eql(u8, name, "method")) { + const method_name = try getAttributeAlloc(allocator, reader, "name") orelse continue; + const desc = try readNestedDescription(allocator, reader, "method"); + try methods.append(allocator,.{ .name = method_name, .description = desc }); + } else if (std.mem.eql(u8, name, "member")) { + const member_name = try getAttributeAlloc(allocator, reader, "name") orelse continue; + const desc = try readTextContent(allocator, reader); + try properties.append(allocator,.{ .name = member_name, .description = desc }); + } else if (std.mem.eql(u8, name, "signal")) { + const signal_name = try getAttributeAlloc(allocator, reader, "name") orelse continue; + const desc = try readNestedDescription(allocator, reader, "signal"); + try signals.append(allocator,.{ .name = signal_name, .description = desc }); + } else if (std.mem.eql(u8, name, "constant")) { + const constant_name = try getAttributeAlloc(allocator, reader, "name") orelse continue; + const desc = try readTextContent(allocator, reader); + try constants.append(allocator,.{ .name = constant_name, .description = desc }); + } + }, + else => continue, + } + } + + if (!found_class) return ParseError.MissingClassElement; + + doc.tutorials = if (tutorials.items.len > 0) try tutorials.toOwnedSlice(allocator) else null; + doc.methods = if (methods.items.len > 0) try methods.toOwnedSlice(allocator) else null; + doc.properties = if (properties.items.len > 0) try properties.toOwnedSlice(allocator) else null; + doc.signals = if (signals.items.len > 0) try signals.toOwnedSlice(allocator) else null; + doc.constants = if (constants.items.len > 0) try constants.toOwnedSlice(allocator) else null; + + return doc; +} + +pub fn freeClassDoc(allocator: Allocator, doc: ClassDoc) void { + allocator.free(doc.name); + if (doc.inherits) |s| allocator.free(s); + if (doc.brief_description) |s| allocator.free(s); + if (doc.description) |s| allocator.free(s); + + if (doc.tutorials) |tutorials| { + for (tutorials) |t| { + allocator.free(t.title); + allocator.free(t.url); + } + allocator.free(tutorials); + } + + inline for (.{ "methods", "properties", "signals", "constants" }) |field| { + if (@field(doc, field)) |members| { + for (members) |m| { + allocator.free(m.name); + if (m.description) |d| allocator.free(d); + } + allocator.free(members); + } + } +} + +fn getAttributeAlloc(allocator: Allocator, reader: *xml.Reader, name: []const u8) Allocator.Error!?[]const u8 { + const idx = reader.attributeIndex(name) orelse return null; + return try reader.attributeValueAlloc(allocator, idx); +} + +fn readTextContent(allocator: Allocator, reader: *xml.Reader) ParseError!?[]const u8 { + var text_buf: std.Io.Writer.Allocating = .init(allocator); + defer text_buf.deinit(); + + var depth: usize = 1; + while (depth > 0) { + const node = reader.read() catch return ParseError.MalformedXml; + switch (node) { + .eof => break, + .element_start => depth += 1, + .element_end => depth -= 1, + .text => { + text_buf.writer.writeAll(reader.textRaw()) catch return ParseError.OutOfMemory; + }, + else => continue, + } + } + + const written = text_buf.written(); + if (written.len == 0) return null; + + const trimmed = std.mem.trim(u8, written, " \t\r\n"); + if (trimmed.len == 0) return null; + + return try allocator.dupe(u8, trimmed); +} + +fn readNestedDescription(allocator: Allocator, reader: *xml.Reader, container_element: []const u8) ParseError!?[]const u8 { + // Read through the container element looking for a nested element. + var depth: usize = 1; + while (depth > 0) { + const node = reader.read() catch return ParseError.MalformedXml; + switch (node) { + .eof => break, + .element_start => { + const name = reader.elementName(); + if (depth == 1 and std.mem.eql(u8, name, "description")) { + return try readTextContent(allocator, reader); + } + depth += 1; + }, + .element_end => { + const name = reader.elementName(); + if (depth == 1 and std.mem.eql(u8, name, container_element)) { + break; + } + depth -= 1; + }, + else => continue, + } + } + return null; +} + +fn expandDocsUrl(allocator: Allocator, url: []const u8) Allocator.Error![]const u8 { + const prefix = "$DOCS_URL"; + if (std.mem.startsWith(u8, url, prefix)) { + return try std.fmt.allocPrint(allocator, "{s}{s}", .{ docs_url, url[prefix.len..] }); + } + return url; +} + +// Tests +const test_xml = + \\ + \\ + \\ A 2D game object. + \\ Node2D is the base class for 2D. + \\ + \\ $DOCS_URL/tutorials/2d/custom_drawing.html + \\ https://github.com/godotengine/godot-demo-projects/tree/master/2d + \\ + \\ + \\ + \\ + \\ + \\ Multiplies the current scale by the ratio vector. + \\ + \\ + \\ + \\ + \\Position, relative to the node's parent. + \\ + \\ + \\ + \\ + \\ Emitted when something happens. + \\ + \\ + \\ + \\ + \\Maximum allowed value. + \\ + \\ + \\ +; + +test "parses class name and inherits" { + const allocator = std.testing.allocator; + const doc = try parseClassDoc(allocator, test_xml); + defer freeClassDoc(allocator, doc); + + try std.testing.expectEqualStrings("Node2D", doc.name); + try std.testing.expectEqualStrings("CanvasItem", doc.inherits.?); +} + +test "parses brief_description and description" { + const allocator = std.testing.allocator; + const doc = try parseClassDoc(allocator, test_xml); + defer freeClassDoc(allocator, doc); + + try std.testing.expectEqualStrings("A 2D game object.", doc.brief_description.?); + try std.testing.expectEqualStrings("Node2D is the base class for 2D.", doc.description.?); +} + +test "parses tutorials with $DOCS_URL expansion" { + const allocator = std.testing.allocator; + const doc = try parseClassDoc(allocator, test_xml); + defer freeClassDoc(allocator, doc); + + const tutorials = doc.tutorials.?; + try std.testing.expectEqual(2, tutorials.len); + try std.testing.expectEqualStrings("Custom drawing in 2D", tutorials[0].title); + try std.testing.expectEqualStrings( + "https://docs.godotengine.org/en/stable/tutorials/2d/custom_drawing.html", + tutorials[0].url, + ); +} + +test "external tutorial URLs left unchanged" { + const allocator = std.testing.allocator; + const doc = try parseClassDoc(allocator, test_xml); + defer freeClassDoc(allocator, doc); + + const tutorials = doc.tutorials.?; + try std.testing.expectEqualStrings("All 2D Demos", tutorials[1].title); + try std.testing.expectEqualStrings( + "https://github.com/godotengine/godot-demo-projects/tree/master/2d", + tutorials[1].url, + ); +} + +test "parses methods with descriptions" { + const allocator = std.testing.allocator; + const doc = try parseClassDoc(allocator, test_xml); + defer freeClassDoc(allocator, doc); + + const methods = doc.methods.?; + try std.testing.expectEqual(1, methods.len); + try std.testing.expectEqualStrings("apply_scale", methods[0].name); + try std.testing.expectEqualStrings("Multiplies the current scale by the ratio vector.", methods[0].description.?); +} + +test "parses properties from members element" { + const allocator = std.testing.allocator; + const doc = try parseClassDoc(allocator, test_xml); + defer freeClassDoc(allocator, doc); + + const props = doc.properties.?; + try std.testing.expectEqual(1, props.len); + try std.testing.expectEqualStrings("position", props[0].name); + try std.testing.expectEqualStrings("Position, relative to the node's parent.", props[0].description.?); +} + +test "parses signals with descriptions" { + const allocator = std.testing.allocator; + const doc = try parseClassDoc(allocator, test_xml); + defer freeClassDoc(allocator, doc); + + const sigs = doc.signals.?; + try std.testing.expectEqual(1, sigs.len); + try std.testing.expectEqualStrings("some_signal", sigs[0].name); + try std.testing.expectEqualStrings("Emitted when something happens.", sigs[0].description.?); +} + +test "parses constants with descriptions" { + const allocator = std.testing.allocator; + const doc = try parseClassDoc(allocator, test_xml); + defer freeClassDoc(allocator, doc); + + const consts = doc.constants.?; + try std.testing.expectEqual(1, consts.len); + try std.testing.expectEqualStrings("MAX_VALUE", consts[0].name); + try std.testing.expectEqualStrings("Maximum allowed value.", consts[0].description.?); +} + +test "freeClassDoc doesn't leak" { + const allocator = std.testing.allocator; + const doc = try parseClassDoc(allocator, test_xml); + freeClassDoc(allocator, doc); + // testing allocator will catch leaks +} diff --git a/src/cache.zig b/src/cache.zig index b7592e2..a2232aa 100644 --- a/src/cache.zig +++ b/src/cache.zig @@ -144,6 +144,26 @@ pub fn cacheIsPopulated(allocator: Allocator, cache_path: []const u8) !bool { return true; } +pub fn getXmlDocsDirInCache(allocator: Allocator, cache_dir: []const u8) ![]const u8 { + return std.fmt.allocPrint( + allocator, + "{f}", + .{std.fs.path.fmtJoin(&[_][]const u8{ cache_dir, "xml_docs" })}, + ); +} + +pub fn xmlDocsArePopulated(allocator: Allocator, cache_dir: []const u8) !bool { + const xml_dir = try getXmlDocsDirInCache(allocator, cache_dir); + defer allocator.free(xml_dir); + + const marker = source_fetch.readCompleteMarker(allocator, xml_dir); + if (marker) |m| { + allocator.free(m); + return true; + } + return false; +} + test "getCacheDir returns cache directory path" { const allocator = std.testing.allocator; @@ -744,3 +764,4 @@ const Writer = std.Io.Writer; const known_folders = @import("known-folders"); const DocDatabase = @import("DocDatabase.zig"); +const source_fetch = @import("source_fetch.zig"); diff --git a/src/root.zig b/src/root.zig index fb7a348..ab8d645 100644 --- a/src/root.zig +++ b/src/root.zig @@ -35,17 +35,31 @@ pub fn markdownForSymbol(allocator: Allocator, symbol: []const u8, api_json_path const cache_path = try cache.getCacheDir(allocator); defer allocator.free(cache_path); - if (!try cache.cacheIsPopulated(allocator, cache_path)) { + const needs_full_rebuild = !try cache.cacheIsPopulated(allocator, cache_path); + + if (needs_full_rebuild) { try cache.ensureDirectoryExists(cache_path); try api.generateApiJsonIfNotExists(allocator, "godot", cache_path); + // Fetch XML docs if missing (best-effort, requires godot) + if (!try cache.xmlDocsArePopulated(allocator, cache_path)) { + fetchXmlDocs(allocator, cache_path); + } + + var spinner: Spinner = .{ .message = "Building documentation cache..." }; + if (!xmlSupplementationDisabled()) spinner.start(); + defer spinner.finish(); + const json_path = try cache.getJsonCachePathInDir(allocator, cache_path); defer allocator.free(json_path); const json_file = try std.fs.openFileAbsolute(json_path, .{}); defer json_file.close(); - const db = try DocDatabase.loadFromJsonFileLeaky(arena.allocator(), json_file); + var db = try DocDatabase.loadFromJsonFileLeaky(arena.allocator(), json_file); + + mergeXmlDocs(arena.allocator(), allocator, &db, cache_path); + try cache.generateMarkdownCache(allocator, db, cache_path); } @@ -401,6 +415,156 @@ test "markdownForSymbol generates markdown cache when cache is empty" { cache.clearCache(allocator) catch {}; } +fn xmlSupplementationDisabled() bool { + var buf: [256]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&buf); + return std.process.hasEnvVar(fba.allocator(), "GDOC_NO_XML") catch false; +} + +fn fetchXmlDocs(allocator: Allocator, cache_path: []const u8) void { + if (xmlSupplementationDisabled()) return; + + const xml_dir = cache.getXmlDocsDirInCache(allocator, cache_path) catch return; + defer allocator.free(xml_dir); + + cache.ensureDirectoryExists(xml_dir) catch return; + + const version = source_fetch.getGodotVersion(allocator) orelse return; + defer version.deinit(allocator); + + var url_buf: [256]u8 = undefined; + const url = source_fetch.buildTarballUrl(&url_buf, version) orelse return; + + var spinner = Spinner{ .message = "Downloading XML docs..." }; + spinner.start(); + defer spinner.finish(); + + source_fetch.fetchAndExtractXmlDocs(allocator, url, xml_dir) catch |err| { + // Try hash-based fallback URL + if (version.hash) |hash| { + var hash_url_buf: [256]u8 = undefined; + const hash_url = source_fetch.buildTarballUrlFromHash(&hash_url_buf, hash) orelse return; + source_fetch.fetchAndExtractXmlDocs(allocator, hash_url, xml_dir) catch { + std.log.warn("XML doc fetch failed ({}), proceeding without XML supplementation", .{err}); + return; + }; + } else { + std.log.warn("XML doc fetch failed ({}), proceeding without XML supplementation", .{err}); + return; + } + }; + + var version_buf: [64]u8 = undefined; + const version_str = version.formatVersion(&version_buf) orelse return; + + source_fetch.writeCompleteMarker(allocator, xml_dir, version_str) catch return; +} + +fn mergeXmlDocs(arena_allocator: Allocator, tmp_allocator: Allocator, db: *DocDatabase, cache_path: []const u8) void { + if (xmlSupplementationDisabled()) return; + + const xml_dir = cache.getXmlDocsDirInCache(tmp_allocator, cache_path) catch return; + defer tmp_allocator.free(xml_dir); + + var dir = std.fs.openDirAbsolute(xml_dir, .{ .iterate = true }) catch return; + defer dir.close(); + + var iter = dir.iterate(); + while (iter.next() catch return) |entry| { + if (entry.kind != .file) continue; + if (!std.mem.endsWith(u8, entry.name, ".xml")) continue; + + const class_name = entry.name[0 .. entry.name.len - 4]; // strip .xml + + // Read XML file + const content = dir.readFileAlloc(tmp_allocator, entry.name, 2 * 1024 * 1024) catch continue; + defer tmp_allocator.free(content); + + // Parse with arena_allocator so strings outlive this function + const class_doc = XmlDocParser.parseClassDoc(arena_allocator, content) catch |err| { + std.log.warn("failed to parse XML doc for {s}: {}", .{ class_name, err }); + continue; + }; + // Do NOT freeClassDoc -- arena owns the memory + + // Merge tutorials + if (class_doc.tutorials) |tutorials| { + if (tutorials.len > 0) { + if (db.symbols.getPtr(class_name)) |db_entry| { + if (db_entry.tutorials == null) { + const db_tutorials = arena_allocator.alloc(DocDatabase.Tutorial, tutorials.len) catch continue; + for (tutorials, 0..) |t, i| { + db_tutorials[i] = .{ .title = t.title, .url = t.url }; + } + db_entry.tutorials = db_tutorials; + } + } + } + } + + // Fill missing class description + if (class_doc.description) |xml_desc| { + if (db.symbols.getPtr(class_name)) |db_entry| { + if (db_entry.description == null) { + db_entry.description = xml_desc; + } + } + } + + // Merge member descriptions (methods, properties, signals) + if (class_doc.methods) |members| { + for (members) |member| { + const member_key = std.fmt.allocPrint(tmp_allocator, "{s}.{s}", .{ class_name, member.name }) catch continue; + defer tmp_allocator.free(member_key); + + if (db.symbols.getPtr(member_key)) |db_entry| { + if (db_entry.description == null) { + db_entry.description = member.description; + } + } + } + } + + if (class_doc.properties) |members| { + for (members) |member| { + const member_key = std.fmt.allocPrint(tmp_allocator, "{s}.{s}", .{ class_name, member.name }) catch continue; + defer tmp_allocator.free(member_key); + + if (db.symbols.getPtr(member_key)) |db_entry| { + if (db_entry.description == null) { + db_entry.description = member.description; + } + } + } + } + + if (class_doc.signals) |members| { + for (members) |member| { + const member_key = std.fmt.allocPrint(tmp_allocator, "{s}.{s}", .{ class_name, member.name }) catch continue; + defer tmp_allocator.free(member_key); + + if (db.symbols.getPtr(member_key)) |db_entry| { + if (db_entry.description == null) { + db_entry.description = member.description; + } + } + } + } + + // Add entries for classes found in XML but not in JSON + if (db.symbols.get(class_name) == null) { + const key = std.fmt.allocPrint(arena_allocator, "{s}", .{class_name}) catch continue; + db.symbols.put(arena_allocator, key, .{ + .key = key, + .name = key, + .kind = .class, + .description = class_doc.description, + .brief_description = class_doc.brief_description, + }) catch continue; + } + } +} + comptime { std.testing.refAllDecls(@This()); } @@ -415,8 +579,11 @@ const AllocatingWriter = Writer.Allocating; const known_folders = @import("known-folders"); pub const DocDatabase = @import("DocDatabase.zig"); +pub const XmlDocParser = @import("XmlDocParser.zig"); pub const cache = @import("cache.zig"); pub const api = @import("api.zig"); +pub const source_fetch = @import("source_fetch.zig"); +const Spinner = @import("Spinner.zig"); const zigdown = @import("zigdown"); const ConsoleRenderer = zigdown.ConsoleRenderer; diff --git a/src/source_fetch.zig b/src/source_fetch.zig new file mode 100644 index 0000000..74ba4a4 --- /dev/null +++ b/src/source_fetch.zig @@ -0,0 +1,372 @@ +pub const VersionInfo = struct { + major: []const u8, + minor: []const u8, + patch: []const u8, + hash: ?[]const u8, + /// When non-null, this is the allocated buffer that backs major/minor/patch/hash. + /// Call `deinit(allocator)` to free it. + backing: ?[]u8 = null, + + pub fn deinit(self: VersionInfo, allocator: Allocator) void { + if (self.backing) |buf| allocator.free(buf); + } + + /// Formats the version as "major.minor.patch" into the provided buffer. + pub fn formatVersion(self: VersionInfo, buf: []u8) ?[]const u8 { + return std.fmt.bufPrint(buf, "{s}.{s}.{s}", .{ self.major, self.minor, self.patch }) catch null; + } +}; + +/// Parses a Godot version string like "4.6.1.stable.official.14d19694e". +/// Returns null for empty or malformed strings. +/// The returned slices point into `version_str`; the caller must ensure it outlives the result. +pub fn parseGodotVersion(version_str: []const u8) ?VersionInfo { + if (version_str.len == 0) return null; + + var it = std.mem.splitScalar(u8, version_str, '.'); + + const major = it.next() orelse return null; + const minor = it.next() orelse return null; + const patch = it.next() orelse return null; + + // Validate that major, minor, patch are numeric + for (major) |c| if (!std.ascii.isDigit(c)) return null; + for (minor) |c| if (!std.ascii.isDigit(c)) return null; + for (patch) |c| if (!std.ascii.isDigit(c)) return null; + + if (major.len == 0 or minor.len == 0 or patch.len == 0) return null; + + // 4th segment: stability label (stable/dev/beta/rc) - skip + _ = it.next() orelse return null; + + // 5th segment: build type + const build_type = it.next() orelse return null; + + // 6th segment: commit hash (only if build_type is "official") + const hash: ?[]const u8 = if (std.mem.eql(u8, build_type, "official")) + it.next() + else + null; + + return VersionInfo{ + .major = major, + .minor = minor, + .patch = patch, + .hash = hash, + }; +} + +/// Runs the godot executable at `godot_path` with `--version` and parses the output. +/// Returns null if the process fails or the output is malformed. +/// The returned VersionInfo owns its backing buffer; call `result.deinit(allocator)` when done. +pub fn getGodotVersionFromPath(allocator: Allocator, godot_path: []const u8) ?VersionInfo { + const result = std.process.Child.run(.{ + .argv = &.{ godot_path, "--version" }, + .allocator = allocator, + }) catch return null; + defer allocator.free(result.stdout); + defer allocator.free(result.stderr); + + switch (result.term) { + .Exited => |code| if (code != 0) return null, + else => return null, + } + + const trimmed = std.mem.trimRight(u8, result.stdout, &std.ascii.whitespace); + const owned = allocator.dupe(u8, trimmed) catch return null; + var info = parseGodotVersion(owned) orelse { + allocator.free(owned); + return null; + }; + info.backing = owned; + return info; +} + +/// Convenience wrapper that calls `getGodotVersionFromPath` with "godot" from PATH. +pub fn getGodotVersion(allocator: Allocator) ?VersionInfo { + return getGodotVersionFromPath(allocator, "godot"); +} + +test "parseGodotVersion: standard version with hash" { + const result = parseGodotVersion("4.6.1.stable.official.14d19694e"); + try std.testing.expect(result != null); + const v = result.?; + try std.testing.expectEqualStrings("4", v.major); + try std.testing.expectEqualStrings("6", v.minor); + try std.testing.expectEqualStrings("1", v.patch); + try std.testing.expectEqualStrings("14d19694e", v.hash.?); +} + +test "parseGodotVersion: custom build without hash" { + const result = parseGodotVersion("4.6.1.stable.custom_build"); + try std.testing.expect(result != null); + const v = result.?; + try std.testing.expectEqualStrings("4", v.major); + try std.testing.expectEqualStrings("6", v.minor); + try std.testing.expectEqualStrings("1", v.patch); + try std.testing.expect(v.hash == null); +} + +test "parseGodotVersion: dev build with hash" { + const result = parseGodotVersion("4.7.0.dev.official.abc123def"); + try std.testing.expect(result != null); + const v = result.?; + try std.testing.expectEqualStrings("4", v.major); + try std.testing.expectEqualStrings("7", v.minor); + try std.testing.expectEqualStrings("0", v.patch); + try std.testing.expectEqualStrings("abc123def", v.hash.?); +} + +test "parseGodotVersion: empty string returns null" { + const result = parseGodotVersion(""); + try std.testing.expect(result == null); +} + +test "parseGodotVersion: malformed string returns null" { + const result = parseGodotVersion("not-a-version"); + try std.testing.expect(result == null); +} + +test "VersionInfo.formatVersion produces correct output" { + const v = VersionInfo{ + .major = "4", + .minor = "6", + .patch = "1", + .hash = "14d19694e", + }; + var buf: [32]u8 = undefined; + const formatted = v.formatVersion(&buf); + try std.testing.expect(formatted != null); + try std.testing.expectEqualStrings("4.6.1", formatted.?); +} + +/// Builds a tarball URL for a specific Godot version tag. +/// Example: version 4.6.1 -> "https://github.com/godotengine/godot/archive/refs/tags/4.6.1-stable.tar.gz" +pub fn buildTarballUrl(buf: []u8, version: VersionInfo) ?[]const u8 { + return std.fmt.bufPrint(buf, "https://github.com/godotengine/godot/archive/refs/tags/{s}.{s}.{s}-stable.tar.gz", .{ + version.major, version.minor, version.patch, + }) catch null; +} + +/// Builds a tarball URL from a commit hash. +/// Example: hash "14d19694e" -> "https://github.com/godotengine/godot/archive/14d19694e.tar.gz" +pub fn buildTarballUrlFromHash(buf: []u8, hash: []const u8) ?[]const u8 { + return std.fmt.bufPrint(buf, "https://github.com/godotengine/godot/archive/{s}.tar.gz", .{hash}) catch null; +} + +/// Downloads a .tar.gz from a URL and extracts only XML doc files. +/// Matches: +/// - `*/doc/classes/*.xml` (core class docs) +/// - `*/modules/*/doc_classes/*.xml` (module class docs) +/// +/// Extracted files are written to `xml_docs_dir` with their basename only. +pub fn fetchAndExtractXmlDocs(allocator: Allocator, url: []const u8, xml_docs_dir: []const u8) !void { + cache.ensureDirectoryExists(xml_docs_dir) catch {}; + + var client: std.http.Client = .{ .allocator = allocator }; + defer client.deinit(); + + const uri = std.Uri.parse(url) catch return error.InvalidUrl; + + var redirect_buf: [2048]u8 = undefined; + var req = try client.request(.GET, uri, .{ + .redirect_behavior = std.http.Client.Request.RedirectBehavior.init(10), + }); + defer req.deinit(); + + try req.sendBodiless(); + + var response = try req.receiveHead(&redirect_buf); + + if (response.head.status != .ok) return error.HttpRequestFailed; + + // Get the raw HTTP response body reader + var transfer_buf: [8192]u8 = undefined; + const http_reader: *std.Io.Reader = response.reader(&transfer_buf); + + // Decompress gzip + var decompress_buf: [std.compress.flate.max_window_len]u8 = undefined; + var decompressor = std.compress.flate.Decompress.init(http_reader, .gzip, &decompress_buf); + + // Iterate tar entries + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + var link_buf: [std.fs.max_path_bytes]u8 = undefined; + var tar_iter = std.tar.Iterator.init(&decompressor.reader, .{ + .file_name_buffer = &path_buf, + .link_name_buffer = &link_buf, + .diagnostics = null, + }); + + var dir = try std.fs.openDirAbsolute(xml_docs_dir, .{}); + defer dir.close(); + + while (try tar_iter.next()) |file| { + if (file.kind != .file) continue; + + const name = file.name; + if (!std.mem.endsWith(u8, name, ".xml")) continue; + + // Match */doc/classes/*.xml or */modules/*/doc_classes/*.xml + const is_core_doc = matchesPattern(name, "/doc/classes/"); + const is_module_doc = matchesPattern(name, "/doc_classes/"); + + if (!is_core_doc and !is_module_doc) continue; + + // Extract basename + const basename = std.fs.path.basename(name); + + var out_file = dir.createFile(basename, .{}) catch continue; + defer out_file.close(); + + var write_buf: [4096]u8 = undefined; + var file_writer = out_file.writer(&write_buf); + tar_iter.streamRemaining(file, &file_writer.interface) catch continue; + file_writer.interface.flush() catch continue; + } +} + +fn matchesPattern(path: []const u8, pattern: []const u8) bool { + return std.mem.indexOf(u8, path, pattern) != null; +} + +/// Writes a version string to `xml_docs_dir/.complete` as a cache marker. +pub fn writeCompleteMarker(allocator: Allocator, xml_docs_dir: []const u8, version_str: []const u8) !void { + const marker_path = try std.fmt.allocPrint(allocator, "{f}", .{ + std.fs.path.fmtJoin(&[_][]const u8{ xml_docs_dir, ".complete" }), + }); + defer allocator.free(marker_path); + + var file = try std.fs.createFileAbsolute(marker_path, .{}); + defer file.close(); + + var buf: [4096]u8 = undefined; + var file_writer = file.writer(&buf); + var writer = &file_writer.interface; + + try writer.writeAll(version_str); + try writer.flush(); +} + +/// Reads the content of `xml_docs_dir/.complete`, or returns null if not present. +pub fn readCompleteMarker(allocator: Allocator, xml_docs_dir: []const u8) ?[]const u8 { + const marker_path = std.fmt.allocPrint(allocator, "{f}", .{ + std.fs.path.fmtJoin(&[_][]const u8{ xml_docs_dir, ".complete" }), + }) catch return null; + defer allocator.free(marker_path); + + const file = std.fs.openFileAbsolute(marker_path, .{}) catch return null; + defer file.close(); + + var buf: [4096]u8 = undefined; + var file_reader = file.reader(&buf); + var reader = &file_reader.interface; + + var allocating: std.Io.Writer.Allocating = .init(allocator); + errdefer allocating.deinit(); + + _ = reader.stream(&allocating.writer, .unlimited) catch return null; + + const result = allocating.toOwnedSlice() catch return null; + if (result.len == 0) { + allocator.free(result); + return null; + } + + return result; +} + +test "getGodotVersionFromPath with fake godot script" { + const allocator = std.testing.allocator; + + var tmp_dir = std.testing.tmpDir(.{}); + defer tmp_dir.cleanup(); + + const tmp_path = try tmp_dir.dir.realpathAlloc(allocator, "."); + defer allocator.free(tmp_path); + + const script = "#!/bin/sh\necho '4.6.1.stable.official.14d19694e'"; + try tmp_dir.dir.writeFile(.{ .sub_path = "fake-godot", .data = script }); + + var file = try tmp_dir.dir.openFile("fake-godot", .{}); + try file.chmod(0o755); + file.close(); + + const fake_path = try std.fmt.allocPrint(allocator, "{s}/fake-godot", .{tmp_path}); + defer allocator.free(fake_path); + + const result = getGodotVersionFromPath(allocator, fake_path); + try std.testing.expect(result != null); + defer result.?.deinit(allocator); + try std.testing.expectEqualStrings("14d19694e", result.?.hash.?); +} + +test "buildTarballUrl with version 4.6.1" { + var buf: [256]u8 = undefined; + const url = buildTarballUrl(&buf, .{ + .major = "4", + .minor = "6", + .patch = "1", + .hash = null, + }); + try std.testing.expect(url != null); + try std.testing.expectEqualStrings( + "https://github.com/godotengine/godot/archive/refs/tags/4.6.1-stable.tar.gz", + url.?, + ); +} + +test "buildTarballUrlFromHash with commit hash" { + var buf: [256]u8 = undefined; + const url = buildTarballUrlFromHash(&buf, "14d19694e"); + try std.testing.expect(url != null); + try std.testing.expectEqualStrings( + "https://github.com/godotengine/godot/archive/14d19694e.tar.gz", + url.?, + ); +} + +test "buildTarballUrl returns null when buffer too small" { + var buf: [10]u8 = undefined; + const url = buildTarballUrl(&buf, .{ + .major = "4", + .minor = "6", + .patch = "1", + .hash = null, + }); + try std.testing.expect(url == null); +} + +test "writeCompleteMarker and readCompleteMarker round-trip" { + const allocator = std.testing.allocator; + + var tmp_dir = std.testing.tmpDir(.{}); + defer tmp_dir.cleanup(); + + const tmp_path = try tmp_dir.dir.realpathAlloc(allocator, "."); + defer allocator.free(tmp_path); + + const version_str = "4.6.1"; + try writeCompleteMarker(allocator, tmp_path, version_str); + + const result = readCompleteMarker(allocator, tmp_path); + try std.testing.expect(result != null); + defer allocator.free(result.?); + try std.testing.expectEqualStrings(version_str, result.?); +} + +test "readCompleteMarker returns null for non-existent marker" { + const allocator = std.testing.allocator; + + var tmp_dir = std.testing.tmpDir(.{}); + defer tmp_dir.cleanup(); + + const tmp_path = try tmp_dir.dir.realpathAlloc(allocator, "."); + defer allocator.free(tmp_path); + + const result = readCompleteMarker(allocator, tmp_path); + try std.testing.expect(result == null); +} + +const Allocator = std.mem.Allocator; +const std = @import("std"); +const cache = @import("cache.zig");