diff --git a/.agents/skills/edgeflow-node-developer/SKILL.md b/.agents/skills/edgeflow-node-developer/SKILL.md index 05acb5fa..c26fff80 100644 --- a/.agents/skills/edgeflow-node-developer/SKILL.md +++ b/.agents/skills/edgeflow-node-developer/SKILL.md @@ -74,7 +74,7 @@ LLM 生成参数复用 `GenerateParameters()`，默认 `max_tokens = 128`；自 ## 验证与接入 优先扩展现有测试；`tests/unit/nodes/test_*.cpp` 自动收集到节点 runner，GoogleTest 自动发现 -用例并生成 CTest 入口；脚手架 `--write-test` 沿用 `ScaffoldGeneratedNodeTest`，模型替身来自 `tests/support/recording_models.h`。聚焦测试执行真实 `Run`， +用例并生成 CTest 入口；脚手架 `--write-test` 沿用 `ScaffoldGeneratedNodeTest`，用 `tests/support/node_probe.h` 的 `NodeUnderTest` 按逻辑端口执行，模型替身来自 `tests/support/recording_models.h`；共享 Session 或串联节点时才用 `NodeProbe`。聚焦测试执行真实 `Run`， 检查实际业务输出、空/缺失输入、多个请求及非零 `sub_id`、数量与来源、模型失败不发布结果； Control/缓存改动补回滚与并发测试。仅创建成功或检查 Catalog 不证明算法正确。 diff --git a/doc/dev_guide/custom_node_concepts.md b/doc/dev_guide/custom_node_concepts.md index fbed7ab0..8854b99d 100644 --- a/doc/dev_guide/custom_node_concepts.md +++ b/doc/dev_guide/custom_node_concepts.md @@ -381,8 +381,8 @@ TextEmbedding 会话缓存，以及模板和规则的字段 Control。无需按 `NodeResult::Failure`，原因写明字段或来源。执行包装统一处理诊断和发布，普通算法不保存 Context 指针。配置更新由包装提供一致快照。含外部资源的更新仍需明确资源生命周期。 -第一次测试可复用 [NodeProbe](../../tests/support/node_probe.h)，将断言加入现有 -Node 套件。需要模型时使用 [recording_models.h](../../tests/support/recording_models.h) +业务测试使用 [NodeUnderTest](../../tests/support/node_probe.h)，只写逻辑端口、配置、输入和断言； +共享 Session 或串联多个节点的框架测试使用同一文件中的 `NodeProbe`。需要模型时使用 [recording_models.h](../../tests/support/recording_models.h) 中的固定回答替身，它记录每次调用，也能让指定调用失败。命令见[本地快速验证](../../src/custom_nodes/README.md#本地快速验证)。 先用两个不同请求、非零 `sub_id` 验证不会串结果，再检查算法自己的正常和失败输出。 diff --git a/doc/dev_guide/first_custom_node.md b/doc/dev_guide/first_custom_node.md index 769ee8a3..b3d190ad 100644 --- a/doc/dev_guide/first_custom_node.md +++ b/doc/dev_guide/first_custom_node.md @@ -29,8 +29,12 @@ 得到 `src/custom_nodes/my_text_normalize_node.cpp` 和 `tests/unit/nodes/test_my_text_normalize_node.cpp`。已有文件会被保留；如果已做过练习，继续编辑它们。 +教程中的每段代码都整体替换源码或测试中的同名定义；新增的结构体放在已有结构体之后。 +仓库测试会把这些代码应用到脚手架输出上，编译并运行三个阶段，教程与实际行为保持一致。 + 在源码中把 `Transform` 替换为： + ```cpp std::string Transform(const std::string& text) { const std::string from = "开机检查"; @@ -48,8 +52,29 @@ std::string Transform(const std::string& text) { `Spec` 声明 `input` 和 `output` 都是文本批次，输入一项就返回一项。 修改文本时创建新值，不修改共享的只读输入。 -在生成的测试中，将输入 `"example"` 改成 `"请做一次开机检查"`， -将 `item.data` 的期望值改成 `"请做一次自检"`。保留数量与来源编号的断言。 +在生成的测试文件中，保留开头的 `#include` 和命名空间，用下面的测试替换其中的 `TEST`： + + +```cpp +TEST(ScaffoldGeneratedNodeTest, my_text_normalize_ReplacesFixedPhrase) { + test_support::NodeUnderTest node; + ASSERT_TRUE(node.Init("my_text_normalize", nlohmann::json::object(), + {"input"}, {"output"})) + << node.diagnostic; + const auto run = + node.Process({{"input", TextBatch{{7, 2, "请做一次开机检查"}}}}); + ASSERT_EQ(run.status, 0) << run.diagnostic; + const auto* result = run.Output("output"); + ASSERT_NE(result, nullptr); + ASSERT_EQ(result->size(), 1u); + EXPECT_EQ(result->front().req_id, 7u); + EXPECT_EQ(result->front().sub_id, 2u); + EXPECT_EQ(result->front().data, "请做一次自检"); +} +``` + +`NodeUnderTest` 按逻辑端口 `input`、`output` 执行真实注册的节点； +Session、内部数据名和请求上下文由它管理，测试只写配置、输入和期望结果。 每完成一个阶段，都用下面的命令格式化、编译和运行这个测试： @@ -65,21 +90,29 @@ cmake --build build --target edgeflow_test_nodes_runner alg_pipeline_tool -j 4 ## 2. 为同一个节点增加普通配置 -现在希望不同方案能选择替换后的词语。在 `Inputs` 后添加： +现在希望不同方案能选择替换后的词语。添加 `Params`，并让 `Transform` 从参数读取替换文本： + ```cpp struct Params { std::string replacement; }; -``` -将 `Transform` 的签名改为 -`std::string Transform(const std::string& text, const Params& params)`， -把函数体中的 `const std::string to = "自检";` 改成 -`const std::string& to = params.replacement;`。其余替换逻辑保持不变。 +std::string Transform(const std::string& text, const Params& params) { + const std::string from = "开机检查"; + const std::string& to = params.replacement; + std::string result = text; + for (auto pos = result.find(from); pos != std::string::npos; + pos = result.find(from, pos + to.size())) { + result.replace(pos, from.size(), to); + } + return result; +} +``` 用下面的代码替换原来的 `Run` 和 `Spec`，保留原来的注册语句： + ```cpp NodeResult Run(const Inputs& inputs, const Params& params) { return MapPayloads(*inputs.input, [¶ms](const std::string& text) { @@ -102,9 +135,41 @@ auto Spec() { 框架从 Pipeline 节点的 `params` 读取 `replacement`，未填写时使用 `自检`。 这些配置在创建节点时生效。只有需要运行时更新，才添加 [Control](first_control.md)。 -先运行原来的测试，确认默认值仍然生效。再把测试中 `probe.Init` 的 -`nlohmann::json::object()` 改为 `nlohmann::json{{"replacement", "启动自检"}}`， -将输出期望值改为 `"请做一次启动自检"`，重新运行测试。 +用下面两个测试替换上一阶段的测试，分别检查默认值和配置值： + + +```cpp +TEST(ScaffoldGeneratedNodeTest, my_text_normalize_UsesDefaultReplacement) { + test_support::NodeUnderTest node; + ASSERT_TRUE(node.Init("my_text_normalize", nlohmann::json::object(), + {"input"}, {"output"})) + << node.diagnostic; + const auto run = + node.Process({{"input", TextBatch{{7, 2, "请做一次开机检查"}}}}); + ASSERT_EQ(run.status, 0) << run.diagnostic; + const auto* result = run.Output("output"); + ASSERT_NE(result, nullptr); + ASSERT_EQ(result->size(), 1u); + EXPECT_EQ(result->front().data, "请做一次自检"); +} + +TEST(ScaffoldGeneratedNodeTest, my_text_normalize_UsesConfiguredReplacement) { + test_support::NodeUnderTest node; + ASSERT_TRUE(node.Init("my_text_normalize", {{"replacement", "启动自检"}}, + {"input"}, {"output"})) + << node.diagnostic; + const auto run = + node.Process({{"input", TextBatch{{7, 2, "请做一次开机检查"}}}}); + ASSERT_EQ(run.status, 0) << run.diagnostic; + const auto* result = run.Output("output"); + ASSERT_NE(result, nullptr); + ASSERT_EQ(result->size(), 1u); + EXPECT_EQ(result->front().req_id, 7u); + EXPECT_EQ(result->front().sub_id, 2u); + EXPECT_EQ(result->front().data, "请做一次启动自检"); +} +``` + Catalog 现在应显示节点说明和 `replacement`，仍无模型依赖。 ## 3. 在同一个节点中调用 LLM @@ -112,6 +177,7 @@ Catalog 现在应显示节点说明和 `replacement`，仍无模型依赖。 继续编辑同一个文件。保留 `Inputs`、`Transform` 和注册语句。 给现有 `Params` 增加生成参数，并在它后面添加 `Models`： + ```cpp struct Params { std::string replacement; @@ -125,6 +191,7 @@ struct Models { 用下面的代码替换 `Run` 和 `Spec`。处理顺序是词语替换、构造提示词、调用模型、清理回答： + ```cpp NodeResult Run(const Inputs& inputs, const Params& params, const Models& models) { @@ -161,36 +228,44 @@ auto Spec() { ### 更新已有测试 -继续修改同一个测试文件。加入头文件 `tests/support/recording_models.h`， -在 `SessionContext session;` 后加入以下模型替身： - -```cpp -auto model = std::make_shared(); -model->replies = {{"请执行启动自检\n"}}; -ASSERT_TRUE(test_support::RegisterNodeModel(session, "fixture", model)); -``` - -将 `probe.Init` 的参数对象改为： - -```cpp -nlohmann::json{{"replacement", "启动自检"}, - {"bind_model", "fixture"}, - {"max_tokens", 32}} -``` - -输入仍为 `"请做一次开机检查"`，将输出期望值改为 `"请执行启动自检"`， -以确认回答末尾的换行已被清理。在原有结果断言之后补充： +在测试文件开头加入模型替身的头文件，并用下面的测试替换上一阶段的测试。 +`RecordingLlm` 返回固定回答，并记录每次调用的提示词和生成参数： + ```cpp -ASSERT_EQ(model->calls.size(), 1u); -ASSERT_EQ(model->calls.front().size(), 1u); -EXPECT_EQ(model->calls.front().front().data, - "将下面的操作说明改写为简洁的规范指令，只输出改写结果：\n请做一次启动自检"); -EXPECT_EQ(model->call_options.front().max_tokens, 32); +#include "tests/support/recording_models.h" + +TEST(ScaffoldGeneratedNodeTest, my_text_normalize_BuildsPromptAndCleansAnswer) { + test_support::NodeUnderTest node; + auto model = std::make_shared(); + model->replies = {{"请执行启动自检\n"}}; + ASSERT_TRUE(node.AddModel("fixture", model)); + ASSERT_TRUE(node.Init("my_text_normalize", + {{"replacement", "启动自检"}, + {"bind_model", "fixture"}, + {"max_tokens", 32}}, + {"input"}, {"output"})) + << node.diagnostic; + const auto run = + node.Process({{"input", TextBatch{{7, 2, "请做一次开机检查"}}}}); + ASSERT_EQ(run.status, 0) << run.diagnostic; + const auto* result = run.Output("output"); + ASSERT_NE(result, nullptr); + ASSERT_EQ(result->size(), 1u); + EXPECT_EQ(result->front().req_id, 7u); + EXPECT_EQ(result->front().sub_id, 2u); + EXPECT_EQ(result->front().data, "请执行启动自检"); + ASSERT_EQ(model->calls.size(), 1u); + ASSERT_EQ(model->calls.front().size(), 1u); + EXPECT_EQ(model->calls.front().front().data, + "将下面的操作说明改写为简洁的规范指令，只输出改写结果：\n" + "请做一次启动自检"); + EXPECT_EQ(model->call_options.front().max_tokens, 32); +} ``` -按第一节的命令重新编译和测试。现在测试验证了替换词、实际提示词、生成参数和回答清理； -Catalog 应显示更新后的节点说明、`bind_model`、生成参数和一个 LLM 依赖。 +按第一节的命令重新编译和测试。现在测试验证了替换词、实际提示词、生成参数和回答清理， +其中回答末尾的换行已被清理。Catalog 应显示更新后的节点说明、`bind_model`、生成参数和一个 LLM 依赖。 模型替身只提供确定的回答，业务函数无需识别测试环境。 ## 4. 把这个节点接入 Pipeline diff --git a/src/custom_nodes/README.md b/src/custom_nodes/README.md index ee252607..13fc4589 100644 --- a/src/custom_nodes/README.md +++ b/src/custom_nodes/README.md @@ -50,9 +50,11 @@ cmake --build build --target edgeflow_test_nodes_runner -j 4 ./build/edgeflow_test_nodes_runner --gtest_filter='ScaffoldGeneratedNodeTest.*' ``` -[Node 测试辅助](../../tests/support/node_probe.h)用已校验的 Plan 和 Session 初始化注册的 Node。 -将请求输入放入新的 `AlgContext`。调用 Process 后，断言实际输出和 `(req_id, sub_id)`。 -带模型的节点用 [recording_models.h](../../tests/support/recording_models.h) 中的固定回答替身。 +[Node 测试辅助](../../tests/support/node_probe.h)的 `NodeUnderTest` 用已校验的 Plan 初始化注册的 Node， +并管理 Session、内部数据名和请求上下文。测试按逻辑端口给出配置和输入，用 `Process` 的结果断言 +返回码、诊断、实际输出和 `(req_id, sub_id)`；失败时输出不发布。 +带模型的节点用 [recording_models.h](../../tests/support/recording_models.h) 中的固定回答替身，经 `AddModel` 注册。 +共享 Session、指定端口生命周期或在同一上下文串联多个节点时，直接使用底层 `NodeProbe`。 还需覆盖空输入、非法输入和失败行为。脚手架的 `--write-test` 在 `tests/unit/nodes/` 生成完整测试文件，节点 runner 自动收录，并通过 GoogleTest 原生发现生成 CTest 入口。发现检查核对实际编译用例都有执行归属。 修改注册或 Definition 后重新构建 `alg_pipeline_tool`，再用同一构建检查组合后的方案。 diff --git a/tests/ScaffoldFixtures.cmake b/tests/ScaffoldFixtures.cmake index 27f46119..e466c17c 100644 --- a/tests/ScaffoldFixtures.cmake +++ b/tests/ScaffoldFixtures.cmake @@ -20,9 +20,8 @@ add_custom_command( DEPENDS "${PROJECT_SOURCE_DIR}/tools/scaffold_custom_node.py" "${PROJECT_SOURCE_DIR}/doc/dev_guide/first_custom_node.md" - "${PROJECT_SOURCE_DIR}/doc/dev_guide/first_control.md" "${PROJECT_SOURCE_DIR}/tests/tooling/generate_scaffold_fixtures.py" - COMMENT "Generating custom Node snippets and standalone behavioral test fixtures" + COMMENT "Generating scaffold and first custom Node tutorial fixtures" VERBATIM) # 在测试 runner 中编译文档里的原样函数示例，绝不进入 SDK Catalog。 diff --git a/tests/support/node_probe.h b/tests/support/node_probe.h index b3010d62..5e656488 100644 --- a/tests/support/node_probe.h +++ b/tests/support/node_probe.h @@ -1,12 +1,17 @@ #pragma once +#include + #include +#include +#include #include #include #include #include #include "contracts/config_schema_validation.h" +#include "core/alg_context.h" #include "core/node_registry.h" #include "core/session_context.h" #include "core/validated_node_plan.h" @@ -86,7 +91,13 @@ struct NodeProbe { diagnostic = "Node creator returned null: " + type; return false; } - return node->Init({&plan, &session, &diagnostic}); + // node stays null unless Init succeeds, so callers never reach a node whose + // model binding or parameter preparation failed. + if (!node->Init({&plan, &session, &diagnostic})) { + node.reset(); + return false; + } + return true; } }; @@ -100,4 +111,107 @@ inline bool RegisterNodeModel(SessionContext& session, std::string name, return session.GetModelManager().RegisterBatch({std::move(registration)}); } +inline std::string NodeInputKey(const std::string& port) { + return "input." + port; +} +inline std::string NodeOutputKey(const std::string& port) { + return "output." + port; +} + +// One request value for a logical input port. +struct NodeInput { + template + NodeInput(std::string port, Batch batch) + : port(std::move(port)), + publish([batch = std::move(batch)](AlgContext& context, + const std::string& key) { + return context.Publish(key, batch); + }) {} + + std::string port; + std::function publish; +}; + +// One Process call. Output views belong to the request context owned here and +// stay valid while this object lives. +class NodeRun { + public: + int status = 0; + std::string diagnostic; + + template + const Batch* Output(const std::string& port) const { + return context_->Read(NodeOutputKey(port)); + } + bool Published(const std::string& port) const { + return context_->Has(NodeOutputKey(port)); + } + + private: + friend class NodeUnderTest; + std::unique_ptr context_ = std::make_unique(); +}; + +// Default entry for business Node tests: the author names logical ports and +// supplies inputs, params and model doubles; the session, storage keys and +// request contexts stay here. Tests that share a session, override port +// lifetimes or chain nodes in one context use NodeProbe directly. +class NodeUnderTest { + public: + std::string diagnostic; + + bool AddModel(std::string name, std::shared_ptr model) { + return RegisterNodeModel(session_, std::move(name), std::move(model)); + } + + bool Init(const std::string& type, const nlohmann::json& params, + const std::vector& inputs, + const std::vector& outputs) { + const auto bind = [](const std::vector& ports, + std::string (*key)(const std::string&)) { + std::vector bindings; + for (const auto& port : ports) bindings.push_back({port, key(port)}); + return bindings; + }; + const bool ok = + probe_.Init(type, params, session_, bind(inputs, &NodeInputKey), + bind(outputs, &NodeOutputKey)); + diagnostic = probe_.diagnostic; + return ok; + } + + NodeRun Process(std::initializer_list inputs) { + NodeRun run; + if (!probe_.node) { + ADD_FAILURE() << "Process before a successful Init"; + run.status = -1; + return run; + } + for (const auto& input : inputs) { + if (!probe_.plan.FindPort(input.port, PortDirection::kInput) || + !input.publish(*run.context_, NodeInputKey(input.port))) { + ADD_FAILURE() << "Input port not connected or given twice: " + << input.port; + run.status = -1; + return run; + } + } + run.status = probe_.node->Process(run.context_.get()); + run.diagnostic = run.context_->GetErrorMessage(); + return run; + } + + NodeControlResult Control(int command, const std::string& payload) { + if (!probe_.node) { + ADD_FAILURE() << "Control before a successful Init"; + return NodeControlResult::Unsupported(); + } + return probe_.node->Control(command, payload); + } + + private: + SessionContext session_; + NodeProbe probe_; +}; + } // namespace llm_edgeflow::test_support diff --git a/tests/tooling/generate_scaffold_fixtures.py b/tests/tooling/generate_scaffold_fixtures.py index cdf57555..acb32e06 100644 --- a/tests/tooling/generate_scaffold_fixtures.py +++ b/tests/tooling/generate_scaffold_fixtures.py @@ -3,6 +3,9 @@ Only anonymous namespace names are isolated for the existing single-TU runner. The scaffold owns all generated algorithms and their emitted consumer tests. +The first custom Node tutorial is replayed on the same scaffold output: each +anchored code block replaces its same-named definition, and every stage is +compiled, executed and checked against the Catalog facts the tutorial states. """ import os @@ -14,6 +17,16 @@ ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "tools/scaffold_custom_node.py" +TUTORIAL = ROOT / "doc/dev_guide/first_custom_node.md" +TUTORIAL_NODE = "my_text_normalize" +ANCHOR = re.compile(r"\n```cpp\n(.*?)\n```", re.S) +LITERAL = re.compile(r'"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'') +# Catalog facts each tutorial stage states: own config fields and model slots. +TUTORIAL_CATALOG = { + "stage1": ([], []), + "stage2": (["replacement"], []), + "stage3": (["replacement", "max_tokens", "bind_model"], [("model", "llm", "bind_model")]), +} def isolated_namespaces(content, namespace): @@ -28,7 +41,122 @@ def replace(match): return re.sub(r"namespace\s*\{", replace, content) -def generate(output): +def tutorial_blocks(text): + blocks = {} + for stage, kind, code in ANCHOR.findall(text): + blocks.setdefault(stage, {"node": [], "test": []})[kind].append(code) + if sorted(blocks) != sorted(TUTORIAL_CATALOG): + raise RuntimeError(f"Tutorial stages {sorted(blocks)} differ from {sorted(TUTORIAL_CATALOG)}") + for stage, kinds in blocks.items(): + if not kinds["node"] or len(kinds["test"]) != 1: + raise RuntimeError(f"Tutorial {stage} needs node blocks and exactly one test block") + return blocks + + +def definitions(lines): + """Top-level definitions starting at column 0 as (name, kind, first, last) line ranges.""" + found = [] + index = 0 + while index < len(lines): + line = lines[index] + if not line or line[0] in " \t}#/" or line.startswith(("namespace", "REGISTER_FUNCTION_NODE")): + index += 1 + continue + match = re.match(r"struct\s+(\w+)", line) + name, kind = (match.group(1), "struct") if match else (re.search(r"(\w+)\s*\(", line).group(1), "function") + depth, opened, last = 0, False, index + while True: + code = LITERAL.sub("", lines[last]) + depth += code.count("{") - code.count("}") + opened |= "{" in code + if (opened and depth == 0) or (not opened and code.rstrip().endswith(";")): + break + last += 1 + found.append((name, kind, index, last)) + index = last + 1 + return found + + +def apply_node_block(source, block): + lines = source.split("\n") + block_lines = block.split("\n") + for name, kind, first, last in definitions(block_lines): + replacement = block_lines[first:last + 1] + current = definitions(lines) + existing = [item for item in current if item[0] == name] + if existing: + _, _, start, end = existing[0] + lines[start:end + 1] = replacement + elif kind == "struct": + end = [item for item in current if item[1] == "struct"][-1][3] + lines[end + 1:end + 1] = [""] + replacement + else: + raise RuntimeError(f"Tutorial block defines unknown function {name}") + return "\n".join(lines) + + +def apply_test_block(test, block): + lines = test.split("\n") + includes = [line for line in block.split("\n") if line.startswith("#include")] + body = [line for line in block.split("\n") if not line.startswith("#include")] + while body and not body[0]: + body.pop(0) + anchor = lines.index('#include "tests/support/node_probe.h"') + lines[anchor + 1:anchor + 1] = [line for line in includes if line not in lines] + first = next(i for i, line in enumerate(lines) if line.startswith("TEST(")) + last = next(i for i, line in enumerate(lines) if line.startswith("} // namespace")) + lines[first:last] = body + return "\n".join(lines) + + +def catalog_check(stage, node_type): + fields, models = TUTORIAL_CATALOG[stage] + field_checks = "".join( + f''' EXPECT_TRUE(std::any_of(definition->config_fields.begin(), definition->config_fields.end(), + [](const auto& field) {{ return field.name == "{name}"; }})) << "{name}"; +''' for name in fields) + model_checks = "".join( + f''' EXPECT_EQ(definition->model_dependencies[{i}].name, "{name}"); + EXPECT_EQ(definition->model_dependencies[{i}].model_type, "{model_type}"); + EXPECT_EQ(definition->model_dependencies[{i}].config_field, "{field}"); +''' for i, (name, model_type, field) in enumerate(models)) + exact = "" if models else f" EXPECT_EQ(definition->config_fields.size(), {len(fields)}u);\n" + return f''' +// Catalog facts stated by the tutorial for {stage}. +namespace llm_edgeflow {{ +TEST(FirstCustomNodeTutorialTest, {stage}_CatalogMatchesTutorial) {{ + const auto definition = NodeRegistry::Instance().Find("{node_type}"); + ASSERT_TRUE(definition.has_value()); + EXPECT_FALSE(definition->description.empty()); +{exact}{field_checks} ASSERT_EQ(definition->model_dependencies.size(), {len(models)}u); +{model_checks}}} +}} // namespace llm_edgeflow +''' + + +def tutorial_sections(root, environment, tutorial): + blocks = tutorial_blocks(Path(tutorial).read_text(encoding="utf-8")) + result = subprocess.run([sys.executable, str(SCRIPT), TUTORIAL_NODE, "--write-test"], + env=environment, capture_output=True, text=True, timeout=30) + if result.returncode: + raise RuntimeError(f"Scaffold generation failed for the tutorial:\n{result.stdout}\n{result.stderr}") + source = (root / "src/custom_nodes" / f"{TUTORIAL_NODE}_node.cpp").read_text(encoding="utf-8") + test = (root / "tests/unit/nodes" / f"test_{TUTORIAL_NODE}_node.cpp").read_text(encoding="utf-8") + sections = [] + for stage in sorted(blocks): + for block in blocks[stage]["node"]: + source = apply_node_block(source, block) + test = apply_test_block(test, blocks[stage]["test"][0]) + node_type = f"tutorial_{stage}_{TUTORIAL_NODE}" + for kind, content in (("source", source), ("test", test)): + sections.append(f"\n// Tutorial {stage} {kind}: {TUTORIAL.name} applied to scaffold output\n") + sections.append(isolated_namespaces(content.replace(TUTORIAL_NODE, node_type), + f"fixture_tutorial_{stage}_{kind}")) + sections.append(catalog_check(stage, node_type)) + return sections + + +def generate(output, tutorial=TUTORIAL): # Each entry selects a distinct body of the single Node structure or typed call signature. requests = [ ("scaffold_text_identity", ["--in-port", "sentences:TextBatch", "--out-port", "answers:TextBatch"]), @@ -59,6 +187,7 @@ def generate(output): raise RuntimeError(f"Generated consumer still depends on retired helpers: {path.name}") sections.append(f"\n// Exact generated {kind}: {path.name}\n") sections.append(isolated_namespaces(content, f"fixture_{name}_{kind}")) + sections.extend(tutorial_sections(root, environment, tutorial)) output.parent.mkdir(parents=True, exist_ok=True) # Publish a complete fixture artifact; interrupted generation retains the prior one. with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=output.parent, diff --git a/tests/tooling/test_scaffold_custom_node.py b/tests/tooling/test_scaffold_custom_node.py index 26e0269d..837ef59b 100755 --- a/tests/tooling/test_scaffold_custom_node.py +++ b/tests/tooling/test_scaffold_custom_node.py @@ -59,6 +59,11 @@ def test_custom_ports_description_and_generated_test_are_a_single_requested_arti test_content = generated_test.read_text() self.assertIn('tests/support/node_probe.h', test_content) self.assertIn('consumer_named', test_content) + # 生成的测试按逻辑端口读写，Session、内部数据名与请求上下文由 NodeUnderTest 管理。 + self.assertIn('node.Process({{"source", inputs}})', test_content) + self.assertIn('run.Output("answer")', test_content) + for internal in ('SessionContext', 'AlgContext', 'NodeProbe', 'Publish(', 'Read<'): + self.assertNotIn(internal, test_content) self.assertNotIn('node_harness.h', test_content) self.assertNotIn('node_test_utils.h', test_content) self.assertTrue(source.exists() and generated_test.exists()) @@ -190,5 +195,53 @@ def test_new_file_race_preserves_competing_file_and_never_replaces_it(self): self.assertFalse(list(self.source_dir.glob('*.edgeflow-tmp'))) +FIXTURES_SCRIPT = ROOT / "tests/tooling/generate_scaffold_fixtures.py" +FIXTURES_SPEC = importlib.util.spec_from_file_location("scaffold_fixtures_under_test", FIXTURES_SCRIPT) +fixtures = importlib.util.module_from_spec(FIXTURES_SPEC) +FIXTURES_SPEC.loader.exec_module(fixtures) + + +class TutorialFixtureTest(unittest.TestCase): + """教程代码块经由脚手架输出进入编译 fixture；结构不符时生成失败。""" + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="tutorial-fixture-") + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.tutorial = fixtures.TUTORIAL.read_text(encoding="utf-8") + + def generate(self, tutorial_text): + tutorial = self.root / "first_custom_node.md" + tutorial.write_text(tutorial_text, encoding="utf-8") + output = self.root / "fixture.cpp" + fixtures.generate(output, tutorial) + return output.read_text(encoding="utf-8") + + def test_every_stage_replays_tutorial_blocks_on_scaffold_output(self): + content = self.generate(self.tutorial) + for stage in fixtures.TUTORIAL_CATALOG: + node_type = f"tutorial_{stage}_my_text_normalize" + self.assertIn(f"REGISTER_FUNCTION_NODE({node_type}, Spec());", content) + self.assertIn(f"TEST(FirstCustomNodeTutorialTest, {stage}_CatalogMatchesTutorial)", content) + self.assertIn('const std::string& to = params.replacement;', content) + self.assertIn('#include "tests/support/recording_models.h"', content) + # 教程测试替换脚手架生成的 TEST。 + self.assertNotIn("tutorial_stage1_my_text_normalize_ExecutesDeclaredContract", content) + + def test_edited_tutorial_code_reaches_the_compiled_fixture(self): + broken = self.tutorial.replace(" return result;", " return missing_tutorial_value;", 1) + self.assertNotEqual(broken, self.tutorial) + self.assertIn("return missing_tutorial_value;", self.generate(broken)) + + def test_structural_tutorial_drift_is_rejected(self): + without_stage_test = self.tutorial.replace("\n", "", 1) + with self.assertRaisesRegex(RuntimeError, "exactly one test block"): + self.generate(without_stage_test) + unknown_function = self.tutorial.replace( + "std::string Transform(const std::string& text) {", "std::string Rewrite(const std::string& text) {", 1) + with self.assertRaisesRegex(RuntimeError, "unknown function Rewrite"): + self.generate(unknown_function) + + if __name__ == '__main__': unittest.main() diff --git a/tests/unit/nodes/test_node_under_test.cpp b/tests/unit/nodes/test_node_under_test.cpp new file mode 100644 index 00000000..38b0c7f7 --- /dev/null +++ b/tests/unit/nodes/test_node_under_test.cpp @@ -0,0 +1,74 @@ +#include +#include + +#include +#include + +#include "core/common_contracts.h" +#include "tests/support/node_probe.h" +#include "tests/support/recording_models.h" + +namespace llm_edgeflow { +namespace { +using test_support::NodeUnderTest; + +TEST(NodeUnderTestTest, FailedRunPublishesNothingAndNextRunSucceeds) { + NodeUnderTest node; + auto model = std::make_shared(); + model->replies = {{std::string("unused")}, {std::string("answer")}}; + model->fail_call = 1; + ASSERT_TRUE(node.AddModel("fixture", model)); + ASSERT_TRUE(node.Init("prompt_guided_llm", {{"bind_model", "fixture"}}, + {"input"}, {"output"})) + << node.diagnostic; + const TextBatch inputs{{3, 4, "question"}}; + + const auto failed = node.Process({{"input", inputs}}); + EXPECT_NE(failed.status, 0); + EXPECT_FALSE(failed.diagnostic.empty()); + EXPECT_FALSE(failed.Published("output")); + EXPECT_EQ(failed.Output("output"), nullptr); + + const auto next = node.Process({{"input", inputs}}); + ASSERT_EQ(next.status, 0) << next.diagnostic; + const auto* result = next.Output("output"); + ASSERT_NE(result, nullptr); + ASSERT_EQ(result->size(), 1u); + EXPECT_EQ(result->front().req_id, 3u); + EXPECT_EQ(result->front().sub_id, 4u); + EXPECT_EQ(result->front().data, "answer"); + EXPECT_EQ(model->calls.size(), 2u); +} + +TEST(NodeUnderTestTest, ReportsInvalidInitAndUnconnectedInputs) { + NodeUnderTest unregistered; + EXPECT_FALSE(unregistered.Init("missing_node", nlohmann::json::object(), + {"input"}, {"output"})); + EXPECT_NE(unregistered.diagnostic.find("missing_node"), std::string::npos); + EXPECT_NONFATAL_FAILURE(unregistered.Process({}), "successful Init"); + + // A registered Node whose model binding fails must not run either; an empty + // batch would otherwise succeed without touching the model. + NodeUnderTest unbound; + EXPECT_FALSE(unbound.Init("prompt_guided_llm", {{"bind_model", "absent"}}, + {"input"}, {"output"})); + EXPECT_NE(unbound.diagnostic.find("absent"), std::string::npos); + EXPECT_NONFATAL_FAILURE(unbound.Process({{"input", TextBatch{}}}), + "successful Init"); + EXPECT_NONFATAL_FAILURE(unbound.Control(1, "{}"), "successful Init"); + + NodeUnderTest node; + auto model = std::make_shared(); + ASSERT_TRUE(node.AddModel("fixture", model)); + ASSERT_TRUE(node.Init("prompt_guided_llm", {{"bind_model", "fixture"}}, + {"input"}, {"output"})) + << node.diagnostic; + const TextBatch inputs{{1, 0, "question"}}; + EXPECT_NONFATAL_FAILURE(node.Process({{"context", inputs}}), "context"); + EXPECT_NONFATAL_FAILURE(node.Process({{"input", inputs}, {"input", inputs}}), + "given twice"); + EXPECT_TRUE(model->calls.empty()); +} + +} // namespace +} // namespace llm_edgeflow diff --git a/tools/scaffold_custom_node.py b/tools/scaffold_custom_node.py index 869086c4..52ff2540 100755 --- a/tools/scaffold_custom_node.py +++ b/tools/scaffold_custom_node.py @@ -241,7 +241,7 @@ def render_node(name, description, in_port, out_port, capability=None, control_i def render_standalone_test(name, in_port, out_port, capability=None, control_id=None): - """生成的测试执行真实注册的 Node；端口声明由 Init 绑定和类型化读写检查。""" + """生成的测试执行真实注册的 Node；Session、内部 key 与请求 Context 由 NodeUnderTest 管理。""" in_name, in_type, _, _ = in_port out_name, out_type, _, _ = out_port input_payload = SAMPLE_INPUTS.get(in_type, f"{payload_type(in_type)}{{}}") @@ -254,15 +254,17 @@ def render_standalone_test(name, in_port, out_port, capability=None, control_id= support = '#include "tests/support/recording_models.h"\n' setup = f''' auto model = std::make_shared(); model->replies = {{{{{reply}}}}}; - ASSERT_TRUE(test_support::RegisterNodeModel(session, "fixture", model)); + ASSERT_TRUE(node.AddModel("fixture", model)); ''' params = 'nlohmann::json{{"bind_model", "fixture"}}' if control_id is not None: params = 'nlohmann::json{{"prefix", "before:"}}' payload_checks = 'EXPECT_EQ(item.data, "before:example");' + in_ref = cpp_string(in_name) + out_ref = cpp_string(out_name) if capability or (is_preserved(in_port, out_port) and in_type == out_type): - execution = f''' ASSERT_EQ(probe.node->Process(&context), 0) << context.GetErrorMessage(); - const auto* result = context.Read<{out_type}>("result"); + execution = f''' ASSERT_EQ(run.status, 0) << run.diagnostic; + const auto* result = run.Output<{out_type}>({out_ref}); ASSERT_NE(result, nullptr); ASSERT_EQ(result->size(), 1u); const auto& item = result->front(); @@ -271,8 +273,8 @@ def render_standalone_test(name, in_port, out_port, capability=None, control_id= {payload_checks} ''' else: - execution = ''' EXPECT_EQ(probe.node->Process(&context), error::kBusinessError); - EXPECT_FALSE(context.Has("result")); + execution = f''' EXPECT_EQ(run.status, error::kBusinessError); + EXPECT_FALSE(run.Published({out_ref})); ''' if capability: execution += ''' ASSERT_EQ(model->calls.size(), 1u); @@ -280,13 +282,12 @@ def render_standalone_test(name, in_port, out_port, capability=None, control_id= EXPECT_EQ(model->calls.front().front().sub_id, 2u); ''' if control_id is not None: - execution += f''' ASSERT_EQ(probe.node->Control({control_id}, R"({{"prefix":"after:"}})").status, + execution += f''' ASSERT_EQ(node.Control({control_id}, R"({{"prefix":"after:"}})").status, NodeControlStatus::kHandled); const auto verify_after = [&] {{ - AlgContext next; - EXPECT_TRUE(next.Publish("input", inputs)); - EXPECT_EQ(probe.node->Process(&next), 0); - const auto* updated = next.Read<{out_type}>("result"); + const auto next = node.Process({{{{{in_ref}, inputs}}}}); + EXPECT_EQ(next.status, 0) << next.diagnostic; + const auto* updated = next.Output<{out_type}>({out_ref}); ASSERT_NE(updated, nullptr); ASSERT_EQ(updated->size(), 1u); EXPECT_EQ(updated->front().req_id, 7u); @@ -295,7 +296,7 @@ def render_standalone_test(name, in_port, out_port, capability=None, control_id= }}; verify_after(); const auto invalid = nlohmann::json{{{{"prefix", std::string(65, 'x')}}}}.dump(); - EXPECT_EQ(probe.node->Control({control_id}, invalid).status, NodeControlStatus::kFailed); + EXPECT_EQ(node.Control({control_id}, invalid).status, NodeControlStatus::kFailed); verify_after(); ''' return f'''#include @@ -304,21 +305,17 @@ def render_standalone_test(name, in_port, out_port, capability=None, control_id= #include #include -#include "core/alg_context.h" #include "core/common_contracts.h" #include "tests/support/node_probe.h" {support} namespace llm_edgeflow::scaffold_{name}_test {{ TEST(ScaffoldGeneratedNodeTest, {name}_ExecutesDeclaredContract) {{ - SessionContext session; -{setup} test_support::NodeProbe probe; - ASSERT_TRUE(probe.Init({cpp_string(name)}, {params}, session, - {{{{{cpp_string(in_name)}, "input"}}}}, - {{{{{cpp_string(out_name)}, "result"}}}})) << probe.diagnostic; + test_support::NodeUnderTest node; +{setup} ASSERT_TRUE(node.Init({cpp_string(name)}, {params}, {{{in_ref}}}, {{{out_ref}}})) + << node.diagnostic; {in_type} inputs; inputs.emplace_back(7, 2, {input_payload}); - AlgContext context; - ASSERT_TRUE(context.Publish("input", inputs)); + const auto run = node.Process({{{{{in_ref}, inputs}}}}); {execution}}} }} // namespace llm_edgeflow::scaffold_{name}_test '''