Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
212 changes: 212 additions & 0 deletions tests/skills/tools/test_skill_exec.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,15 @@
from trpc_agent_sdk.skills.tools._skill_exec import SkillExecTool
from trpc_agent_sdk.skills.tools._skill_exec import WriteStdinTool
from trpc_agent_sdk.skills.tools._skill_exec import _close_session
from trpc_agent_sdk.skills.tools._skill_exec import _collect_final_result
from trpc_agent_sdk.skills.tools._skill_exec import _detect_interaction
from trpc_agent_sdk.skills.tools._skill_exec import _ExecSession
from trpc_agent_sdk.skills.tools._skill_exec import _has_selection_items
from trpc_agent_sdk.skills.tools._skill_exec import _last_non_empty_line
from trpc_agent_sdk.skills.tools._skill_exec import _start_session
from trpc_agent_sdk.skills.tools._skill_exec import create_exec_tools
from trpc_agent_sdk.skills.tools._skill_run import SkillRunFile
from trpc_agent_sdk.skills.tools._skill_run import SkillRunInput


def _make_exec_tool() -> SkillExecTool:
Expand Down Expand Up @@ -107,3 +112,210 @@ async def test_close_session(self):
sess.proc.close = AsyncMock()
await _close_session(sess)
sess.proc.close.assert_awaited_once()


class TestStartSession:
@pytest.mark.asyncio
async def test_start_session_stores_workspace_runtime(self):
workspace_runtime = MagicMock(name="workspace_runtime")
ws = MagicMock(name="ws")
proc = MagicMock(name="proc")
runner = MagicMock()
runner.start_program = AsyncMock(return_value=proc)
inputs = ExecInput(skill="s", command="echo hi")

session = await _start_session(
runner=runner,
tool_context=MagicMock(),
inputs=inputs,
ws=ws,
workspace_runtime=workspace_runtime,
rel_cwd="skills/s",
env={"A": "1"},
)

assert isinstance(session, _ExecSession)
assert session.workspace_runtime is workspace_runtime
assert session.ws is ws
assert session.proc is proc
assert session.in_data is inputs


class TestCollectFinalResult:
"""Regression coverage for issue #350.

``_prepare_outputs`` requires ``workspace_runtime``. A plain ``AsyncMock``
swallows the missing-argument ``TypeError``, so the collection path silently
degrades to empty files. These tests bind a signature-strict stub so a
dropped argument fails the assertions instead of being absorbed.
"""

def _make_session(self, workspace_runtime, in_data=None, run_result=None):
proc = MagicMock()
proc.run_result = AsyncMock(return_value=run_result or MagicMock(
stdout="done\n",
stderr="",
exit_code=0,
timed_out=False,
duration=0.0,
))
return _ExecSession(
proc=proc,
ws=MagicMock(name="ws"),
workspace_runtime=workspace_runtime,
in_data=in_data or ExecInput(skill="s", command="echo hi", output_files=["out/result.txt"]),
)

def _make_run_tool(self, prepare_side_effect=None, files=None):
run_tool = MagicMock()
run_tool._prepare_outputs = AsyncMock(
side_effect=prepare_side_effect,
return_value=(files if files is not None else [], None),
)
run_tool._attach_artifacts_if_requested = AsyncMock()
run_tool._merge_manifest_artifact_refs = MagicMock()
return run_tool

@pytest.mark.asyncio
async def test_collect_final_result_passes_workspace_runtime_and_files(self):
workspace_runtime = MagicMock(name="workspace_runtime")
session = self._make_session(workspace_runtime)
expected_files = [SkillRunFile(
name="out/result.txt",
content="ok",
mime_type="text/plain",
size_bytes=2,
)]

async def _prepare_outputs(ctx, ws, runtime, input_data):
# Signature-strict: a missing argument raises TypeError here and
# the production `except Exception` would degrade to empty files.
assert runtime is workspace_runtime
assert input_data.output_files == ["out/result.txt"]
return list(expected_files), None

run_tool = self._make_run_tool(prepare_side_effect=_prepare_outputs)
ctx = MagicMock()
result = await _collect_final_result(ctx, session, run_tool)

assert result is not None
assert result.exit_code == 0
assert result.output_files == expected_files
assert result.primary_output is not None
assert result.primary_output.name == "out/result.txt"
assert result.timed_out is False
assert result.stderr == ""
assert session.finalized is True
assert session.final_result is result

run_tool._prepare_outputs.assert_awaited_once()
call_args = run_tool._prepare_outputs.await_args.args
expected_input = SkillRunInput(
skill="s",
command="echo hi",
output_files=["out/result.txt"],
)
assert call_args == (ctx, session.ws, workspace_runtime, expected_input)

@pytest.mark.asyncio
async def test_collect_final_result_returns_cached_when_finalized(self):
session = self._make_session(MagicMock())
cached = MagicMock(name="cached")
session.finalized = True
session.final_result = cached

run_tool = self._make_run_tool()

result = await _collect_final_result(MagicMock(), session, run_tool)
assert result is cached
run_tool._prepare_outputs.assert_not_awaited()

@pytest.mark.asyncio
async def test_collect_final_result_omit_inline_content_clears_file_content(self):
in_data = ExecInput(
skill="s",
command="echo hi",
output_files=["out/result.txt"],
omit_inline_content=True,
)
session = self._make_session(MagicMock(), in_data=in_data)
collected = [SkillRunFile(
name="out/result.txt",
content="secret payload",
mime_type="text/plain",
size_bytes=14,
)]
run_tool = self._make_run_tool(files=collected)

result = await _collect_final_result(MagicMock(), session, run_tool)

assert result is not None
assert len(result.output_files) == 1
assert result.output_files[0].content == ""
assert result.output_files[0].name == "out/result.txt"
assert result.output_files[0].size_bytes == 14
assert result.primary_output is not None
assert result.primary_output.content == ""

@pytest.mark.asyncio
async def test_collect_final_result_propagates_timed_out(self):
run_result = MagicMock(stdout="", stderr="", exit_code=124, timed_out=True, duration=1.5)
session = self._make_session(MagicMock(), run_result=run_result)
collected = [SkillRunFile(name="out/partial.txt", content="", mime_type="text/plain", size_bytes=0)]
run_tool = self._make_run_tool(files=collected)

result = await _collect_final_result(MagicMock(), session, run_tool)

assert result is not None
assert result.timed_out is True
assert result.exit_code == 124
assert result.duration_ms == 1500
# Failed + timed_out run drops empty output files (same as skill_run).
assert result.output_files == []
assert any("empty output_files" in w for w in result.warnings)

@pytest.mark.asyncio
async def test_collect_final_result_fills_stderr(self):
run_result = MagicMock(stdout="out\n", stderr="boom\n", exit_code=1, timed_out=False, duration=0.25)
session = self._make_session(MagicMock(), run_result=run_result)
run_tool = self._make_run_tool()

result = await _collect_final_result(MagicMock(), session, run_tool)

assert result is not None
assert result.stdout == "out\n"
assert result.stderr == "boom\n"
assert result.duration_ms == 250

@pytest.mark.asyncio
async def test_collect_final_result_warns_on_collection_failure(self):
async def _prepare_outputs(ctx, ws, runtime, input_data):
raise RuntimeError("glob failed")

session = self._make_session(MagicMock())
run_tool = self._make_run_tool(prepare_side_effect=_prepare_outputs)

result = await _collect_final_result(MagicMock(), session, run_tool)

assert result is not None
assert result.output_files == []
assert any("output collection failed" in w for w in result.warnings)
# Still finalized so later polls do not retry a broken collection.
assert session.finalized is True

@pytest.mark.asyncio
async def test_collect_final_result_exception_branch_survives_log_failure(self):
session = self._make_session(MagicMock())
session.proc.run_result = AsyncMock(side_effect=RuntimeError("session gone"))
session.proc.log = AsyncMock(side_effect=RuntimeError("log gone"))
session.exit_code = None
run_tool = self._make_run_tool()

result = await _collect_final_result(MagicMock(), session, run_tool)

assert result is not None
assert result.stdout == ""
# Unknown exit code is surfaced via warnings instead of silent success.
assert result.exit_code == 0
assert any("exit code unavailable" in w for w in result.warnings)
assert session.finalized is True
54 changes: 44 additions & 10 deletions trpc_agent_sdk/skills/tools/_skill_exec.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
from pydantic import Field
from trpc_agent_sdk.code_executors import BaseProgramRunner
from trpc_agent_sdk.code_executors import BaseProgramSession
from trpc_agent_sdk.code_executors import BaseWorkspaceRuntime
from trpc_agent_sdk.code_executors import DEFAULT_EXEC_YIELD_MS
from trpc_agent_sdk.code_executors import DEFAULT_IO_YIELD_MS
from trpc_agent_sdk.code_executors import DEFAULT_SESSION_KILL_SEC
Expand Down Expand Up @@ -90,6 +91,8 @@
from ._skill_run import SkillRunInput
from ._skill_run import SkillRunOutput
from ._skill_run import SkillRunTool
from ._skill_run import _WARN_STDERR_TRUNCATED
from ._skill_run import _WARN_STDOUT_TRUNCATED
from ._skill_run import _filter_failed_empty_outputs
from ._skill_run import _select_primary_output
from ._skill_run import _truncate_output
Expand Down Expand Up @@ -216,6 +219,7 @@ class _ExecSession:

proc: BaseProgramSession
ws: WorkspaceInfo
workspace_runtime: BaseWorkspaceRuntime
in_data: ExecInput

# Final state
Expand Down Expand Up @@ -458,6 +462,7 @@ async def _run_async_impl(
tool_context=tool_context,
inputs=inputs,
ws=ws,
workspace_runtime=workspace_runtime,
rel_cwd=rel_cwd,
env=merged_env,
)
Expand Down Expand Up @@ -730,6 +735,7 @@ async def _start_session(
tool_context: InvocationContext,
inputs: ExecInput,
ws: WorkspaceInfo,
workspace_runtime: BaseWorkspaceRuntime,
rel_cwd: str,
env: dict[str, str],
) -> _ExecSession:
Expand All @@ -744,7 +750,7 @@ async def _start_session(
tty=inputs.tty,
)
proc = await runner.start_program(tool_context, ws, spec)
return _ExecSession(proc=proc, ws=ws, in_data=inputs)
return _ExecSession(proc=proc, ws=ws, workspace_runtime=workspace_runtime, in_data=inputs)


async def _write_stdin(exec_session: _ExecSession, chars: str, submit: bool) -> None:
Expand Down Expand Up @@ -778,35 +784,56 @@ async def _collect_final_result(
inputs=in_data.inputs,
outputs=in_data.outputs,
)
warnings: list[str] = []
try:
files, manifest = await run_tool._prepare_outputs(ctx, exec_session.ws, fake_run_input)
files, manifest = await run_tool._prepare_outputs(ctx, exec_session.ws, exec_session.workspace_runtime,
fake_run_input)
Comment on lines +789 to +790

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

问题: _collect_final_result 在此变更行启动的真实收集路径中,将 omit_inline_content 透传给 fake_run_input(第 781 行),但组装 SkillRunOutput 时从不把 output_files/primary_output 的 content 置空。对照 _skill_run.py:742-747 在 skill_run 路径会按该标志清空内容。本次变更前 _prepare_outputs 因缺参必然抛 TypeError 被吞掉、files 恒为空,该标志没有任何可观察效果;本次变更在调用处(即本行)补上 workspace_runtime 使收集真正生效后,标志被静默忽略成为可观察缺陷。

触发条件: 调用 skill_exec(或经 skill_poll_session/skill_write_stdin 完成最终收集)且 ExecInput 设置 omit_inline_content=True、声明了 output_files 或 outputs 产物,_prepare_outputs 返回非空 files 后进入后续组装。

实际影响: 调用方按契约(ExecInput.omit_inline_content 字段描述及 _skill_processor.py 对模型的引导:设置该标志后 output_files 只返回元数据、用 ref 按需读取)期望省略内联内容,实际却拿到全部文件全文,与 skill_run 行为不一致;大文件内容直接膨胀进工具响应,消耗模型上下文预算,本 PR 之前不会发生。

修正方向: 在 _collect_final_result 组装 SkillRunOutput 前,参照 _skill_run.py:742-747,当 in_data.omit_inline_content 为真时对 files 及选出的 primary_output 置空 content,并补充该场景的回归测试。

Comment on lines +789 to +790

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

问题: 本次变更使 _prepare_outputs 收集路径首次真正执行(本行新增的第四个参数 exec_session.workspace_runtime 修复了收集恒抛 TypeError 的问题),但后续组装时丢弃了 WorkspaceRunResult.timed_out 字段(LocalProgramSession.run_result 返回 timed_out=self._timed_out,命令超时被杀时为真),_filter_failed_empty_outputs(exit_code, False, files) 硬编码 timed_out=False,SkillRunOutput.timed_out 恒为默认 False。对照 _skill_run.py:722/731 会把真实 result.timed_out 传入过滤器并写入输出。此前收集恒失败、结果恒为空,本缺陷被掩盖;本次变更使其变为可观察。

触发条件: skill_exec 设置 timeout 且命令实际超时被本地运行器杀死(enforce_timeout 置 _timed_out=True、exit_code 非 0),随后在 status==_STATUS_EXITED 时进入 _collect_final_result。

实际影响: 超时被杀的命令被报告为普通失败:按 skill_run 语义应被丢弃的空输出文件被保留、_WARN_FAILED_RUN_EMPTY_OUTPUTS 不产生,下游(LLM/调用方)无法区分“超时中止”与“正常失败”,输出契约与 skill_run 不一致。

修正方向: 从 run_result() 中取出 timed_out 并透传给 _filter_failed_empty_outputs(exit_code, timed_out, files),同时写入 SkillRunOutput(timed_out=timed_out)(异常分支可保留现有兜底)。

Comment on lines +789 to +790

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

问题: 本次变更使输出收集(_collect_final_result 内、本变更行所在的 _prepare_outputs 调用)首次真正生效,但 KillSessionTool._run_async_impl 与 TTL 回收路径(_gc_expired_sessions → cleanup_expired_sessions → _close_session)仍只做 proc.close() 和会话移除,从不调用 _collect_final_result。输出收集仅存在于 SkillExecTool/WriteStdinTool/PollSessionTool 三个入口,且都以 status==_STATUS_EXITED and not exec_session.finalized 为前置,这两条路径仍会把结果静默丢弃。

触发条件: 交互进程已退出而 LLM 未先成功轮询到退出态,直接调用 skill_kill_session(proc.poll(None) 返回已退出时 kill 仍然 ok=True),或会话退出了但 ttl 内没有任何轮询观察到退出而被 _gc_expired_sessions 逐出(_refresh_exit_state 会主动把 exited_at 打上,使逐出成为必然)。

实际影响: 会话的最终 SkillRunOutput(output_files、save_as_artifacts 持久化侧效应、artifact 状态增量)永远不会生成,且 skill_kill_session 仍返回 ok=True, status="exited" 造成结果已交付的假象;快速退出且未轮询的交互会话输出静默丢失。

修正方向: 在 kill 与 TTL 逐出路径中,若 not exec_session.finalized 且进程已退出,先调用 _collect_final_result(带上当时上下文)再关闭/移除会话,使结果与 artifact 侧效应不被静默丢弃。

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个问题我认同,但是它涉及 skill_kill_session / TTL 回收时的 finalization 语义,以及是否需要在清理路径里触发 artifact/output 收集,范围已经超出 #350 这次修复。
本 PR 先聚焦恢复现有 _collect_final_result 路径,kill / TTL 路径我建议单独开 issue 跟进,避免把这次 bug fix 扩成 session lifecycle 的行为调整。

Comment on lines +789 to +790

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

问题: 本次变更使 _collect_final_result 的异常分支首次可达(本变更行所在的 _prepare_outputs 调用不再恒抛 TypeError,run_result() 才可能被真正执行并进入其 except 处理器),该处理器内 await exec_session.proc.log(None, None) 位于 except 内部且无嵌套保护,而三个调用点(475/549/613)外层均无 try/except。若 log 同样抛出,异常将直接逃逸出 _collect_final_result,把整个工具调用打成未分类错误;同时该分支 exit_code = exec_session.exit_code or 0 会把 None(未知退出码)映射成成功 0。

触发条件: run_result() 抛出(如会话正被 GC/关闭等终态竞态)后 log() 在异常会话状态下再次抛出;或 exit_code 尚未被任何轮询填充时走异常分支。

实际影响: 静默降级路径变成向上抛异常(本来旨在优雅返回空结果以结束交互链,反而中断整个工具调用);or 0 则使一次实际失败/被杀但未知退出码的运行被报告为 exit_code=0 的成功结果,_filter_failed_empty_outputs 因而保留空输出文件且无警告。

修正方向: 给异常分支的 log() 调用再加一层 try/except 兜底为 total_out="";exit_code 使用 exec_session.exit_code if exec_session.exit_code is not None else 0 显式处理 None,避免错误地报告成功。

Comment on lines +789 to +790

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

问题: 本次变更使收集路径首次真正执行(本变更行补上 workspace_runtime 后 _prepare_outputs 不再恒抛 TypeError),但 _collect_final_result 组装 SkillRunOutput 时把 stdout 与 stderr 合并进 stdout(第 794 行 total_out),stderr 字段恒为默认空串,duration_ms 恒为 0;而 _skill_run.py:717-721 分别填充 stderr、duration_ms。结果路径激活后,这些输出契约不一致成为可观察行为。

触发条件: 交互命令向 stderr 输出内容(例如 20KB 的错误堆栈)且成功收集;或调用方检查 result.stderr/result.duration_ms 做下游判断。

实际影响: 按 result.stderr 非空判断失败的下游逻辑拿到空串而误判;合并后的 stdout 被截断时警告文案仍是“stdout truncated”(_truncate_output 只作用于合并串),诊断信息归属错误;duration_ms 缺失使依赖耗时的调用方拿到 0。

修正方向: 从 run_result() 分别取 stdout/stderr 并各自 _truncate_output 后分别写入 SkillRunOutput.stdout/stderr,同时填充 duration_ms(异常分支保留合并兜底),使与 skill_run 的输出契约一致。

except Exception as ex: # pylint: disable=broad-except
logger.warning("skill_exec: collect outputs failed: %s", ex)
files, manifest = [], None
Comment on lines 788 to 793

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

问题: 本次变更修复了 _prepare_outputs 的缺参 TypeError(第 786-787 行),使收集首次真正执行,但其外包的 except Exception: files, manifest = [], None(仅 logger.warning)把所有失败(FS 读写/glob 错误、未来的签名漂移——正是 #350 那一类)都降级为空文件,且 SkillRunOutput 没有任何字段或 warnings 条目用于标识“收集失败”,与“程序真没产出”在消费者侧完全无法区分;_filter_failed_empty_outputs 仅在 exit_code!=0 且有文件被丢弃时才有警告,files==[]、exit_code==0 时无任何提示。新测试只覆盖成功路径,未覆盖该分支。

触发条件: 进程退出后收集阶段 _prepare_outputs 抛出任意异常(如本地 collect_files_with_glob 对权限/路径错误抛出的 Exception,或今后对 _prepare_outputs 签名的再次改动)。

实际影响: 用户得到 output_files==[]、primary_output==None、无任何 warnings 的“成功”结果,LLM 会据此得出“程序没有产出”的错误结论(重复运行或误报),save_as_artifacts 侧效应也静默消失;这正是 #350 的症状通过另一条开口复现的路径。

修正方向: 将收集失败显式表达:在该分支向 warnings 追加可消费者可见的提示(如 output collection failed: ...)或给 SkillRunOutput 增加标识字段,并补充失败路径测试,使 #350 类的调用侧错误不再被静默吞掉。

warnings.append(f"output collection failed: {ex}")

timed_out = False
duration_ms = 0
try:
run_result = await exec_session.proc.run_result()
total_out = (run_result.stdout or "") + (run_result.stderr or "")
stdout_raw = run_result.stdout or ""
stderr_raw = run_result.stderr or ""
exit_code = run_result.exit_code
timed_out = bool(getattr(run_result, "timed_out", False))
duration_ms = int((getattr(run_result, "duration", 0) or 0) * 1000)
except Exception: # pylint: disable=broad-except
run_log = await exec_session.proc.log(None, None)
total_out = run_log.output or ""
exit_code = exec_session.exit_code or 0
try:
run_log = await exec_session.proc.log(None, None)
stdout_raw = run_log.output or ""
except Exception: # pylint: disable=broad-except
stdout_raw = ""
stderr_raw = ""
if exec_session.exit_code is not None:
exit_code = exec_session.exit_code
else:
exit_code = 0
warnings.append("exit code unavailable after run_result failure; reported as 0")

# Reuse the same output-quality helpers as skill_run
warnings: list[str] = []
stdout, trunc = _truncate_output(total_out)
stdout, trunc = _truncate_output(stdout_raw)
if trunc:
warnings.append("stdout truncated")
warnings.append(_WARN_STDOUT_TRUNCATED)
stderr, trunc = _truncate_output(stderr_raw)
if trunc:
warnings.append(_WARN_STDERR_TRUNCATED)

files, filter_warns = _filter_failed_empty_outputs(exit_code, False, files)
files, filter_warns = _filter_failed_empty_outputs(exit_code, timed_out, files)
warnings.extend(filter_warns)

primary = _select_primary_output(files)

result = SkillRunOutput(
stdout=stdout,
stderr=stderr,
exit_code=exit_code,
timed_out=timed_out,
duration_ms=duration_ms,
output_files=files,
primary_output=primary,
warnings=warnings,
Expand All @@ -820,6 +847,13 @@ async def _collect_final_result(
if manifest:
run_tool._merge_manifest_artifact_refs(manifest, result)

# omit_inline_content: keep metadata only, match skill_run contract
if in_data.omit_inline_content:
for f in result.output_files:
f.content = ""
if result.primary_output:
result.primary_output.content = ""

exec_session.final_result = result
exec_session.finalized = True
return result
Expand Down
Loading