Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion products/desktop/apps/web/src/web-skill-bundler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,11 @@ export async function bundleExportedSkill(
// split), matching how desktop's installTeamSkill writes it to disk.
files["SKILL.md"] = strToU8(
serializeSkillMarkdown(
{ name: exported.name, description: exported.description },
{
name: exported.name,
description: exported.description,
disableModelInvocation: exported.disableModelInvocation,
},
exported.body,
),
);
Expand Down
2 changes: 2 additions & 0 deletions products/desktop/packages/api-client/src/posthog-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5622,6 +5622,7 @@ export class PostHogAPIClient {
description: string;
body: string;
files?: LlmSkillFileInput[];
metadata?: Record<string, unknown>;
}): Promise<LlmSkill> {
const teamId = await this.getTeamId();
const urlPath = `/api/environments/${teamId}/llm_skills/`;
Expand Down Expand Up @@ -5654,6 +5655,7 @@ export class PostHogAPIClient {
body: string;
description?: string;
files?: LlmSkillFileInput[];
metadata?: Record<string, unknown>;
base_version: number;
},
): Promise<LlmSkill> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,71 @@ describe("TeamSkillsService.publishSkill", () => {
body: "# Body",
description: "Shepherds PRs",
files: exported.files,
metadata: {},
base_version: 2,
});
expect(result).toEqual({ version: 3 });
});

it("stores disable-model-invocation in metadata on first publish", async () => {
const createLlmSkill = vi.fn().mockResolvedValue(makeItem({ version: 1 }));
const client = {
listLlmSkills: vi.fn().mockResolvedValue([]),
createLlmSkill,
} as unknown as PostHogAPIClient;

await makeService().publishSkill(client, {
...exported,
disableModelInvocation: true,
});

expect(createLlmSkill).toHaveBeenCalledWith(
expect.objectContaining({
metadata: { "disable-model-invocation": true },
}),
);
});

it.each([
{
case: "sets the key and keeps other metadata",
disableModelInvocation: true as const,
existingMetadata: { author: "dev" },
expected: { author: "dev", "disable-model-invocation": true },
},
{
case: "clears the key when the flag is gone",
disableModelInvocation: undefined,
existingMetadata: { author: "dev", "disable-model-invocation": true },
expected: { author: "dev" },
},
])(
"republish $case",
async ({ disableModelInvocation, existingMetadata, expected }) => {
const publishLlmSkillVersion = vi
.fn()
.mockResolvedValue(makeItem({ version: 3 }));
const client = {
listLlmSkills: vi
.fn()
.mockResolvedValue([
makeItem({ version: 2, metadata: existingMetadata }),
]),
publishLlmSkillVersion,
} as unknown as PostHogAPIClient;

await makeService().publishSkill(client, {
...exported,
disableModelInvocation,
});

expect(publishLlmSkillVersion).toHaveBeenCalledWith(
"pr-shepherd",
expect.objectContaining({ metadata: expected }),
);
},
);

it("rejects publishing without a description", async () => {
await expect(
makeService().publishSkill(makeClient([]), {
Expand Down Expand Up @@ -207,6 +267,26 @@ describe("TeamSkillsService.fetchSkillForInstall", () => {
],
});
});

it("maps disable-model-invocation metadata onto the exported skill", async () => {
const client = {
getLlmSkillByName: vi.fn().mockResolvedValue({
name: "pr-shepherd",
description: "Shepherds PRs",
body: "# Body",
metadata: { "disable-model-invocation": true },
files: [],
}),
getLlmSkillFile: vi.fn(),
} as unknown as PostHogAPIClient;

const skill = await makeService().fetchSkillForInstall(
client,
"pr-shepherd",
);

expect(skill.disableModelInvocation).toBe(true);
});
});

describe("TeamSkillsService.publishLocalSkill", () => {
Expand Down
27 changes: 26 additions & 1 deletion products/desktop/packages/core/src/skills/teamSkillsService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ import type {
LlmSkillListItem,
PostHogAPIClient,
} from "@posthog/api-client/posthog-client";
import type { ExportedSkill } from "@posthog/shared";
import {
DISABLE_MODEL_INVOCATION_METADATA_KEY,
type ExportedSkill,
} from "@posthog/shared";
import { inject, injectable } from "inversify";
import { SKILLS_WORKSPACE_CLIENT } from "./identifiers";

Expand Down Expand Up @@ -127,13 +130,20 @@ export class TeamSkillsService {
body: exported.body,
description: exported.description,
files: exported.files,
metadata: withDisableModelInvocation(
existing.metadata,
exported.disableModelInvocation,
),
base_version: existing.latest_version ?? existing.version,
})
: await client.createLlmSkill({
name: exported.name,
description: exported.description,
body: exported.body,
files: exported.files,
...(exported.disableModelInvocation
? { metadata: { [DISABLE_MODEL_INVOCATION_METADATA_KEY]: true } }
: {}),
});

return { version: published.version };
Expand All @@ -158,11 +168,26 @@ export class TeamSkillsService {
name: detail.name,
description: detail.description,
body: detail.body,
...(detail.metadata?.[DISABLE_MODEL_INVOCATION_METADATA_KEY] === true
? { disableModelInvocation: true }
: {}),
files,
};
}
}

// Clearing the key keeps a republish that dropped the frontmatter from staying manual-only.
function withDisableModelInvocation(
metadata: Record<string, unknown> | undefined,
disableModelInvocation: boolean | undefined,
): Record<string, unknown> {
const { [DISABLE_MODEL_INVOCATION_METADATA_KEY]: _removed, ...rest } =
metadata ?? {};
return disableModelInvocation
? { ...rest, [DISABLE_MODEL_INVOCATION_METADATA_KEY]: true }
: rest;
}

function toTeamSkillInfo(item: LlmSkillListItem): TeamSkillInfo {
return {
id: item.id,
Expand Down
1 change: 1 addition & 0 deletions products/desktop/packages/shared/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,7 @@ export type {
UploadableSkillSource,
} from "./skills";
export {
DISABLE_MODEL_INVOCATION_METADATA_KEY,
SKILL_EXISTS_MARKER,
serializeSkillMarkdown,
stripFrontmatter,
Expand Down
12 changes: 11 additions & 1 deletion products/desktop/packages/shared/src/skills.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ export interface SkillInfo {
editable: boolean;
/** Size of SKILL.md in bytes (context-cost signal). */
skillMdBytes: number;
/** Frontmatter `disable-model-invocation: true`: only an explicit user invocation runs the skill, never the agent on its own. */
disableModelInvocation?: boolean;
}

export interface SkillFileEntry {
Expand All @@ -31,8 +33,11 @@ export interface ExportedSkill {
description: string;
body: string;
files: ExportedSkillFile[];
disableModelInvocation?: boolean;
}

export const DISABLE_MODEL_INVOCATION_METADATA_KEY = "disable-model-invocation";

/**
* Serializes a SKILL.md file from frontmatter metadata plus a markdown body.
*
Expand All @@ -44,13 +49,18 @@ export interface ExportedSkill {
* sandbox, so it must not drift between hosts.
*/
export function serializeSkillMarkdown(
meta: { name: string; description: string },
meta: {
name: string;
description: string;
disableModelInvocation?: boolean;
Comment thread
veria-ai[bot] marked this conversation as resolved.
},
body: string,
): string {
const frontmatter = [
"---",
`name: ${serializeSkillScalar(meta.name)}`,
`description: ${serializeSkillScalar(meta.description)}`,
...(meta.disableModelInvocation ? ["disable-model-invocation: true"] : []),
"---",
].join("\n");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,13 @@ export function SkillCard({
{skill.repoName}
</Badge>
)}
{skill.disableModelInvocation && (
<Tooltip content="The agent won't use this skill on its own. It runs only when you invoke it">
<Badge size="1" variant="soft" color="gray" className="shrink-0">
Manual
</Badge>
</Tooltip>
)}
</>
}
/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
DownloadSimple,
FilePlus,
Folder,
HandTap,
LockSimple,
PencilSimple,
Trash,
Expand Down Expand Up @@ -286,6 +287,14 @@ export function SkillDetailPanel({
Read-only
</Badge>
)}
{skill.disableModelInvocation && (
<Tooltip content="The agent won't use this skill on its own. It runs only when you invoke it">
<Badge size="1" variant="soft" color="gray">
<HandTap size={10} className="text-gray-9" />
Manual-only
</Badge>
</Tooltip>
)}
{skill.source === "codex" && (
<Button
size="1"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
import type { SkillInfo } from "@posthog/shared";
import { toast } from "@posthog/ui/primitives/toast";
import { Box, Button, Flex, Text, TextArea, TextField } from "@radix-ui/themes";
import {
Box,
Button,
Flex,
Switch,
Text,
TextArea,
TextField,
} from "@radix-ui/themes";
import { useRef, useState } from "react";
import { SkillCodeEditor } from "./SkillCodeEditor";
import { skillErrorDescription } from "./skillErrors";
Expand All @@ -25,6 +33,9 @@ export function SkillManifestEditor({
}: SkillManifestEditorProps) {
const [name, setName] = useState(skill.name);
const [description, setDescription] = useState(skill.description);
const [disableModelInvocation, setDisableModelInvocation] = useState(
skill.disableModelInvocation ?? false,
);
// Captured at mount: background refetches must not reset in-flight edits.
const [mountedBody] = useState(initialBody);
const bodyRef = useRef(mountedBody);
Expand All @@ -37,6 +48,7 @@ export function SkillManifestEditor({
name,
description,
body: bodyRef.current,
disableModelInvocation,
});
onSaved();
} catch (error) {
Expand Down Expand Up @@ -72,6 +84,22 @@ export function SkillManifestEditor({
placeholder="When should an agent use this skill?"
/>
</Box>
<Flex align="center" justify="between" gap="2">
<Box>
<Text className="block text-[12px] text-gray-12">
Manual invocation only
</Text>
<Text className="block text-[11px] text-gray-10">
The agent won't use this skill on its own. It runs only when you
invoke it
</Text>
</Box>
<Switch
size="1"
checked={disableModelInvocation}
onCheckedChange={setDisableModelInvocation}
/>
</Flex>
</Flex>

<Box className="min-h-0 flex-1 border-t border-t-(--gray-5)">
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,63 @@
import { describe, expect, it } from "vitest";
import { parseSkillDependencies } from "./parse-skill-frontmatter";
import {
parseSkillDependencies,
parseSkillFrontmatter,
} from "./parse-skill-frontmatter";

describe("parseSkillFrontmatter disable-model-invocation", () => {
it.each([
["absent", `---\nname: a\ndescription: d\n---\nbody`, false],
[
"true",
`---\nname: a\ndescription: d\ndisable-model-invocation: true\n---\nbody`,
true,
],
[
"capitalized True",
`---\nname: a\ndescription: d\ndisable-model-invocation: True\n---\nbody`,
true,
],
[
"quoted true",
`---\nname: a\ndescription: d\ndisable-model-invocation: "true"\n---\nbody`,
true,
],
[
"false",
`---\nname: a\ndescription: d\ndisable-model-invocation: false\n---\nbody`,
false,
],
[
"non-boolean value",
`---\nname: a\ndescription: d\ndisable-model-invocation: maybe\n---\nbody`,
false,
],
[
"true with trailing comment",
`---\nname: a\ndescription: d\ndisable-model-invocation: true # manual only\n---\nbody`,
true,
],
[
"quoted true with trailing comment",
`---\nname: a\ndescription: d\ndisable-model-invocation: "true" # manual only\n---\nbody`,
true,
],
[
"false with trailing comment",
`---\nname: a\ndescription: d\ndisable-model-invocation: false # keep automatic\n---\nbody`,
false,
],
[
"quoted string that only starts with true",
`---\nname: a\ndescription: d\ndisable-model-invocation: "true # manual only"\n---\nbody`,
false,
],
])("parses %s", (_label, content, expected) => {
expect(parseSkillFrontmatter(content)?.disableModelInvocation).toBe(
expected,
);
});
});

describe("parseSkillDependencies", () => {
it.each([
Expand Down
Loading
Loading