Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 67 additions & 100 deletions architecture.md

Large diffs are not rendered by default.

12 changes: 2 additions & 10 deletions packages/cli/src/deno-workflow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ import { gitWorkspaceAttachment } from "@executablemd/git/deno";
import type { WorkflowExecutionTransitions } from "@executablemd/workflow/deno";
import type { WorkflowRunDatabase } from "@executablemd/workflow";
import type { HelperAssembly } from "@executablemd/git/credential-helper";
import { readLegacyDefinitionSource } from "./workflow-source.ts";
import type { WorkflowHost } from "./workflow.ts";
import { useWorkflowAgentProfile } from "./workflow-agent.ts";

Expand All @@ -49,17 +48,10 @@ export function* useDenoWorkflowHost(helper: HelperAssembly): Operation<Workflow
configured === undefined || configured === "" ? DEFAULT_RUN_STORAGE_ROOT : configured;
return {
useRunHost(): Operation<WorkflowExecutionTransitions> {
// The same reader the lifecycle installation captures. A version-1 run
// needs its Markdown to begin, fork or stage, and the transitions this
// returns are what admit those β€” so a host that could export a legacy run
// and not resume one would be two hosts wearing one name.
return useWorkflowRunHost({ root, legacySource: readLegacyDefinitionSource });
return useWorkflowRunHost({ root });
},
useLifecycle(): Operation<void> {
// The reader goes into the provider's closure, not onto a request. An
// export seals the document a run was of, and a caller that could hand
// that in would be sealing its own bytes as somebody else's evidence.
return useWorkflowLifecycle({ root, legacySource: readLegacyDefinitionSource });
return useWorkflowLifecycle({ root });
},
useDelivery(): Operation<void> {
return useWorkflowInputDelivery({ root });
Expand Down
89 changes: 0 additions & 89 deletions packages/cli/src/workflow-bundle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,6 @@
* core resolves the names against. Because the hash is identity, changing what
* a component says changes the definition rather than changing what a retained
* definition executes.
*
* ## The legacy half
*
* `reconstructBundle()` rebuilds a version-1 run's bundle out of the commit it
* pinned, through the contextual `Git` capability. It is reached only from the
* legacy source reader this host supplies to the Workflow lifecycle; nothing
* establishing a new bundle goes near it.
*/

import { readFile } from "node:fs/promises";
Expand All @@ -47,7 +40,6 @@ import {
import type { WorkflowBundleComponent } from "@executablemd/core/host";
import { decodeSourceText, sourceContentHash } from "@executablemd/workflow";
import { readGitObject, resolveGitRevision } from "@executablemd/git/api";
import type { WorkflowComponentEntry } from "@executablemd/workflow";
import type { GitObjectFormat } from "@executablemd/git/api";
import type { EstablishedComponent } from "./workflow-definition.ts";

Expand Down Expand Up @@ -352,84 +344,3 @@ function* parseComponent(name: string, path: string, content: string): Operation
);
}
}

/**
* Rebuild the execution view of a bundle a run already retained.
*
* The retained entries are definitive: each is read from the retained
* commit, at the retained path, and the blob's object id must be the hash the
* definition holds. A component whose object changed, went missing, or is no
* longer a blob refuses the resume β€” it does not resolve to whatever is there
* now, which would continue a procedure under code it never executed.
*/
export function* reconstructBundle(
pinnedCommit: string,
retained: readonly WorkflowComponentEntry[],
objectFormat: GitObjectFormat,
): Operation<Result<readonly WorkflowBundleComponent[]>> {
const loaded = yield* readGitBundle(
pinnedCommit,
retained.map((entry) => ({ name: entry.name, path: entry.path })),
objectFormat,
);
if (!loaded.ok) {
return loaded;
}
for (const [index, component] of loaded.value.entries()) {
if (component.sourceHash !== retained[index]?.sourceHash) {
return Err(
unavailable(
`the component "${component.name}" is no longer the object this run's definition ` +
"names. The run is left exactly as it is.",
),
);
}
}
return loaded;
}

/**
* Read every retained component out of one commit, for a version-1 resume.
*
* The legacy half, and the only place in this module that reaches Git. A
* version-1 definition pins a commit and a path per component, so its bundle is
* rebuilt from objects rather than from files β€” a working tree edited since the
* run started continues the run it started.
*/
function* readGitBundle(
pinnedCommit: string,
declared: readonly DeclaredComponent[],
objectFormat: GitObjectFormat,
): Operation<Result<readonly WorkflowBundleComponent[]>> {
const components: WorkflowBundleComponent[] = [];
for (const { name, path } of declared) {
let content: string;
let sourceHash: string;
try {
// `cat-file blob` first: it refuses a tree, so a declaration that names a
// directory fails before its object id is taken as a component's identity.
content = yield* readGitObject(pinnedCommit, path);
sourceHash = (yield* resolveGitRevision(`${pinnedCommit}:${path}`)).toLowerCase();
} catch (error) {
return Err(
unavailable(
`the component "${name}" is not a file this workflow's commit holds at ${path}.`,
error,
),
);
}
if (sourceHash.length !== OBJECT_ID_LENGTHS[objectFormat] || !/^[0-9a-f]+$/.test(sourceHash)) {
return Err(
unavailable(
`the component "${name}" did not resolve to an object this repository's format names.`,
),
);
}
const parsed = yield* parseComponent(name, path, content);
if (!parsed.ok) {
return parsed;
}
components.push({ name, path, sourceHash, content });
}
return Ok(Object.freeze(components));
}
126 changes: 11 additions & 115 deletions packages/cli/src/workflow-definition.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,6 @@
* started from when that is cheaply available, as replaceable metadata; failing
* to learn it does not fail a start, and nothing ever reads it back to find the
* source. The source is in the run.
*
* ## The legacy path
*
* Version-1 runs still exist, and their Markdown still lives in a repository.
* `loadRetainedDefinition()` is what reaches it, and it is used from exactly one
* place: the adapter this host hands the Workflow lifecycle as its legacy
* source reader. Nothing establishes a version-1 definition any more.
*/

import { readFile } from "node:fs/promises";
Expand All @@ -44,34 +37,32 @@ import type { DocumentInfo, FileRootDocument } from "@executablemd/core";
import type { WorkflowBundleComponent } from "@executablemd/core/host";
import {
decodeSourceText,
definitionComponents,
parseSourceBundleDefinition,
sourceBundleHash,
sourceContentHash,
} from "@executablemd/workflow";
import { gitObjectFormat, gitRoot, readGitObject, resolveGitRevision } from "@executablemd/git/api";
import { gitObjectFormat, gitRoot, resolveGitRevision } from "@executablemd/git/api";
import type {
GitWorkflowDefinitionV1,
SourceBundleEntryV2,
SourceBundleSnapshotEntryV2,
SourceBundleWorkflowDefinitionV2,
SourceBundleEntry,
SourceBundleSnapshotEntry,
WorkflowDefinition,
} from "@executablemd/workflow";
import { declaredBundle, readBundle, reconstructBundle } from "./workflow-bundle.ts";
import { declaredBundle, readBundle } from "./workflow-bundle.ts";

/** How this host recorded where a run was started from. Never read back. */
export const RETRIEVAL_KIND = "local-checkout";

/** Everything one `start` establishes before a run can exist. */
export interface EstablishedDefinition {
readonly definition: SourceBundleWorkflowDefinitionV2;
readonly definition: WorkflowDefinition;
/**
* The exact bytes behind every logical path, in the descriptor's own order.
*
* Owned copies from the one read of each file. They travel to the lifecycle
* transition, which copies them again before it validates β€” so nothing
* between here and storage can change what the run is of.
*/
readonly sourceSnapshot: readonly SourceBundleSnapshotEntryV2[];
readonly sourceSnapshot: readonly SourceBundleSnapshotEntry[];
/** Credential-free provenance, when it was cheaply available. */
readonly retrieval?: Json;
/** The entrypoint as text, for the caller that is about to import it. */
Expand Down Expand Up @@ -252,8 +243,8 @@ function withProvenance(retrieval: Json | undefined): { retrieval?: Json } {

/** What one established candidate is, descriptor and bytes together. */
interface BuiltBundle {
readonly definition: SourceBundleWorkflowDefinitionV2;
readonly sourceSnapshot: readonly SourceBundleSnapshotEntryV2[];
readonly definition: WorkflowDefinition;
readonly sourceSnapshot: readonly SourceBundleSnapshotEntry[];
}

/**
Expand Down Expand Up @@ -335,8 +326,8 @@ function* buildSourceBundle(
}

const ordered = [...byPath.keys()].sort(compareUtf8);
const sources: SourceBundleEntryV2[] = [];
const sourceSnapshot: SourceBundleSnapshotEntryV2[] = [];
const sources: SourceBundleEntry[] = [];
const sourceSnapshot: SourceBundleSnapshotEntry[] = [];
for (const path of ordered) {
const bytes = byPath.get(path);
if (bytes === undefined) {
Expand Down Expand Up @@ -364,8 +355,6 @@ function* buildSourceBundle(
// goes through the same closed parser storage reads one back through, so a
// candidate that is not canonical is refused here rather than retained.
const definition = parseSourceBundleDefinition({
version: 2,
kind: "source-bundle",
hashAlgorithm: "sha256",
bundleHash,
entrypoint,
Expand Down Expand Up @@ -444,96 +433,3 @@ function* provenance(directory: string): Operation<Json | undefined> {
function describeCause(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}

/**
* The checkout a retained version-1 locator names, reauthorized before use.
*
* A retained path is replaceable metadata rather than permission a host already
* has, so it is checked against the repository it claims to be: a directory
* that is no longer a working tree, or is now a different one, fails rather
* than quietly resolving the definition somewhere else.
*/
function parseRetrieval(metadata: Json | undefined): Result<string> {
if (typeof metadata !== "object" || metadata === null || Array.isArray(metadata)) {
return Err(
unavailable(
"this run retains no usable retrieval metadata, so its definition cannot be located. " +
"The run is left exactly as it is.",
),
);
}
const record = Object.fromEntries(Object.entries(metadata));
if (record.kind !== RETRIEVAL_KIND || typeof record.checkout !== "string") {
return Err(
unavailable(
"this run's retrieval metadata does not describe a local checkout this host can " +
"reach. The run is left exactly as it is.",
),
);
}
return Ok(record.checkout);
}

/**
* Load the exact object a retained version-1 definition names.
*
* It never substitutes the current `HEAD` or a same-named file in the working
* tree. A resume that could do either would silently continue a different
* document under the same run id.
*
* Reached from one place only: the legacy source reader this host supplies to
* the Workflow lifecycle. Nothing else in the CLI loads a definition β€” a
* version-2 run's source comes out of the run.
*/
export function* loadRetainedDefinition(
definition: GitWorkflowDefinitionV1,
metadata: Json | undefined,
): Operation<Result<RetainedSources>> {
const checkout = parseRetrieval(metadata);
if (!checkout.ok) {
return checkout;
}

try {
return yield* inDirectory(checkout.value, function* (): Operation<Result<RetainedSources>> {
const root = yield* gitRoot();
if (resolve(root) !== resolve(checkout.value)) {
return Err(
unavailable(
"the checkout this run retains is no longer the root of the repository it names. " +
"The run is left exactly as it is.",
),
);
}
const format = yield* gitObjectFormat();
if (format !== definition.objectFormat) {
return Err(
unavailable(
"the retained checkout names its objects with a different format than this run's " +
"definition. The run is left exactly as it is.",
),
);
}
const source = yield* readGitObject(definition.objectId, definition.rootDocumentPath);
// Every retained component, from the retained commit, verified against
// the hash the definition holds. The working tree is never consulted, so
// a checkout edited since the run started continues the run it started.
const retained = definitionComponents(definition);
const components =
retained.length === 0
? Ok([])
: yield* reconstructBundle(definition.objectId, retained, format);
if (!components.ok) {
return components;
}
return Ok({ source, components: components.value });
});
} catch (error) {
return Err(
unavailable(
"this run's retained definition could not be loaded: " + describeCause(error),
error,
),
);
}
}
5 changes: 2 additions & 3 deletions packages/cli/src/workflow-fork.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ import type { ExecutionInstallation } from "@executablemd/core/host";
import type { ForkSelection, WorkflowRun } from "@executablemd/workflow";
import type { WorkflowRunDatabase } from "@executablemd/workflow";
import type {
SourceBundleWorkflowRunCreationV2,
WorkflowRunCreation,
WorkflowExecutionTransitions,
} from "@executablemd/workflow/deno";
import type { EstablishedDefinition } from "./workflow-definition.ts";
Expand All @@ -85,7 +85,7 @@ export interface ForkRequest {
readonly sourceRunId: string;
readonly checkpointEventId: string;
readonly established: EstablishedDefinition;
readonly creation: SourceBundleWorkflowRunCreationV2;
readonly creation: WorkflowRunCreation;
}

/**
Expand Down Expand Up @@ -149,7 +149,6 @@ export function* preflightFork(
// The fork's own run value, in the shape its candidate's version declares.
const run: WorkflowRun = {
runId: request.runId,
definitionVersion: 2,
bundleHash: request.creation.definition.bundleHash,
...(request.creation.definition.targetPath === undefined
? {}
Expand Down
12 changes: 2 additions & 10 deletions packages/cli/src/workflow-management.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,7 @@ import { exists, rm } from "@effectionx/fs";
import { linkSync, mkdtempSync, rmSync } from "node:fs";
import { rename } from "node:fs/promises";
import { dirname, join, resolve } from "node:path";
import {
isGitWorkflowRunRecord,
WorkflowInputDelivery,
WorkflowLifecycle,
} from "@executablemd/workflow";
import { WorkflowInputDelivery, WorkflowLifecycle } from "@executablemd/workflow";
import type {
DefinitionRetrieval,
WorkflowArtifactIdentity,
Expand Down Expand Up @@ -234,7 +230,6 @@ function renderStatus(
// Only a Git run has one. A source-bundle run started from exact bytes
// rather than from a repository state, and printing a base for it would be
// naming a revision it never had.
...(isGitWorkflowRunRecord(record) ? [`base: ${record.base}`] : []),
`props: ${JSON.stringify(record.props)}`,
`created: ${record.createdAt}`,
`updated: ${record.updatedAt}`,
Expand Down Expand Up @@ -393,10 +388,7 @@ function describeResult(event: DurableEvent): string {
function describeDefinition(snapshot: WorkflowInspectionSnapshot): string {
const { definition } = snapshot.record;
const target = definition.targetPath === undefined ? "" : `#${definition.targetPath}`;
if (definition.kind === "source-bundle") {
return `${definition.bundleHash} ${definition.entrypoint}${target}`;
}
return `${definition.objectId} ${definition.rootDocumentPath}${target}`;
return `${definition.bundleHash} ${definition.entrypoint}${target}`;
}

function describeReason(record: WorkflowRunRecord): string {
Expand Down
Loading
Loading