Skip to content

fix(sdk): prevent canceled health checks from stranding probes #83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aryasaatvik merged 2 commits into from
Sep 6, 2026
Merged

fix(sdk): prevent canceled health checks from stranding probes #83

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bfaab5d
fix(sdk): prevent canceled health checks from stranding probes
aryasaatvik Sep 6, 2026
430c513
test(sdk): verify detached health probe timeouts (greptile)
aryasaatvik Sep 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 87 additions & 1 deletion packages/core/sdk/src/connections.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { describe, expect, it } from "@effect/vitest";
import { describe, expect, it, vi } from "@effect/vitest";
import {
Cause,
Deferred,
Expand All @@ -11,6 +11,7 @@ import {
Predicate,
Result,
Schema,
Scheduler,
Tracer,
} from "effect";

Expand Down Expand Up @@ -3524,6 +3525,91 @@ describe("credential-only health path", () => {
});

describe("health probe gate lifecycle", () => {
it.effect("cancellation between gate registration and fork cannot strand later checks", () =>
Effect.gen(function* () {
const { executor, counters, stamp } = yield* makeHealthHarness();
yield* stamp({ oauth_client: "acme", expires_at: null });
const ref = { owner: "org", integration: INTEG, name: ConnectionName.make("main") } as const;
const controller = new AbortController();
let registered = false;
let interrupted = false;
let cancelOnResume = false;
const scheduler = new Scheduler.MixedScheduler("async", (resume) => {
const timer = setTimeout(() => {
if (cancelOnResume) {
cancelOnResume = false;
controller.abort();
}
resume();
}, 0);
return () => clearTimeout(timer);
});
scheduler.shouldYield = () => {
if (!registered || interrupted) return false;
interrupted = true;
cancelOnResume = true;
return true;
};

// Observe registration only to place cancellation in the otherwise tiny
// register/fork window. All writes retain their normal behavior, and the
// assertions below use the public health API.
const originalSet = Map.prototype.set;
const registration = vi.spyOn(Map.prototype, "set").mockImplementation(function (
this: Map<unknown, unknown>,
key: unknown,
value: unknown,
) {
const result = originalSet.call(this, key, value);
if (
typeof key === "string" &&
key.endsWith(',"vercel","main"]') &&
Deferred.isDeferred(value)
) {
registered = true;
}
return result;
});
yield* Effect.promise(() =>
Effect.runPromiseExit(executor.connections.checkHealth(ref), {
signal: controller.signal,
scheduler,
}),
).pipe(Effect.ensuring(Effect.sync(() => registration.mockRestore())));

expect(interrupted).toBe(true);
const next = yield* Effect.promise(() =>
Effect.runPromise(
executor.connections.checkHealth(ref).pipe(Effect.timeoutOption("1 second")),
),
);
expect(Option.isSome(next)).toBe(true);
expect(Option.getOrThrow(next).status).toBe("healthy");
expect(counters.resolves).toBeGreaterThan(0);
}),
);

it.live("a detached probe remains interruptible and releases its gate after timeout", () =>
Effect.gen(function* () {
let firstProbe = true;
const { executor, counters } = yield* makeHealthHarness({
probe: Effect.suspend(() => {
if (firstProbe) {
firstProbe = false;
return Effect.never.pipe(Effect.timeout("10 millis"));
}
return Effect.succeed({ status: "healthy" as const, checkedAt: Date.now() });
}),
});
const ref = { owner: "org", integration: INTEG, name: ConnectionName.make("main") } as const;
const first = yield* executor.connections.checkHealth(ref).pipe(Effect.exit);
expect(Exit.isFailure(first)).toBe(true);
const next = yield* executor.connections.checkHealth(ref);
expect(next.status).toBe("healthy");
expect(counters.probes).toBe(2);
}),
);

it.effect("a failed probe clears its gate entry for the next check", () =>
Effect.gen(function* () {
let firstProbe = true;
Expand Down
152 changes: 79 additions & 73 deletions packages/core/sdk/src/executor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6053,79 +6053,85 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
// caller's interruption cannot fail the peers awaiting the same
// entry; each caller awaits the shared deferred and stamps its own
// span with the outcome.
const outcome = yield* Effect.suspend(() => {
const key = healthProbeGateKey(tenant, connectionRow);
const existing = healthProbeInFlight.get(key);
if (existing) return Deferred.await(existing);
const deferred = Deferred.makeUnsafe<HealthProbeOutcome, StorageFailure>();
// Nothing suspends between the lookup above and this registration,
// so check-and-set is atomic against peer fibers.
healthProbeInFlight.set(key, deferred);
const freshVerdict: Effect.Effect<HealthProbeOutcome, StorageFailure> =
spec === undefined && connectionRow.oauth_client != null
? // No probe operation is declared, so "healthy" here means only
// "the credential resolved (refreshing if due)" — a refresh
// failure is the one real signal this path can produce, and it
// must not hide inside a green span.
oauthCredentialHealthWithoutProbe(connectionRow).pipe(
Effect.tap((result) => persistProbeHealthResult(ref, result)),
Effect.map((result) => ({
source: "credential_only" as const,
result,
})),
)
: foldCredentialResolutionIntoVerdict(
Effect.gen(function* () {
const values = yield* resolveConnectionValues(connectionRow);
const record = rowToIntegrationRecord(
integrationRow,
yield* describeAuthMethodsForRow(integrationRow),
);
const grantedScopes = grantedScopesFromRow(connectionRow);
const credential: ToolInvocationCredential = {
owner: connectionRow.owner as Owner,
integration: ref.integration,
connection: ConnectionName.make(connectionRow.name),
template: AuthTemplateSlug.make(connectionRow.template),
value: values[PRIMARY_INPUT_VARIABLE] ?? null,
values,
config: record.config,
...(grantedScopes ? { grantedScopes } : {}),
};
// Core resolves the declared spec (its own column) and
// hands it to the plugin; plugins no longer read it out of
// their config.
return yield* foldPluginFailure(
check({
ctx: runtime.ctx,
integration: record,
credential,
spec,
}),
`Health check for connection "${ref.name}" failed.`,
);
}),
).pipe(
// Persist the verdict on the connection row so the accounts
// list shows alive/expired at a glance, AND so the freshness
// gate above has something to serve. A probe that could not
// resolve its credential persists too: it is the connection
// most likely to be re-probed by every surface on every
// mount, so leaving it unwritten is what turns one broken
// connection into unbounded upstream and error traffic.
Effect.tap((result) => persistProbeHealthResult(ref, result)),
Effect.map((result) => ({
source: "probe" as const,
result,
})),
);
const run = freshVerdict.pipe(
Effect.exit,
Effect.flatMap((exit) => Deferred.done(deferred, exit)),
Effect.ensuring(Effect.sync(() => void healthProbeInFlight.delete(key))),
);
return Effect.forkDetach(run).pipe(Effect.andThen(Deferred.await(deferred)));
});
// Registration and startup must complete together. Cancellation between them
// would leave a Deferred that every later check waits on forever.
const outcome = yield* Effect.uninterruptibleMask((restore) =>
Effect.suspend(() => {
const key = healthProbeGateKey(tenant, connectionRow);
const existing = healthProbeInFlight.get(key);
if (existing) return restore(Deferred.await(existing));
const deferred = Deferred.makeUnsafe<HealthProbeOutcome, StorageFailure>();
// Nothing suspends between the lookup above and this registration,
// so check-and-set is atomic against peer fibers.
healthProbeInFlight.set(key, deferred);
const freshVerdict: Effect.Effect<HealthProbeOutcome, StorageFailure> =
spec === undefined && connectionRow.oauth_client != null
? // No probe operation is declared, so "healthy" here means only
// "the credential resolved (refreshing if due)" — a refresh
// failure is the one real signal this path can produce, and it
// must not hide inside a green span.
oauthCredentialHealthWithoutProbe(connectionRow).pipe(
Effect.tap((result) => persistProbeHealthResult(ref, result)),
Effect.map((result) => ({
source: "credential_only" as const,
result,
})),
)
: foldCredentialResolutionIntoVerdict(
Effect.gen(function* () {
const values = yield* resolveConnectionValues(connectionRow);
const record = rowToIntegrationRecord(
integrationRow,
yield* describeAuthMethodsForRow(integrationRow),
);
const grantedScopes = grantedScopesFromRow(connectionRow);
const credential: ToolInvocationCredential = {
owner: connectionRow.owner as Owner,
integration: ref.integration,
connection: ConnectionName.make(connectionRow.name),
template: AuthTemplateSlug.make(connectionRow.template),
value: values[PRIMARY_INPUT_VARIABLE] ?? null,
values,
config: record.config,
...(grantedScopes ? { grantedScopes } : {}),
};
// Core resolves the declared spec (its own column) and
// hands it to the plugin; plugins no longer read it out of
// their config.
return yield* foldPluginFailure(
check({
ctx: runtime.ctx,
integration: record,
credential,
spec,
}),
`Health check for connection "${ref.name}" failed.`,
);
}),
).pipe(
// Persist the verdict on the connection row so the accounts
// list shows alive/expired at a glance, AND so the freshness
// gate above has something to serve. A probe that could not
// resolve its credential persists too: it is the connection
// most likely to be re-probed by every surface on every
// mount, so leaving it unwritten is what turns one broken
// connection into unbounded upstream and error traffic.
Effect.tap((result) => persistProbeHealthResult(ref, result)),
Effect.map((result) => ({
source: "probe" as const,
result,
})),
);
const run = freshVerdict.pipe(
Effect.exit,
Effect.flatMap((exit) => Deferred.done(deferred, exit)),
Effect.ensuring(Effect.sync(() => void healthProbeInFlight.delete(key))),
);
return Effect.forkDetach(run, { startImmediately: true }).pipe(
Comment thread
aryasaatvik marked this conversation as resolved.
Effect.andThen(restore(Deferred.await(deferred))),
);
}),
);
yield* annotateHealthVerdict(outcome.source, outcome.result, previous);
return outcome.result;
}).pipe(
Expand Down
Loading