mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-10-26 13:40:26 +00:00
feat(gitops): stage-aware retry and resume re-evaluation (#1917)
* feat(gitops): retry resumes at the failed stage An operator retry previously always re-issued a fetch, even when the source had already accepted a generation whose dispatch never landed. retry() now reads the live pointer evidence (accepted generation, active direct target, applied pointer not yet caught up) and routes the request to the stage the evidence actually stopped at: a persisted source-stage failure or nothing pending goes to the fetch arm, an undelivered accepted generation goes straight through the shared dispatch boundary, and suspended/in-flight/recovering rows defer to the projection instead of starting work beside it. Blocked dispatch refusals come back as blocked outcomes with per-refusal advice; a refused dispatch reserves nothing. * feat(gitops): resume triggers immediate source re-evaluation Clearing a suspension previously left the source idle until its next poll or retry cursor. The resume route now fires SourceController.evaluateNow() fire-and-forget after the service returns: the woken source is re-evaluated immediately through the same eligibility guards the due-scan applies (manual, suspended, missing, or in-flight rows are skipped, and a tick that already owns the application is never queued behind itself). The response still reports that suspension was cleared, not the outcome of a reconcile that has not finished yet. evaluateNow absorbs evaluation failures by design and logs them; the route's catch covers only a store error before any evaluation could start. * test(gitops): cover suspension semantics across resume and dispatch Pins the three coexistence rules the stage-aware retry and resume wake rely on: a rollout pause survives the whole suspend/resume cycle on both application and target rows (suspension is a decision about source work, not a rollout pause); an interrupted apply's terminal still lands after suspension (suspension gates new work, never the accounting of work that already committed); and in-flight deploy transitions, tracked on the target row alone, stay admissible while the source is suspended. * test(e2e): suspend and resume a Git source over the API Drives the full journey against a local git server: a configured automatic source pulls, suspension stops reconciliation (a pull while suspended is refused outright), and resume is re-evaluated within seconds rather than at the next scheduled cursor. Polling is parked at the slowest floor the settings allow and restored afterwards.
This commit is contained in:
@@ -883,6 +883,8 @@ describe('POST /api/stacks/:stackName/git-source/resume', () => {
|
||||
it('returns the normalized result', async () => {
|
||||
const resumeSpy = vi.spyOn(GitSourceService.getInstance(), 'resume')
|
||||
.mockResolvedValue({ outcome: 'no_source_change', reason: 'ok', nextAction: 'none' });
|
||||
const { SourceController } = await import('../services/gitops/SourceController');
|
||||
const evaluateSpy = vi.spyOn(SourceController.getInstance(), 'evaluateNow').mockResolvedValue();
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/existing-stack/git-source/resume')
|
||||
.set('Authorization', `Bearer ${adminToken()}`)
|
||||
@@ -890,7 +892,38 @@ describe('POST /api/stacks/:stackName/git-source/resume', () => {
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.outcome).toBe('no_source_change');
|
||||
expect(resumeSpy).toHaveBeenCalledWith('existing-stack', expect.objectContaining({ actor: expect.any(String) }));
|
||||
// The route wakes the source immediately: the controller's resume
|
||||
// evaluation is fired for the same stack the service just resumed.
|
||||
expect(evaluateSpy).toHaveBeenCalledWith('existing-stack');
|
||||
resumeSpy.mockRestore();
|
||||
evaluateSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('still returns the resume outcome when the immediate evaluation cannot start', async () => {
|
||||
const resumeSpy = vi.spyOn(GitSourceService.getInstance(), 'resume')
|
||||
.mockResolvedValue({ outcome: 'no_source_change', reason: 'ok', nextAction: 'none' });
|
||||
const { SourceController } = await import('../services/gitops/SourceController');
|
||||
const evaluateSpy = vi.spyOn(SourceController.getInstance(), 'evaluateNow')
|
||||
.mockRejectedValue(new Error('database is locked'));
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
try {
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/existing-stack/git-source/resume')
|
||||
.set('Authorization', `Bearer ${adminToken()}`)
|
||||
.send({});
|
||||
// The response reports that suspension was cleared, not the
|
||||
// outcome of an evaluation that never ran: it stays 200 with
|
||||
// resume's derived result, and the failure is logged, not
|
||||
// swallowed.
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.outcome).toBe('no_source_change');
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(errorSpy).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
resumeSpy.mockRestore();
|
||||
evaluateSpy.mockRestore();
|
||||
errorSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('denies without the stack:edit permission', async () => {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -735,6 +735,107 @@ describe('gitops transitions', () => {
|
||||
expect(application.suspended_at).not.toBeNull();
|
||||
});
|
||||
|
||||
it('keeps a rollout pause across source suspension and resume', () => {
|
||||
const store = GitOpsStore.getInstance();
|
||||
const tx = GitOpsTransitions.getInstance();
|
||||
tx.activateDirect({ application: app('app-pause-coex', 'pause-coex-web'), nodeId: 1, envelope: envelope('op-act-pause-coex') });
|
||||
tx.rolloutPaused('app-pause-coex', null, 'waiting for the maintenance window', envelope('op-rp-app'));
|
||||
tx.rolloutPaused('app-pause-coex', 1, 'fleet rollout paused', envelope('op-rp-target'));
|
||||
|
||||
tx.sourceSuspended('app-pause-coex', 'operator hold', envelope('op-susp-pause-coex'));
|
||||
// Suspension is a decision about future source work. It is not a rollout
|
||||
// pause, and it must neither overwrite one nor read like one.
|
||||
let application = store.getApplication('app-pause-coex')!;
|
||||
expect(application.source_suspended_reason).toBe('operator hold');
|
||||
expect(application.pause_reason).toBe('waiting for the maintenance window');
|
||||
expect(store.getTarget('app-pause-coex', 1)?.pause_reason).toBe('fleet rollout paused');
|
||||
|
||||
tx.sourceUnsuspended('app-pause-coex', envelope('op-unsusp-pause-coex'));
|
||||
application = store.getApplication('app-pause-coex')!;
|
||||
expect(application.suspended_at).toBeNull();
|
||||
expect(application.source_suspended_reason).toBeNull();
|
||||
// The rollout's own pause survives the whole suspend/resume cycle on
|
||||
// both rows: only rolloutUnpaused clears it.
|
||||
expect(application.pause_at).not.toBeNull();
|
||||
expect(application.pause_reason).toBe('waiting for the maintenance window');
|
||||
const target = store.getTarget('app-pause-coex', 1)!;
|
||||
expect(target.pause_at).not.toBeNull();
|
||||
expect(target.pause_reason).toBe('fleet rollout paused');
|
||||
});
|
||||
|
||||
it('records a promotion whose apply completed after suspension interrupted it', () => {
|
||||
const store = GitOpsStore.getInstance();
|
||||
const tx = GitOpsTransitions.getInstance();
|
||||
tx.activateDirect({ application: app('app-susp-promo', 'susp-promo-web'), nodeId: 1, envelope: envelope('op-act-susp-promo') });
|
||||
store.insertGeneration(gen('gen-susp-promo', 'app-susp-promo'));
|
||||
tx.fetchStarted('app-susp-promo', envelope('op-f-susp-promo'));
|
||||
tx.fetched('app-susp-promo', 'abc123', envelope('op-f-susp-promo'));
|
||||
tx.candidateReady('app-susp-promo', 'gen-susp-promo', false, envelope('op-c-susp-promo'));
|
||||
const applyOp = envelope('op-a-susp-promo');
|
||||
tx.applyStarted('app-susp-promo', 'gen-susp-promo', applyOp);
|
||||
|
||||
// Suspension interrupts the in-flight operation without discarding it:
|
||||
// the apply is marked interrupted, and the candidate stays staged.
|
||||
tx.sourceSuspended('app-susp-promo', 'operator hold', envelope('op-susp-susp-promo'));
|
||||
expect(store.getApplication('app-susp-promo')?.interruption_stage).toBe('apply_started');
|
||||
|
||||
// The apply's terminal event still lands after the interrupt. Suspension
|
||||
// gates new work (fetch, apply, accept); it never cancels the accounting
|
||||
// of work that already committed to disk.
|
||||
tx.applied({
|
||||
applicationId: 'app-susp-promo',
|
||||
generationId: 'gen-susp-promo',
|
||||
artifactSetId: 'art-susp-promo',
|
||||
sourceAcceptanceId: 'acc-susp-promo',
|
||||
authority: 'operator',
|
||||
envelope: applyOp,
|
||||
});
|
||||
const application = store.getApplication('app-susp-promo')!;
|
||||
expect(application.accepted_generation_id).toBe('gen-susp-promo');
|
||||
expect(application.interruption_stage).toBeNull();
|
||||
// Recording an in-flight promotion does not resume anything.
|
||||
expect(application.suspended_at).not.toBeNull();
|
||||
expect(store.getTarget('app-susp-promo', 1)?.applied_generation_id).toBe('gen-susp-promo');
|
||||
});
|
||||
|
||||
it('accepts a matching late deploy result for a suspended source', () => {
|
||||
const store = GitOpsStore.getInstance();
|
||||
const tx = GitOpsTransitions.getInstance();
|
||||
seedApplied('app-late-dep', 'late-dep-web', 'gen-late-dep', 'art-late-dep', 'acc-late-dep');
|
||||
const deployOp = envelope('op-dep-late');
|
||||
tx.deployStarted('app-late-dep', 1, 'gen-late-dep', deployOp);
|
||||
|
||||
// The deploy is in flight when the operator suspends. Suspension gates
|
||||
// new source work, and it interrupts an operation the application row
|
||||
// still shows as active; a deploy is tracked on the target row alone, so
|
||||
// an in-flight deploy is left running rather than marked interrupted.
|
||||
tx.sourceSuspended('app-late-dep', 'operator hold', envelope('op-susp-late-dep'));
|
||||
const suspended = store.getTarget('app-late-dep', 1)!;
|
||||
expect(suspended.active_operation_stage).toBe('deploy_started');
|
||||
expect(suspended.interruption_stage).toBeNull();
|
||||
|
||||
// Its terminal events are therefore admissible while the source is
|
||||
// suspended: requireMatchingDeploy matches the live operation and
|
||||
// records its outcome, so evidence of a deploy nobody can recall is
|
||||
// kept, not dropped.
|
||||
tx.deployBound('app-late-dep', 1, 'gen-late-dep', deployOp);
|
||||
let target = store.getTarget('app-late-dep', 1)!;
|
||||
expect(target.deployed_generation_id).toBe('gen-late-dep');
|
||||
expect(target.failure_stage).toBeNull();
|
||||
|
||||
// The failure arm behaves the same way: a later deploy for the same
|
||||
// applied generation records its failure while still suspended.
|
||||
tx.deployStarted('app-late-dep', 1, 'gen-late-dep', envelope('op-dep-late-2'));
|
||||
tx.deployFailed('app-late-dep', 1, 'post_mutation', envelope('op-dep-late-2'));
|
||||
target = store.getTarget('app-late-dep', 1)!;
|
||||
expect(target.failure_stage).toBe('deploy');
|
||||
expect(target.failure_class).toBe('post_mutation');
|
||||
expect(target.deployed_generation_id).toBe('gen-late-dep');
|
||||
// Neither terminal resumed anything: recording is accounting, not a
|
||||
// source operation.
|
||||
expect(store.getApplication('app-late-dep')?.suspended_at).not.toBeNull();
|
||||
});
|
||||
|
||||
it('targetApplied binds a Direct target only after the generation is accepted', () => {
|
||||
const store = GitOpsStore.getInstance();
|
||||
const tx = GitOpsTransitions.getInstance();
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
import { directApplicationFixture } from './helpers/gitopsFixtures';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { GitOpsStore } from '../services/gitops/store';
|
||||
import { GitSourceService } from '../services/GitSourceService';
|
||||
import { SourceController } from '../services/gitops/SourceController';
|
||||
@@ -213,4 +214,199 @@ describe('SourceController', () => {
|
||||
|
||||
expect(reconcile).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
describe('evaluateNow (resume re-evaluation)', () => {
|
||||
/** Insert a live automatic application row for a stack name. */
|
||||
function seedLive(stackName: string): GitOpsApplicationRow {
|
||||
const app = autoFixture(`app-${stackName}`, stackName);
|
||||
GitOpsStore.getInstance().insertApplication(app);
|
||||
return app;
|
||||
}
|
||||
|
||||
it('fetches a live automatic source immediately, tagged with the resume trigger', async () => {
|
||||
seedLive('now-web');
|
||||
const reconcile = spyOnReconcile().mockResolvedValue(okResult);
|
||||
|
||||
await controller.evaluateNow('now-web');
|
||||
|
||||
expect(reconcile).toHaveBeenCalledTimes(1);
|
||||
expect(reconcile).toHaveBeenCalledWith(expect.objectContaining({
|
||||
intent: 'fetch',
|
||||
applicationId: 'app-now-web',
|
||||
stackName: 'now-web',
|
||||
trigger: 'resume',
|
||||
}));
|
||||
// No timer was running: nothing here waited for a tick.
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('skips a manual-policy source without reconciling it', async () => {
|
||||
const app: GitOpsApplicationRow = { ...autoFixture('app-now-manual', 'now-manual-web'), source_policy: 'manual' };
|
||||
GitOpsStore.getInstance().insertApplication(app);
|
||||
const reconcile = spyOnReconcile();
|
||||
|
||||
await controller.evaluateNow('now-manual-web');
|
||||
|
||||
expect(reconcile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips a suspended source, so a resume race never wakes it early', async () => {
|
||||
const app = { ...autoFixture('app-now-susp', 'now-susp-web'), suspended_at: Date.now() };
|
||||
GitOpsStore.getInstance().insertApplication(app);
|
||||
const reconcile = spyOnReconcile();
|
||||
|
||||
await controller.evaluateNow('now-susp-web');
|
||||
|
||||
expect(reconcile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('is a silent no-op for a stack with no live application', async () => {
|
||||
const reconcile = spyOnReconcile();
|
||||
|
||||
await controller.evaluateNow('now-missing');
|
||||
|
||||
expect(reconcile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('parks a resume wake while a tick owns the application and runs it once when the tick settles', async () => {
|
||||
const app = seedLive('now-busy');
|
||||
// The scan's due list is mocked to the same row, so the tick's
|
||||
// evaluation and the resume path address one identical id.
|
||||
mockDue([app]);
|
||||
let settleFirstCall!: (result: ReconcileResult) => void;
|
||||
const firstCall = new Promise<ReconcileResult>((resolve) => { settleFirstCall = resolve; });
|
||||
const reconcile = spyOnReconcile().mockReturnValueOnce(firstCall).mockResolvedValue(okResult);
|
||||
|
||||
controller.start();
|
||||
await advanceOneTick();
|
||||
expect(reconcile).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Two resumes arrive while the tick's evaluation is still
|
||||
// pending: neither starts work beside it, and together they
|
||||
// coalesce into the single wake they asked for.
|
||||
await controller.evaluateNow('now-busy');
|
||||
await controller.evaluateNow('now-busy');
|
||||
expect(reconcile).toHaveBeenCalledTimes(1);
|
||||
|
||||
settleFirstCall(okResult);
|
||||
// Flush the tick's continuation chain (evaluate's post-reconcile
|
||||
// steps plus the scan's inFlight release, which drains the
|
||||
// parked wake into a fresh resume evaluation).
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
// The wake was honored without any second manual call: the
|
||||
// park-then-drain is what the operator's resume bought. The
|
||||
// two parked resumes produced exactly one fresh evaluation,
|
||||
// tagged as a resume, and the drain's own release leaves no
|
||||
// further wake parked (a third call would have landed).
|
||||
expect(reconcile).toHaveBeenCalledTimes(2);
|
||||
expect(reconcile.mock.calls[1]![0]).toMatchObject({ trigger: 'resume' });
|
||||
// "At most one" pinned: the drain's settle releases the slot
|
||||
// with nothing left parked, so no third evaluation ever fires.
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(reconcile).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('drops a parked wake whose source went stale, without evaluating it', async () => {
|
||||
const app = seedLive('now-stale');
|
||||
mockDue([app]);
|
||||
let settleFirstCall!: (result: ReconcileResult) => void;
|
||||
const firstCall = new Promise<ReconcileResult>((resolve) => { settleFirstCall = resolve; });
|
||||
const reconcile = spyOnReconcile().mockReturnValueOnce(firstCall).mockResolvedValue(okResult);
|
||||
|
||||
controller.start();
|
||||
await advanceOneTick();
|
||||
expect(reconcile).toHaveBeenCalledTimes(1);
|
||||
|
||||
await controller.evaluateNow('now-stale');
|
||||
expect(reconcile).toHaveBeenCalledTimes(1);
|
||||
// The drain re-reads eligibility, so the stale wake is dropped
|
||||
// exactly as a fresh call would drop it: no evaluation of a
|
||||
// suspended source.
|
||||
DatabaseService.getInstance().getDb()
|
||||
.prepare('UPDATE gitops_applications SET suspended_at = ? WHERE id = ?')
|
||||
.run(Date.now(), app.id);
|
||||
settleFirstCall(okResult);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(reconcile).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('drains the parked wake even when the owning evaluation crashes past the reconcile', async () => {
|
||||
const app = seedLive('now-owner-crash');
|
||||
mockDue([app]);
|
||||
let settleFirstCall!: (result: ReconcileResult) => void;
|
||||
const firstCall = new Promise<ReconcileResult>((resolve) => { settleFirstCall = resolve; });
|
||||
const reconcile = spyOnReconcile().mockReturnValueOnce(firstCall).mockResolvedValue(okResult);
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
// The crash lands in evaluate()'s post-reconcile tail (the row
|
||||
// re-read), not in reconcile itself: the owner's chain rejects,
|
||||
// the scan's load-bearing catch logs it, and the release's
|
||||
// finally must still drain the wake. The mock is once-only, so
|
||||
// the drained evaluation's own re-read falls through to the
|
||||
// real store.
|
||||
vi.spyOn(GitOpsStore.getInstance(), 'getApplication')
|
||||
.mockImplementationOnce(() => { throw new Error('simulated crash after reconcile'); });
|
||||
|
||||
controller.start();
|
||||
await advanceOneTick();
|
||||
expect(reconcile).toHaveBeenCalledTimes(1);
|
||||
|
||||
await controller.evaluateNow('now-owner-crash');
|
||||
settleFirstCall(okResult);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(reconcile).toHaveBeenCalledTimes(2);
|
||||
expect(reconcile.mock.calls[1]![0]).toMatchObject({ trigger: 'resume' });
|
||||
const logged = errorSpy.mock.calls.map((args) => args.map(String).join('\n')).join('\n');
|
||||
expect(logged).toContain('evaluation crashed');
|
||||
errorSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('logs a parked-wake drain whose prologue throws instead of rejecting unhandled', async () => {
|
||||
const app = seedLive('now-drain-throws');
|
||||
mockDue([app]);
|
||||
let settleFirstCall!: (result: ReconcileResult) => void;
|
||||
const firstCall = new Promise<ReconcileResult>((resolve) => { settleFirstCall = resolve; });
|
||||
const reconcile = spyOnReconcile().mockReturnValueOnce(firstCall).mockResolvedValue(okResult);
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
|
||||
controller.start();
|
||||
await advanceOneTick();
|
||||
await controller.evaluateNow('now-drain-throws');
|
||||
expect(reconcile).toHaveBeenCalledTimes(1);
|
||||
|
||||
// evaluateNow's eligibility read runs before its own catch, so
|
||||
// a store failure at that point rejects the call. The drain is
|
||||
// a detached fire-and-forget (nothing above it in the release
|
||||
// chain can catch), so it must carry its own guard: the
|
||||
// rejection becomes a logged line, never an unhandled rejection
|
||||
// (which would take the process down).
|
||||
vi.spyOn(GitOpsStore.getInstance(), 'getLiveDirectApplication')
|
||||
.mockImplementationOnce(() => { throw new Error('database is locked'); });
|
||||
settleFirstCall(okResult);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
// No resume evaluation ran (the prologue died before it), but
|
||||
// the failure was logged rather than silently escaping.
|
||||
expect(reconcile).toHaveBeenCalledTimes(1);
|
||||
const logged = errorSpy.mock.calls.map((args) => args.map(String).join('\n')).join('\n');
|
||||
expect(logged).toContain('parked-wake drain could not start');
|
||||
errorSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('survives a reconcile rejection without throwing at the route', async () => {
|
||||
seedLive('now-throws');
|
||||
spyOnReconcile().mockRejectedValue(new Error('boom'));
|
||||
// evaluate() logs the reconcile failure itself and returns;
|
||||
// evaluateNow must not turn it into a rejection for the caller.
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
|
||||
await expect(controller.evaluateNow('now-throws')).resolves.toBeUndefined();
|
||||
|
||||
expect(errorSpy).toHaveBeenCalledTimes(1);
|
||||
errorSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -766,6 +766,32 @@ stackGitSourceRouter.post('/:stackName/git-source/resume', async (req: Request,
|
||||
const result = await GitSourceService.getInstance().resume(stackName, {
|
||||
actor: req.user?.username ?? 'unknown',
|
||||
});
|
||||
// A resumed source is re-evaluated immediately rather than idling
|
||||
// until its next poll or retry cursor: the controller's resume trigger
|
||||
// re-resolves source state and, for an automatic source, target
|
||||
// binding through the shared acceptance/dispatch arm. Fire-and-forget
|
||||
// by design: this response reports that suspension was cleared, not
|
||||
// the outcome of a reconcile that has not finished yet, and the
|
||||
// durable row plus the next tick are the authority on the latter.
|
||||
// The wake has its own guard: a failure starting it must not
|
||||
// retroactively fail a resume that already committed, so it is
|
||||
// logged, never sent to sendGitSourceError. evaluateNow absorbs its
|
||||
// evaluation failures itself; the rejection arm catches anything
|
||||
// before its own try exists (evaluateNow is async, so even its
|
||||
// synchronous prologue, a store error before any evaluation could
|
||||
// run, surfaces as a rejection) and any future escape, and the
|
||||
// try/catch covers a getInstance()-level failure only.
|
||||
const logWakeFailure = (error: unknown): void => {
|
||||
console.error(
|
||||
`[GitSources] Resume re-evaluation could not start for ${sanitizeForLog(stackName)}:`,
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
};
|
||||
try {
|
||||
SourceController.getInstance().evaluateNow(stackName).catch(logWakeFailure);
|
||||
} catch (error) {
|
||||
logWakeFailure(error);
|
||||
}
|
||||
res.json(result);
|
||||
} catch (error) {
|
||||
sendGitSourceError(res, error);
|
||||
|
||||
@@ -244,7 +244,11 @@ export interface HealthGateRunRow {
|
||||
failure_source: 'primary' | 'collateral' | null;
|
||||
/**
|
||||
* Reserved for the GitOps deploy path: the generation live when this run
|
||||
* started. No writer populates it yet, so it is currently always null.
|
||||
* started. Populated from beginStack's binding argument and the deploy
|
||||
* callers pass it when their mutation was GitOps-tracked, plus recovery
|
||||
* reservations (reserveRecoveryRun); null otherwise and for legacy rows.
|
||||
* The stage-aware retry reads it to decide whether a failed stack
|
||||
* verdict belongs to the currently applied generation.
|
||||
*/
|
||||
deployed_generation_id?: string | null;
|
||||
}
|
||||
@@ -4321,6 +4325,13 @@ stmt.run('gitops_schema_version', '1');
|
||||
).get(nodeId, stackName) as HealthGateRunRow | undefined;
|
||||
}
|
||||
|
||||
/** The most recent stack-scoped run for the stack (service gates excluded). */
|
||||
public getLatestStackHealthGateRun(nodeId: number, stackName: string): HealthGateRunRow | undefined {
|
||||
return this.db.prepare(
|
||||
"SELECT * FROM health_gate_runs WHERE node_id = ? AND stack_name = ? AND target_scope = 'stack' ORDER BY started_at DESC, id DESC LIMIT 1"
|
||||
).get(nodeId, stackName) as HealthGateRunRow | undefined;
|
||||
}
|
||||
|
||||
/** Finalize runs left observing by a previous process (startup sweep). */
|
||||
/**
|
||||
* Runs a previous process left observing.
|
||||
|
||||
@@ -39,7 +39,7 @@ import { projectApplication } from './gitops/derive';
|
||||
import { outcomeFromSourceFacet, parseReconcileResultPayload, type ReconcileOutcome, type ReconcileResult } from './gitops/outcomes';
|
||||
import { coalesceKey, deliveryKey, type ReconcileRequest, type ReconcileTrigger } from './gitops/triggers';
|
||||
import { classifyFailure, effectivePollIntervalSecs } from './gitops/backoff';
|
||||
import { BlueprintTargetAdapter, type AcceptedGeneration, type DispatchContext, type DispatchResult } from './gitops/handoff';
|
||||
import { BlueprintTargetAdapter, buildAcceptedGeneration, type AcceptedGeneration, type DispatchContext, type DispatchResult } from './gitops/handoff';
|
||||
import {
|
||||
GitOpsTransitions,
|
||||
GitOpsTransitionError,
|
||||
@@ -3627,8 +3627,13 @@ export class GitSourceService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume acting on a source. Does not itself fetch; the next scheduled
|
||||
* poll, retry, or manual reconcile picks the source back up.
|
||||
* Resume acting on a source. Clears suspension and re-arms the steady-
|
||||
* state poll cursor; it does not itself fetch. The immediate wake is the
|
||||
* resume route's job: after this method returns, the route fires
|
||||
* SourceController.evaluateNow() so the source is re-evaluated right
|
||||
* away (re-resolving source state and, for an automatic source, target
|
||||
* binding through the shared acceptance/dispatch arm) instead of idling
|
||||
* until the next scheduled poll or retry.
|
||||
*
|
||||
* Unlike suspend(), a refused resume is tolerated rather than surfaced.
|
||||
* The result is read back from the row after the attempted write, so a
|
||||
@@ -3683,25 +3688,662 @@ export class GitSourceService {
|
||||
}
|
||||
|
||||
/**
|
||||
* An explicit, operator-initiated re-evaluation: resolves the live
|
||||
* application server-side rather than trusting a caller-supplied id, for
|
||||
* callers that hold only a stack name, and drives a fresh fetch-intent
|
||||
* reconcile through it.
|
||||
* The dispatch evidence stage-aware retry reads: which dispatch stage
|
||||
* an operator retry should resume, or null when no dispatch is
|
||||
* pending. Two arms share this one decision because the arms are
|
||||
* mutually exclusive by evidence, and the helper is the single place
|
||||
* that exclusivity is stated:
|
||||
* - `'promote'`: the cheap pre-filter over the same columns
|
||||
* `dispatchDirectLocked` revalidates (the generation row still readable,
|
||||
* a live application, an active Direct
|
||||
* target whose candidate still names the accepted generation, and an
|
||||
* applied pointer that has not caught up). It lags the dispatch's own
|
||||
* checks (commit sha, artifact-set evidence, recomputed change plan),
|
||||
* and the in-lock revalidation remains the authority. Generation files
|
||||
* were never promoted, so retry hands the generation to the shared
|
||||
* dispatch boundary.
|
||||
* - `'deploy'`: the promotion already committed (the applied pointer
|
||||
* names the accepted generation) and the durable evidence says the
|
||||
* stage after it failed: either the deploy failed (`failure_stage =
|
||||
* 'deploy'`, set by the deploy-failed/deploy-unbound transitions and
|
||||
* cleared by deploy-bound), or the health gate watching that exact
|
||||
* applied generation finalized as failed (the stack's most recent
|
||||
* stack-scoped `health_gate_runs` row, bound to the generation id,
|
||||
* with status `'failed'`). Nothing may be fetched, accepted,
|
||||
* promoted, or re-bound; only the unfinished deploy stage is retried,
|
||||
* and every resume re-opens the health gate on the deploy it runs. A
|
||||
* deploy or health failure with no accepted-generation evidence is
|
||||
* not deploy-resume material: the fetch arm re-decides from the
|
||||
* source instead.
|
||||
* Retry routes an operator's request to the stage the evidence actually
|
||||
* stopped at, and each arm's in-lock revalidation remains the authority
|
||||
* when the state moves in between.
|
||||
*/
|
||||
private acceptedGenerationDispatchStage(app: GitOpsApplicationRow): { stage: 'promote' | 'deploy'; generationId: string } | null {
|
||||
const generationId = app.accepted_generation_id;
|
||||
if (!generationId) return null;
|
||||
if (!GitOpsStore.getInstance().getGeneration(generationId)) {
|
||||
// A live application naming a generation that no longer exists is
|
||||
// a pointer-integrity violation. The fetch arm is a defensible
|
||||
// recovery, but it must announce itself: an operator who asked to
|
||||
// retry a stuck dispatch deserves to see why a fetch ran instead.
|
||||
console.warn(
|
||||
`[GitSource] ${sanitizeForLog(app.stack_name ?? app.id)} names accepted generation ${sanitizeForLog(generationId)} but the row is missing; retry falls back to a fresh fetch.`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const target = GitOpsStore.getInstance().getTarget(app.id, NodeRegistry.getInstance().getDefaultNodeId());
|
||||
if (!target || target.target_status !== 'active') return null;
|
||||
// The deploy arm's two evidence kinds both route to re-running the
|
||||
// deploy stage only: a failed deploy (`failure_stage = 'deploy'`,
|
||||
// set by the deploy-failed/deploy-unbound transitions and cleared
|
||||
// by deploy-bound) left the generation undelivered, and a failed
|
||||
// health gate (below) delivered it but the verdict says the run
|
||||
// is sick. Neither may be fetched, accepted, promoted, or
|
||||
// re-bound. healthFinalized() itself records health outcomes as
|
||||
// promotions of the healthy/LKG pointers and never writes the
|
||||
// target's failure columns, so the health evidence lives in the
|
||||
// gate's own durable run rows, not on the target.
|
||||
if (target.candidate_generation_id === generationId && target.applied_generation_id !== generationId) {
|
||||
return { stage: 'promote', generationId };
|
||||
}
|
||||
if (target.applied_generation_id !== generationId) return null;
|
||||
if (target.failure_stage === 'deploy') {
|
||||
return { stage: 'deploy', generationId };
|
||||
}
|
||||
// A failed health verdict only owns the arm when the deploy it
|
||||
// watched actually bound (deploy-bound clears the failure columns,
|
||||
// so the honest state is NULL): any other target failure belongs
|
||||
// to its own path, not to a deploy resume.
|
||||
if (target.failure_stage === null
|
||||
&& app.stack_name
|
||||
&& this.appliedGenerationHealthFailed(NodeRegistry.getInstance().getDefaultNodeId(), app.stack_name, generationId)) {
|
||||
return { stage: 'deploy', generationId };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the stack's most recent stack-scoped health gate run is a
|
||||
* `failed` verdict bound to exactly the given generation. Service-scoped
|
||||
* gates are excluded: they watch one service, not the applied
|
||||
* generation's stack-wide health. An `observing` or `unknown` row is
|
||||
* not a health-failure claim, and a row bound to a different (or no)
|
||||
* generation cannot vouch for this one, so neither selects an arm;
|
||||
* those fall to the fetch arm, which re-decides from the source.
|
||||
*/
|
||||
private appliedGenerationHealthFailed(nodeId: number, stackName: string, generationId: string): boolean {
|
||||
const run = DatabaseService.getInstance().getLatestStackHealthGateRun(nodeId, stackName);
|
||||
return !!run
|
||||
&& run.status === 'failed'
|
||||
&& run.deployed_generation_id === generationId;
|
||||
}
|
||||
|
||||
/**
|
||||
* The dispatch context for the local Direct target: the boundary's
|
||||
* shared construction used by every dispatch of a locally-accepted
|
||||
* generation (automatic controller dispatch, explicit retry). The
|
||||
* nodeId is resolved per call because it is process configuration, not
|
||||
* durable evidence; the dispatch itself re-reads the live target under
|
||||
* the stack lock and treats this context as routing only.
|
||||
*/
|
||||
public static directDispatchContext(): DispatchContext {
|
||||
return { targetMode: 'direct', nodeId: NodeRegistry.getInstance().getDefaultNodeId(), bindingRevision: null };
|
||||
}
|
||||
|
||||
/**
|
||||
* An explicit, operator-initiated re-evaluation, resuming at the stage
|
||||
* the durable evidence says failed. A fetch or validation failure
|
||||
* recorded on the application, or no dispatch evidence at all, drives
|
||||
* a fresh fetch-intent reconcile. When the accepted
|
||||
* generation is still awaiting delivery (candidate pointer set, applied
|
||||
* pointer behind), retry calls dispatchAcceptedGeneration() against
|
||||
* exactly that generation: the source is never refetched, and
|
||||
* acceptance is never re-run, because both already succeeded and the
|
||||
* failure lives past the acceptance boundary. When the generation is
|
||||
* already applied but its deploy failed or its health gate's latest
|
||||
* stack-scoped verdict is a failure for exactly that generation, retry
|
||||
* resumes at the deploy alone (retryDeploy): fetching, accepting,
|
||||
* promoting, and re-binding are all refused by the evidence (the
|
||||
* applied pointer names this generation; re-binding it would throw),
|
||||
* and the resume re-opens the health gate on the deploy it runs. The
|
||||
* dispatch's and the deploy-resume's own revalidation under the stack
|
||||
* lock stays the authority on whether it may proceed; a refusal comes
|
||||
* back as a blocked outcome naming why. A refusal that reserved an
|
||||
* attempt settles it carrying the same advice; a refusal before any
|
||||
* reservation (no stack bound, no source configured, the stack lock
|
||||
* held, or the in-lock source recheck) reserves nothing, so the
|
||||
* response itself is the only record, which is deliberate: nothing
|
||||
* started, so there is nothing to reconcile later.
|
||||
*
|
||||
* The 'retry' trigger is recorded on the durable attempt for audit and
|
||||
* recovery. It does not yet change fetch behavior; a later permanent-
|
||||
* failure gate can use it to authorize an explicit operator retry.
|
||||
* recovery on all three arms.
|
||||
*/
|
||||
public async retry(stackName: string, opts: { actor: string }): Promise<ReconcileResult> {
|
||||
const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName);
|
||||
if (!app) return GitSourceService.noApplicationResult();
|
||||
return this.reconcile({
|
||||
intent: 'fetch',
|
||||
applicationId: app.id,
|
||||
// A source that is suspended, has an operation in flight, has an
|
||||
// unfinished recovery, or has an unrecovered interruption marker
|
||||
// reports its real state instead of starting work beside it: the
|
||||
// suspended projection is the resume advice, the in-flight
|
||||
// projection says no settled result exists yet, and the recovery
|
||||
// projection is what the recovery path owns. An unrecovered crash
|
||||
// can leave an active_operation_stage set with no live work, and
|
||||
// retrying over it would race the pass that holds that evidence.
|
||||
// Unfinished recovery is inverted, not enumerated: every phase
|
||||
// except the settled 'complete' receipt (including 'capturing' and
|
||||
// any phase added later) defers, so a new intermediate phase can
|
||||
// never silently start retry work beside it. 'complete' is a
|
||||
// settled receipt written by recoverySucceeded or
|
||||
// rollbackCompleted that no code path clears; deferring on it
|
||||
// would strand every future retry on a source that just
|
||||
// recovered.
|
||||
const unfinishedRecovery = app.recovery_phase !== null && app.recovery_phase !== 'complete';
|
||||
if (app.suspended_at || app.active_operation_stage || unfinishedRecovery || app.interruption_stage) {
|
||||
return this.deriveReconcileResult(stackName);
|
||||
}
|
||||
const dispatch = this.acceptedGenerationDispatchStage(app);
|
||||
const sourceStageFailure = app.failure_stage === 'fetch' || app.failure_stage === 'validation';
|
||||
// The fetch arm owns the precedence: a newer source-stage failure
|
||||
// outranks stale dispatch evidence on either arm.
|
||||
if (!dispatch || sourceStageFailure) {
|
||||
return this.reconcile({
|
||||
intent: 'fetch',
|
||||
applicationId: app.id,
|
||||
stackName,
|
||||
trigger: 'retry',
|
||||
actor: opts.actor,
|
||||
});
|
||||
}
|
||||
return dispatch.stage === 'promote'
|
||||
? this.retryDispatch(stackName, app.id, dispatch.generationId, opts.actor)
|
||||
: this.retryDeploy(stackName, app.id, dispatch.generationId, opts.actor);
|
||||
}
|
||||
|
||||
/**
|
||||
* The promote arm of retry(): rebuild the accepted-generation contract
|
||||
* from the persisted generation row and hand it to the shared dispatch
|
||||
* boundary. Only reached when acceptedGenerationDispatchStage proved
|
||||
* the pointers, so a missing or corrupt row here means the evidence
|
||||
* moved or decayed between the check and the read; that is reported,
|
||||
* never papered over with a refetch.
|
||||
*/
|
||||
private async retryDispatch(
|
||||
stackName: string,
|
||||
applicationId: string,
|
||||
generationId: string,
|
||||
actor: string,
|
||||
): Promise<ReconcileResult> {
|
||||
let dispatch: DispatchResult;
|
||||
try {
|
||||
const generationRow = GitOpsStore.getInstance().getGeneration(generationId);
|
||||
if (!generationRow) {
|
||||
return {
|
||||
outcome: 'blocked',
|
||||
reason: 'The accepted generation could not be read; its dispatch evidence is unavailable.',
|
||||
nextAction: 'view_target_results',
|
||||
};
|
||||
}
|
||||
dispatch = await this.dispatchAcceptedGeneration(
|
||||
buildAcceptedGeneration(generationRow),
|
||||
GitSourceService.directDispatchContext(),
|
||||
{ trigger: 'retry', actor },
|
||||
);
|
||||
} catch (e) {
|
||||
// Nothing past this point throws once a reservation exists, so
|
||||
// a throw here means the contract could not be built or the
|
||||
// entry reads failed; the generation stands accepted, and the
|
||||
// operator can retry once the cause is fixed. errorForLog keeps
|
||||
// the stack for debugging while scrubbing credential-shaped
|
||||
// text, which a transport-level failure can carry.
|
||||
console.error(
|
||||
`[GitSource] Explicit retry could not dispatch the accepted generation for ${sanitizeForLog(stackName)}:`,
|
||||
errorForLog(e),
|
||||
);
|
||||
return {
|
||||
outcome: 'blocked',
|
||||
reason: 'The accepted generation could not be dispatched; its evidence was unreadable.',
|
||||
nextAction: 'view_target_results',
|
||||
};
|
||||
}
|
||||
if (dispatch.status === 'dispatched') {
|
||||
// The projection re-read after the dispatch is the truthful
|
||||
// post-state answer; every dispatch path that reserved an
|
||||
// attempt has settled it before returning.
|
||||
return this.deriveReconcileResult(stackName);
|
||||
}
|
||||
return {
|
||||
outcome: 'blocked',
|
||||
reason: dispatch.reason,
|
||||
nextAction: GitSourceService.dispatchRefusalNextAction(dispatch.reason),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The deploy arm of retry(): resume only the unfinished deploy of a
|
||||
* generation the target has already applied. Entry shape mirrors the
|
||||
* promote arm's shared boundary: refuse cheaply before taking the
|
||||
* stack lock, reserve the durable attempt inside the lock, and settle
|
||||
* every reserved path before returning. A lock-held refusal reserves
|
||||
* nothing, matching the dispatch boundary's rule that a retry refused
|
||||
* before any work started leaves only its response as the record.
|
||||
*/
|
||||
private async retryDeploy(
|
||||
stackName: string,
|
||||
applicationId: string,
|
||||
generationId: string,
|
||||
actor: string,
|
||||
): Promise<ReconcileResult> {
|
||||
const app = GitOpsStore.getInstance().getApplication(applicationId);
|
||||
if (!app?.stack_name) {
|
||||
return {
|
||||
outcome: 'blocked',
|
||||
reason: 'No Direct stack is bound to this application.',
|
||||
nextAction: 'resolve_conflict',
|
||||
};
|
||||
}
|
||||
if (!DatabaseService.getInstance().getGitSource(stackName)) {
|
||||
return {
|
||||
outcome: 'blocked',
|
||||
reason: `No Git source is configured for ${stackName}.`,
|
||||
nextAction: 'configure_credentials',
|
||||
};
|
||||
}
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
// Nothing is reserved until the stack lock is held: a deploy
|
||||
// resume refused at the lock never started any work, so it must
|
||||
// not leave a reservation for startup recovery to chase.
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
nodeId,
|
||||
stackName,
|
||||
trigger: 'retry',
|
||||
actor: opts.actor,
|
||||
});
|
||||
'git_apply',
|
||||
actor,
|
||||
() => this.retryDeployLocked(stackName, applicationId, generationId, nodeId, actor),
|
||||
getRegistryDeliveryLockContext(),
|
||||
);
|
||||
if (!lock.ran) {
|
||||
const reason = `Another operation (${lock.existing.action}) is already in progress for ${stackName}.`;
|
||||
return { outcome: 'blocked', reason, nextAction: 'retry' };
|
||||
}
|
||||
return lock.result;
|
||||
}
|
||||
|
||||
/**
|
||||
* The deploy-stage retry, running while the shared stack lock is
|
||||
* held. The evidence the entry predicate proved (accepted generation
|
||||
* applied to the active Direct target with a recorded deploy failure
|
||||
* or a failed stack-scoped health verdict bound to that generation)
|
||||
* is re-read fresh here, because it could have moved between the
|
||||
* predicate and the lock. Only the deploy branch's steps run, replayed
|
||||
* by hand: the intent journal, the policy gate, the tracked Compose
|
||||
* deploy, the health gate, and the post-deploy scan. Fetching,
|
||||
* accepting, promoting, and re-binding are unreachable here by
|
||||
* construction, and the applied pointer itself would refuse a re-bind
|
||||
* (targetApplied() throws on a candidate-less re-bind).
|
||||
* Every path that reserved its attempt settles it before returning.
|
||||
*/
|
||||
private async retryDeployLocked(
|
||||
stackName: string,
|
||||
applicationId: string,
|
||||
generationId: string,
|
||||
nodeId: number,
|
||||
actor: string,
|
||||
): Promise<ReconcileResult> {
|
||||
const store = GitOpsStore.getInstance();
|
||||
const src = DatabaseService.getInstance().getGitSource(stackName);
|
||||
if (!src) {
|
||||
return {
|
||||
outcome: 'blocked',
|
||||
reason: `No Git source is configured for ${stackName}.`,
|
||||
nextAction: 'configure_credentials',
|
||||
};
|
||||
}
|
||||
|
||||
let envelope: ReturnType<GitSourceService['gitopsEnvelope']>;
|
||||
try {
|
||||
const allocated = GitOpsTransitions.getInstance().allocateReconcileAttempt(
|
||||
applicationId,
|
||||
actor,
|
||||
'retry',
|
||||
Date.now(),
|
||||
undefined,
|
||||
// The same markers the promote arm writes, so startup
|
||||
// recovery settles an interrupted deploy-resume against
|
||||
// this attempt's own deploy evidence: the reservation names
|
||||
// the exact generation and records that a deploy was
|
||||
// requested under it (always true here: the deploy retry is
|
||||
// itself the deploy request, where the promote arm writes
|
||||
// the source's auto-deploy flag).
|
||||
generationId,
|
||||
true,
|
||||
);
|
||||
if (!allocated.reserved) {
|
||||
console.error(
|
||||
`[GitSource] Deploy retry of ${sanitizeForLog(stackName)} allocated an already-reserved operation id ${sanitizeForLog(allocated.operationId)}.`,
|
||||
);
|
||||
return {
|
||||
outcome: 'blocked',
|
||||
reason: 'A dispatch attempt for this stack is already recorded; check its outcome before dispatching again.',
|
||||
nextAction: 'view_target_results',
|
||||
};
|
||||
}
|
||||
envelope = this.gitopsEnvelope(allocated.operationId, actor, 'retry');
|
||||
} catch (e) {
|
||||
console.error(
|
||||
`[GitSource] Failed to reserve a durable attempt for deploy retry of ${sanitizeForLog(stackName)}:`,
|
||||
errorForLog(e),
|
||||
);
|
||||
return {
|
||||
outcome: 'blocked',
|
||||
reason: "This stack's GitOps tracking is unavailable; reconfigure the source before dispatching again.",
|
||||
nextAction: 'configure_credentials',
|
||||
};
|
||||
}
|
||||
// Every refusal past the reservation settles the attempt: the
|
||||
// retry was real work that started durably, and recovery must be
|
||||
// able to see how it ended even when it ended in a refusal.
|
||||
const settleRefusal = (
|
||||
nextAction: ReconcileResult['nextAction'],
|
||||
raw: string,
|
||||
): ReconcileResult => {
|
||||
const reason = scrubCredentials(raw);
|
||||
this.settleAttempt(applicationId, envelope, { outcome: 'blocked', reason, nextAction });
|
||||
return { outcome: 'blocked', reason, nextAction };
|
||||
};
|
||||
try {
|
||||
// Re-read the evidence the entry predicate proved. The
|
||||
// promotion-era acceptance pointers moving in between means
|
||||
// this resume is no longer the truthful response, and the
|
||||
// state that replaced it owns what happens next.
|
||||
const app = store.getApplication(applicationId);
|
||||
if (!app || app.stack_name !== stackName) {
|
||||
return settleRefusal('resolve_conflict', 'No Direct stack is bound to this application.');
|
||||
}
|
||||
if (app.suspended_at) {
|
||||
return settleRefusal('resolve_conflict', `Reconciliation is suspended for ${stackName}.`);
|
||||
}
|
||||
if (app.target_mode !== 'direct') {
|
||||
return settleRefusal('resolve_conflict', 'The application is no longer in Direct mode.');
|
||||
}
|
||||
if (app.accepted_generation_id !== generationId) {
|
||||
return settleRefusal('resolve_conflict', 'The accepted generation changed before the deploy retry could run.');
|
||||
}
|
||||
const genRow = store.getGeneration(generationId);
|
||||
if (!genRow) {
|
||||
return settleRefusal('view_target_results', 'The accepted generation could not be read; its deploy evidence is unavailable.');
|
||||
}
|
||||
const target = store.getTarget(applicationId, nodeId);
|
||||
// Mirrors the arm selector exactly: deploy-failure evidence, or
|
||||
// (clear target evidence plus a failed stack health verdict for
|
||||
// this generation). Any other target failure belongs to its own
|
||||
// path, not to a deploy resume.
|
||||
const deployEvidence = !!target
|
||||
&& target.target_status === 'active'
|
||||
&& target.applied_generation_id === generationId
|
||||
&& (target.failure_stage === 'deploy'
|
||||
|| (target.failure_stage === null
|
||||
&& this.appliedGenerationHealthFailed(nodeId, stackName, generationId)));
|
||||
if (!deployEvidence) {
|
||||
// The evidence that selected this arm is gone or has
|
||||
// changed (a successful deploy cleared it, a newer
|
||||
// operation replaced the pointer, or the failed health
|
||||
// verdict is no longer the stack's latest claim for this
|
||||
// generation). Nothing here should have run; say so and
|
||||
// stop without touching the stack.
|
||||
return settleRefusal('view_target_results', 'The target no longer shows this generation as applied with a failed deploy or failed health verdict; nothing was re-deployed.');
|
||||
}
|
||||
|
||||
// The deploy intent, identical to the promote arm's: journal
|
||||
// it, read it back, and refuse rather than start an untracked
|
||||
// deploy (see recordDeployDispatchIntent). A resume whose
|
||||
// intent cannot be recorded and read back settles
|
||||
// recovery_required because the applied files are real
|
||||
// evidence the operator must see.
|
||||
const intent = this.recordDeployDispatchIntent({
|
||||
stackName,
|
||||
applicationId,
|
||||
envelope,
|
||||
generationId,
|
||||
commitSha: genRow.commit_sha,
|
||||
});
|
||||
if (intent.deployOperationId === null) {
|
||||
const cause = intent.refusalCause ? ` (${intent.refusalCause})` : '';
|
||||
const reason = `The deploy retry could not record and read back its deploy intent${cause}, so the deploy was not started.`;
|
||||
this.settlePostCommitAttempt(stackName, actor, applicationId, envelope, {
|
||||
outcome: 'recovery_required',
|
||||
reason,
|
||||
nextAction: 'view_target_results',
|
||||
});
|
||||
return { outcome: 'blocked', reason, nextAction: 'view_target_results' };
|
||||
}
|
||||
const deployOpIntent = intent.deployOperationId;
|
||||
|
||||
try {
|
||||
// The policy gate runs inside the deploy's own failure
|
||||
// classification, exactly as the completion pipeline runs
|
||||
// it there: a gate block after a committed promotion is
|
||||
// evidence the operator must see against the applied
|
||||
// files, not a pre-work refusal, so it settles
|
||||
// recovery_required under the same deploy-failure wording.
|
||||
await assertPolicyGateAllows(
|
||||
stackName,
|
||||
nodeId,
|
||||
buildSystemPolicyGateOptions(actor, {
|
||||
bypass: false,
|
||||
auditPath: `/api/stacks/${stackName}/git-source/apply`,
|
||||
}),
|
||||
);
|
||||
const autoDeploy = await ComposeService.getInstance(nodeId).deployStack(
|
||||
stackName,
|
||||
undefined,
|
||||
undefined,
|
||||
{ source: 'git_apply', actor, gitopsDeployOperationId: deployOpIntent },
|
||||
);
|
||||
HealthGateService.getInstance().beginStack(
|
||||
nodeId,
|
||||
stackName,
|
||||
'deploy',
|
||||
'system:git-source',
|
||||
{ deployedGenerationId: autoDeploy.deployedGenerationId },
|
||||
);
|
||||
console.log(
|
||||
`[GitSource] Deploy retried for ${stackName} at ${genRow.commit_sha.slice(0, 7)}${autoDeploy.gitopsOperationId ? ` (deploy op ${GitSourceService.shortOperationId(autoDeploy.gitopsOperationId)})` : ''}`,
|
||||
);
|
||||
// Fire-and-forget, matching the completion pipeline's
|
||||
// placement: the scan runs only after a successful deploy
|
||||
// and must never delay or fail the retry response.
|
||||
triggerPostDeployScan(stackName, nodeId).catch((err) =>
|
||||
console.error(`[Security] Post-deploy scan failed for ${sanitizeForLog(stackName)}:`, err),
|
||||
);
|
||||
const settledResult = this.finalizeReconcileOutcome(stackName, undefined);
|
||||
const settledPayload: ReconcileResult = {
|
||||
...settledResult,
|
||||
...(autoDeploy.gitopsOperationId
|
||||
? { deployGitopsOperationId: autoDeploy.gitopsOperationId }
|
||||
: {}),
|
||||
};
|
||||
this.settlePostCommitAttempt(stackName, actor, applicationId, envelope, settledPayload);
|
||||
return settledPayload;
|
||||
} catch (e) {
|
||||
// Parity with the completion pipeline: never
|
||||
// auto-compensate a deploy failure; the applied files
|
||||
// stand, the previous workload keeps running, and the
|
||||
// settled row names the failed deploy for recovery and for
|
||||
// the operator.
|
||||
const scrubbed = scrubCredentials((e as Error).message || String(e));
|
||||
console.error(`[GitSource] Deploy retry failed for ${stackName}: ${scrubbed}`);
|
||||
// ComposeService stamps the GitOps deploy operation id on
|
||||
// the error it rethrows when the deploy was tracked; read
|
||||
// it type-guarded so an untracked failure simply yields no
|
||||
// correlation instead of trusting an untyped field.
|
||||
const stamped: unknown = (e as { gitopsDeployOperationId?: unknown }).gitopsDeployOperationId;
|
||||
const deployOpId = typeof stamped === 'string' ? stamped : undefined;
|
||||
const reason = `The source applied, but the deploy failed: ${scrubbed}`;
|
||||
this.settlePostCommitAttempt(stackName, actor, applicationId, envelope, {
|
||||
outcome: 'recovery_required',
|
||||
reason,
|
||||
nextAction: 'view_target_results',
|
||||
...(deployOpId ? { deployGitopsOperationId: deployOpId } : {}),
|
||||
});
|
||||
return { outcome: 'blocked', reason, nextAction: 'view_target_results' };
|
||||
}
|
||||
} catch (e) {
|
||||
// The evidence re-read, the intent journaling, the gate, and
|
||||
// the deploy all run inside this try: a store access or an
|
||||
// unexpected throw must settle the reserved attempt and return
|
||||
// an outcome, never escape runExclusive with it open. Log
|
||||
// first; the settled row keeps only the scrubbed message.
|
||||
console.error(
|
||||
`[GitSource] Deploy retry of ${sanitizeForLog(stackName)} threw:`,
|
||||
errorForLog(e),
|
||||
);
|
||||
return settleRefusal('view_target_results', e instanceof Error ? (e.message || e.name) : String(e));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The deploy intent, shared by the dispatch's deploy branch and the
|
||||
* deploy-stage retry: mint the deploy operation id, journal it, and
|
||||
* return the id read back from the stored row (not the locally minted
|
||||
* one), so what Compose receives is always the exact id recovery will
|
||||
* look up. The journal write is dedupe-replayed: a replayed write
|
||||
* keeps the previously stored row (and reports it not newly
|
||||
* recorded), so the read-back, not the write's return value, is the
|
||||
* authority on what is actually stored. The caller hands the returned
|
||||
* id to Compose so its deploy transitions land under it.
|
||||
*
|
||||
* An attempt that cannot durably record and read back its intent
|
||||
* returns `deployOperationId: null`, and the caller refuses to start
|
||||
* the deploy: an untracked Compose run would leave no evidence row
|
||||
* recovery could link the deploy to, and recovery would have to judge
|
||||
* the attempt from source state alone. Refusing keeps every settled
|
||||
* row's claims backed by recorded rows. Each refusal also journals a
|
||||
* `deploy_intent_refused` witness (best-effort like the promotion
|
||||
* witness), so an attempt whose settle write failed in the same window
|
||||
* still reconstructs as the refusal at recovery instead of an
|
||||
* apply-only completion. `refusalCause` carries the scrubbed why for
|
||||
* the operator-visible settled reason: a cause-less refusal is
|
||||
* undebuggable.
|
||||
*/
|
||||
private recordDeployDispatchIntent(args: {
|
||||
stackName: string;
|
||||
applicationId: string;
|
||||
envelope: EventEnvelope;
|
||||
generationId: string;
|
||||
commitSha: string;
|
||||
}): { deployOperationId: string | null; refusalCause?: string } {
|
||||
const deployOperationId = crypto.randomUUID();
|
||||
const refuse = (cause: string): { deployOperationId: null; refusalCause: string } => {
|
||||
// The cause rides into the operator-visible settled reason, so
|
||||
// it is scrubbed like every other error string the dispatch
|
||||
// persists.
|
||||
const refusalCause = scrubCredentials(cause);
|
||||
if (!this.recordGitOps(args.stackName, 'deploy intent refusal witness', () => {
|
||||
GitOpsTransitions.getInstance().deployIntentRefused({
|
||||
applicationId: args.applicationId,
|
||||
envelope: args.envelope,
|
||||
generationId: args.generationId,
|
||||
commitSha: args.commitSha,
|
||||
});
|
||||
})) {
|
||||
// recordGitOps already logged the rejection; the operation
|
||||
// id ties that line to the intent failure being journaled
|
||||
// here. The reservation's deploy-request fact still
|
||||
// distinguishes this attempt from an apply-only completion
|
||||
// if the settle is lost too, so the witness is
|
||||
// belt-and-suspenders, not the last line.
|
||||
console.error(
|
||||
`[GitOps] Deploy intent refusal witness unavailable for ${sanitizeForLog(args.stackName)} (operation ${sanitizeForLog(args.envelope.operationId)}); recovery falls back to the reservation's recorded deploy request`,
|
||||
);
|
||||
}
|
||||
return { deployOperationId: null, refusalCause };
|
||||
};
|
||||
try {
|
||||
GitOpsTransitions.getInstance().deployDispatched({
|
||||
applicationId: args.applicationId,
|
||||
envelope: args.envelope,
|
||||
generationId: args.generationId,
|
||||
commitSha: args.commitSha,
|
||||
deployOperationId,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[GitOps] Could not record deploy intent for ${sanitizeForLog(args.stackName)} (operation ${sanitizeForLog(args.envelope.operationId)}):`,
|
||||
errorForLog(error),
|
||||
);
|
||||
return refuse(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
let intentRow: GitOpsHistoryRow | undefined;
|
||||
try {
|
||||
intentRow = GitOpsStore.getInstance()
|
||||
.getStageRowForAttempt(args.applicationId, args.envelope.operationId, 'deploy_dispatched');
|
||||
} catch (error) {
|
||||
// The write landed but the read-back throws (a store still
|
||||
// unhealthy in the write's wake): this is the same refusal,
|
||||
// not a deploy failure, and it must not be reported as one.
|
||||
return refuse(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
let journaled: unknown;
|
||||
if (intentRow) {
|
||||
try {
|
||||
journaled = decodeGitOpsJson(intentRow.after_json);
|
||||
} catch (error) {
|
||||
// A corrupt row is the same refusal as a payload that
|
||||
// fails the guard: the stored intent cannot be trusted, so
|
||||
// the deploy cannot be started against it. The decode
|
||||
// error rides into the cause so the corruption is
|
||||
// debuggable from the settled row.
|
||||
return refuse(`the stored intent payload failed validation: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
if (!isDeployDispatchedPayload(journaled)) {
|
||||
return refuse(intentRow
|
||||
? 'the stored intent payload failed validation'
|
||||
: 'the journaled intent row could not be read back');
|
||||
}
|
||||
return { deployOperationId: journaled.deployOperationId };
|
||||
}
|
||||
|
||||
/**
|
||||
* The next-action vocabulary for a blocked dispatch. For refusals that
|
||||
* reserved an attempt this mirrors what the dispatch's own settled row
|
||||
* advises; the lock-held refusal reserves nothing (no settled row
|
||||
* exists), and 'retry' is chosen there because nothing started.
|
||||
* Transient refusals (the stack lock held by another action, the
|
||||
* candidate read that failed on an IO blip) advise retrying the
|
||||
* dispatch; the duplicate-reservation and tracking-unavailable refusals
|
||||
* name their own remedy in the reason and map to the matching check;
|
||||
* post-commit refusals (bind rejection, deploy-intent refusal, dirty
|
||||
* promotion, failed deploy) advise viewing the target results; every
|
||||
* other pre-promotion refusal is a state conflict to resolve. The
|
||||
* reason strings are matched exactly, and the default is the
|
||||
* conservative conflict advice: a dispatch message reworded away from
|
||||
* these constants still returns a truthful blocked outcome, just with
|
||||
* the generic nextAction.
|
||||
*/
|
||||
private static dispatchRefusalNextAction(reason: string): ReconcileResult['nextAction'] {
|
||||
if (
|
||||
reason === 'Cannot read the accepted candidate; try again.'
|
||||
|| /^Another operation \(.+\) is already in progress for .+\.$/.test(reason)
|
||||
) {
|
||||
return 'retry';
|
||||
}
|
||||
if (
|
||||
reason === 'A dispatch attempt for this stack is already recorded; check its outcome before dispatching again.'
|
||||
) {
|
||||
return 'view_target_results';
|
||||
}
|
||||
if (reason === "This stack's GitOps tracking is unavailable; reconfigure the source before dispatching again.") {
|
||||
return 'configure_credentials';
|
||||
}
|
||||
if (
|
||||
reason.startsWith('The promotion committed, but')
|
||||
|| reason.startsWith('The promotion did not complete cleanly')
|
||||
|| reason.startsWith('The source applied, but the deploy failed')
|
||||
) {
|
||||
return 'view_target_results';
|
||||
}
|
||||
return 'resolve_conflict';
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -3910,9 +4552,9 @@ export class GitSourceService {
|
||||
// hook stops the pipeline there: the bindRejected arm (below)
|
||||
// classifies the rejection with the recovery_required vocabulary
|
||||
// the rewritten files warrant, and no deploy ever ran.
|
||||
// Captured by the deploy-intent hook on the refusal path so the
|
||||
// settled reason can name why the intent failed: a cause-less
|
||||
// refusal is undebuggable for the operator.
|
||||
// Captured from the deploy-intent hook on the refusal path so
|
||||
// the settled reason can name why the intent failed: a
|
||||
// cause-less refusal is undebuggable for the operator.
|
||||
let intentRefusalCause: string | undefined;
|
||||
const result = await this.completeGitApply({
|
||||
stackName,
|
||||
@@ -3969,106 +4611,23 @@ export class GitSourceService {
|
||||
planFingerprint: revalidation.plan.fingerprint,
|
||||
});
|
||||
}),
|
||||
// The deploy intent: mint the deploy operation id first,
|
||||
// journal it, and hand the journaled id to the pipeline so
|
||||
// Compose opens its deploy transitions under it. The id
|
||||
// returned is read back from the stored row, not the locally
|
||||
// minted one, so what Compose receives is always the exact id
|
||||
// recovery will look up (a dedupe replay reports recorded:true
|
||||
// even when it kept the stored id; the read-back below is the
|
||||
// authority on what is actually stored).
|
||||
// A dispatch that cannot durably record and read back its
|
||||
// intent returns null and the pipeline refuses to start the
|
||||
// deploy: an untracked Compose run would leave no evidence
|
||||
// row recovery could link the deploy to, and recovery would
|
||||
// have to judge the attempt from source state alone. Refusing
|
||||
// keeps every settled row's claims backed by recorded rows.
|
||||
// Each refusal also journals a `deploy_intent_refused`
|
||||
// witness (best-effort like the promotion witness), so an
|
||||
// attempt whose settle write failed in the same window still
|
||||
// reconstructs as the refusal at recovery instead of an
|
||||
// apply-only completion.
|
||||
// The deploy intent, shared with the deploy-stage retry:
|
||||
// journal it, read it back, and refuse rather than start
|
||||
// an untracked deploy (see recordDeployDispatchIntent).
|
||||
// The returned null keeps the hook's refusal contract: the
|
||||
// pipeline reports deployIntentUnavailable and never hands
|
||||
// the stack to Compose. The cause feeds the settled reason
|
||||
// below, exactly as the retry arm consumes it.
|
||||
deployDispatchIntent: (): string | null => {
|
||||
const deployOperationId = crypto.randomUUID();
|
||||
const refuse = (cause: string): null => {
|
||||
// The cause rides into the operator-visible settled
|
||||
// reason, so it is scrubbed like every other error
|
||||
// string the dispatch persists.
|
||||
intentRefusalCause = scrubCredentials(cause);
|
||||
if (!this.recordGitOps(stackName, 'deploy intent refusal witness', () => {
|
||||
GitOpsTransitions.getInstance().deployIntentRefused({
|
||||
applicationId: app.id,
|
||||
envelope,
|
||||
generationId: generation.generationId,
|
||||
commitSha: generation.commitSha,
|
||||
});
|
||||
})) {
|
||||
// recordGitOps already logged the rejection; the
|
||||
// operation id ties that line to the intent
|
||||
// failure being journaled here. The reservation's
|
||||
// deploy-request fact still distinguishes this
|
||||
// attempt from an apply-only completion if the
|
||||
// settle is lost too, so the witness is
|
||||
// belt-and-suspenders, not the last line.
|
||||
console.error(
|
||||
`[GitOps] Deploy intent refusal witness unavailable for ${sanitizeForLog(stackName)} (operation ${sanitizeForLog(envelope.operationId)}); recovery falls back to the reservation's recorded deploy request`,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
};
|
||||
try {
|
||||
GitOpsTransitions.getInstance().deployDispatched({
|
||||
applicationId: app.id,
|
||||
envelope,
|
||||
generationId: generation.generationId,
|
||||
commitSha: generation.commitSha,
|
||||
deployOperationId,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[GitOps] Could not record deploy intent for ${sanitizeForLog(stackName)} (operation ${sanitizeForLog(envelope.operationId)}):`,
|
||||
errorForLog(error),
|
||||
);
|
||||
return refuse(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
const intentRow = (() => {
|
||||
try {
|
||||
return GitOpsStore.getInstance()
|
||||
.getStageRowForAttempt(app.id, envelope.operationId, 'deploy_dispatched');
|
||||
} catch (error) {
|
||||
// The write landed but the read-back throws (a
|
||||
// store still unhealthy in the write's wake):
|
||||
// this is the same refusal, not a deploy
|
||||
// failure. It must not escape into the
|
||||
// pipeline's deploy-error arm, which would
|
||||
// misreport it as "the deploy failed" when
|
||||
// Compose was never called.
|
||||
return refuse(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
})();
|
||||
if (intentRow === null) return null;
|
||||
let journaled: unknown = undefined;
|
||||
if (intentRow) {
|
||||
try {
|
||||
journaled = decodeGitOpsJson(intentRow.after_json);
|
||||
} catch (error) {
|
||||
// A corrupt row is the same refusal as a payload
|
||||
// that fails the guard: the stored intent cannot
|
||||
// be trusted, so the deploy cannot be started
|
||||
// against it. The throw must not escape into the
|
||||
// pipeline's deploy-error arm, which would
|
||||
// misreport this refusal as a failed deploy.
|
||||
// The decode error rides into the cause so the
|
||||
// corruption is debuggable from the settled row.
|
||||
return refuse(`the stored intent payload failed validation: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
if (!isDeployDispatchedPayload(journaled)) {
|
||||
return refuse(intentRow
|
||||
? 'the stored intent payload failed validation'
|
||||
: 'the journaled intent row could not be read back');
|
||||
}
|
||||
return journaled.deployOperationId;
|
||||
const intent = this.recordDeployDispatchIntent({
|
||||
stackName,
|
||||
applicationId: app.id,
|
||||
envelope,
|
||||
generationId: generation.generationId,
|
||||
commitSha: generation.commitSha,
|
||||
});
|
||||
intentRefusalCause = intent.refusalCause;
|
||||
return intent.deployOperationId;
|
||||
},
|
||||
});
|
||||
if (result.bindRejected) {
|
||||
|
||||
@@ -15,6 +15,7 @@ import { NodeRegistry } from '../NodeRegistry';
|
||||
import { gitSourceLocalComposeFiles } from '../../utils/gitComposeFiles';
|
||||
import { extractImagesFromCompose, loadDotEnv } from '../ImageUpdateService';
|
||||
import type { ReconcileOutcome } from './outcomes';
|
||||
import type { ReconcileTrigger } from './triggers';
|
||||
import { redactSensitiveText, sanitizeForLog } from '../../utils/safeLog';
|
||||
|
||||
/** Outcomes that mean the tick accomplished its work and the source may sleep again. */
|
||||
@@ -42,10 +43,16 @@ const SUCCESS_SHAPED_OUTCOMES: ReadonlySet<ReconcileOutcome> = new Set<Reconcile
|
||||
* Review-policy sources stage candidates for a human; manual
|
||||
* sources never join the unattended cadence.
|
||||
*
|
||||
* Remaining gap against the source policy design: a retry always re-issues
|
||||
* a fetch rather than resuming at the stage that failed (fetch vs. apply
|
||||
* dispatch). Stage-aware retry is follow-on work once failures can carry
|
||||
* enough durable context to resume mid-application.
|
||||
* Stage-aware retry lives in GitSourceService.retry(): an operator retry
|
||||
* at a source-stage failure re-issues a fetch, an accepted generation
|
||||
* still awaiting its promotion is retried against the shared dispatch
|
||||
* boundary, and a generation already applied whose deploy failed (or
|
||||
* whose stack health gate last recorded a failed verdict for it) resumes
|
||||
* at the deploy alone; a source-stage failure outranks dispatch evidence,
|
||||
* and a suspended, in-flight, or under-recovery source defers before any
|
||||
* arm runs. Neither retry arm ever refetches or re-accepts.
|
||||
* This controller's own automated retry scheduling stays at the fetch
|
||||
* stage, where every transient failure it classifies can occur.
|
||||
*
|
||||
* One self-rescheduling timer drives the scan, matching ImageUpdateService.
|
||||
* The re-arm always runs, even when a scan throws, so one bad tick (a
|
||||
@@ -62,12 +69,23 @@ export class SourceController {
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
private polling = false;
|
||||
// Bumped by cancelPending(), so by stop() and restartPolling(). tick() has
|
||||
// no internal await point today, so nothing can currently call either one
|
||||
// no internal await point, so nothing can currently call either one
|
||||
// mid-tick; this is a second, currently-redundant line of defense against a
|
||||
// stale timer firing, kept cheap on purpose for when stage-aware retry (see
|
||||
// above) gives evaluate() a real yield point.
|
||||
// stale timer firing, kept cheap on purpose (an integer compare checked at
|
||||
// tick entry and before re-arming). Stage-aware retry landed in
|
||||
// GitSourceService.retry() instead of evaluate(), so no yield point was
|
||||
// introduced here after all.
|
||||
private scheduleGeneration = 0;
|
||||
private readonly inFlight = new Set<string>();
|
||||
/**
|
||||
* Resume wakes that arrived while their application was already being
|
||||
* evaluated. Keyed by application id (so repeated wakes coalesce into
|
||||
* the one evaluation they asked for) holding the stack name the drain
|
||||
* re-reads eligibility from. Lives only until the in-flight owner
|
||||
* releases the slot: a marker exists only while some evaluation owns
|
||||
* the slot, and that owner's release consumes it.
|
||||
*/
|
||||
private readonly pendingWakes = new Map<string, string>();
|
||||
|
||||
private constructor() { }
|
||||
|
||||
@@ -216,11 +234,97 @@ export class SourceController {
|
||||
`[SourceController] evaluation crashed for ${sanitizeForLog(app.id)}:`,
|
||||
e instanceof Error ? e.message : String(e),
|
||||
))
|
||||
.finally(() => this.inFlight.delete(app.id));
|
||||
.finally(() => this.releaseAndDrainWake(app.id));
|
||||
}
|
||||
}
|
||||
|
||||
private async evaluate(app: GitOpsApplicationRow): Promise<void> {
|
||||
/**
|
||||
* Release an application's evaluation slot and honor any resume wake
|
||||
* parked against it. The wake's evaluateNow runs synchronously before
|
||||
* this returns, so no timer tick can claim the freed slot in between:
|
||||
* a parked operator resume wins the slot against the cadence, which
|
||||
* is the point of a wake. evaluateNow re-reads the row and re-decides
|
||||
* eligibility, so a wake that went stale while parked (re-suspended,
|
||||
* switched to manual, deleted) is dropped exactly as a fresh call
|
||||
* would drop it. evaluateNow's own catch only covers its evaluation
|
||||
* arm: its synchronous prologue (the row read and eligibility guard)
|
||||
* can reject before that catch exists, and this drain runs inside a
|
||||
* release finally with nothing above it to catch, so the detached
|
||||
* wake carries the same defensive log the resume route applies to its
|
||||
* fire-and-forget wake.
|
||||
*/
|
||||
private releaseAndDrainWake(applicationId: string): void {
|
||||
this.inFlight.delete(applicationId);
|
||||
const wakeStackName = this.pendingWakes.get(applicationId);
|
||||
if (wakeStackName !== undefined) {
|
||||
this.pendingWakes.delete(applicationId);
|
||||
this.evaluateNow(wakeStackName).catch((e) => console.error(
|
||||
`[SourceController] parked-wake drain could not start for ${sanitizeForLog(applicationId)}:`,
|
||||
e instanceof Error ? e.message : String(e),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one unattended evaluation for a stack right now, without waiting
|
||||
* for the timer. The resume route calls this after clearing
|
||||
* suspension so the woken source is re-evaluated immediately (source
|
||||
* state and, through the automatic arm, target binding and dispatch)
|
||||
* instead of idling until the next poll or retry cursor lands.
|
||||
* Eligibility is re-decided here rather than inherited from the
|
||||
* due-scan, because this path bypasses the scan: only a live (active or
|
||||
* creating), direct, non-manual, unsuspended source with a stack name is
|
||||
* evaluated. A row that is due for neither poll nor retry is still
|
||||
* evaluated: the operator's resume is the wake. Nothing is thrown at the
|
||||
* caller when the row simply is not eligible; whether the source runs on
|
||||
* a cursor or on demand is the scheduling code's business, and an
|
||||
* ineligible row is not an error.
|
||||
*
|
||||
* An application already being evaluated is never queued behind itself:
|
||||
* the wake is parked instead of discarded, and the in-flight owner's
|
||||
* release runs at most one fresh resume evaluation for it (see
|
||||
* releaseAndDrainWake). Repeated wakes coalesce into the one evaluation
|
||||
* they asked for; eligibility is re-read at drain time, so a wake that
|
||||
* went stale while parked is dropped exactly as a fresh call would drop
|
||||
* it.
|
||||
*
|
||||
* Fetches fired here coalesce with any concurrent tick through the
|
||||
* reconcile submission's per-application joining, so an immediate
|
||||
* evaluation landing seconds before a timer tick cannot double-fetch.
|
||||
*/
|
||||
public async evaluateNow(stackName: string): Promise<void> {
|
||||
const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName);
|
||||
if (!app) return;
|
||||
if (app.source_policy === 'manual' || app.suspended_at) return;
|
||||
if (this.inFlight.has(app.id)) {
|
||||
// Park rather than discard: the resume is a request to
|
||||
// re-evaluate after whatever is running now settles, and an
|
||||
// evaluation about to finish would otherwise silently swallow
|
||||
// the operator's wake (the next timer tick can be a full poll
|
||||
// interval away).
|
||||
this.pendingWakes.set(app.id, stackName);
|
||||
return;
|
||||
}
|
||||
this.inFlight.add(app.id);
|
||||
try {
|
||||
await this.evaluate(app, 'resume');
|
||||
} catch (e) {
|
||||
// Same load-bearing catch the tick path wraps evaluations in:
|
||||
// everything past the reconcile call (the row re-read,
|
||||
// scheduling, automatic acceptance) runs on this promise chain,
|
||||
// and a rejection escaping here would be an unhandled one for
|
||||
// the route that fired it. The durable row is authoritative;
|
||||
// the next tick remains the safety net.
|
||||
console.error(
|
||||
`[SourceController] immediate evaluation crashed for ${sanitizeForLog(app.id)}:`,
|
||||
e instanceof Error ? e.message : String(e),
|
||||
);
|
||||
} finally {
|
||||
this.releaseAndDrainWake(app.id);
|
||||
}
|
||||
}
|
||||
|
||||
private async evaluate(app: GitOpsApplicationRow, triggerOverride?: ReconcileTrigger): Promise<void> {
|
||||
if (!app.stack_name) {
|
||||
console.warn(`[SourceController] Skipping ${sanitizeForLog(app.id)}: direct-mode application has no stack_name.`);
|
||||
return;
|
||||
@@ -234,13 +338,16 @@ export class SourceController {
|
||||
}
|
||||
const stackName = app.stack_name;
|
||||
const isRetry = app.retry_at !== null && app.retry_at <= Date.now();
|
||||
// An out-of-band caller (the resume path) names its own trigger; a
|
||||
// timer tick derives it from the row's cursors as before.
|
||||
const trigger: ReconcileTrigger = triggerOverride ?? (isRetry ? 'retry' : 'poll');
|
||||
let result;
|
||||
try {
|
||||
result = await GitSourceService.getInstance().reconcile({
|
||||
intent: 'fetch',
|
||||
applicationId: app.id,
|
||||
stackName,
|
||||
trigger: isRetry ? 'retry' : 'poll',
|
||||
trigger,
|
||||
actor: 'system:source-controller',
|
||||
});
|
||||
} catch (e) {
|
||||
@@ -268,7 +375,7 @@ export class SourceController {
|
||||
// refuses a durable blocked candidate below
|
||||
// (requireAcceptableCandidate guards against an outcome that
|
||||
// misreports the row's state).
|
||||
await this.maybeAcceptAutomaticCandidate(fresh, stackName, isRetry ? 'retry' : 'poll');
|
||||
await this.maybeAcceptAutomaticCandidate(fresh, stackName, trigger);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -330,7 +437,7 @@ export class SourceController {
|
||||
private async maybeAcceptAutomaticCandidate(
|
||||
app: GitOpsApplicationRow,
|
||||
stackName: string,
|
||||
trigger: 'poll' | 'retry',
|
||||
trigger: ReconcileTrigger,
|
||||
): Promise<void> {
|
||||
if (app.source_policy !== 'automatic') return;
|
||||
if (!app.candidate_generation_id || app.accepted_generation_id === app.candidate_generation_id) return;
|
||||
@@ -387,7 +494,7 @@ export class SourceController {
|
||||
try {
|
||||
const dispatch = await GitSourceService.getInstance().dispatchAcceptedGeneration(
|
||||
buildAcceptedGeneration(generation),
|
||||
{ targetMode: 'direct', nodeId: NodeRegistry.getInstance().getDefaultNodeId(), bindingRevision: null },
|
||||
GitSourceService.directDispatchContext(),
|
||||
{ trigger, actor: 'system:source-controller' },
|
||||
);
|
||||
if (dispatch.status === 'blocked') {
|
||||
|
||||
@@ -648,6 +648,127 @@ test.describe('Git Sources complete-project materialization (local git server)',
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('Git source suspend and resume over the API (local git server)', () => {
|
||||
test.skip(!gitAvailable(), 'system git binary is not available');
|
||||
|
||||
// The whole journey is one scenario with wall-clock steps (an unattended
|
||||
// poll fires on a 60s floor), so it gets a longer timeout than the suite
|
||||
// default rather than being split into state-leaking fragments.
|
||||
test.setTimeout(180_000);
|
||||
|
||||
let server: { url: string; close: () => void };
|
||||
let stackName: string;
|
||||
let previousPollIntervalMins: number;
|
||||
|
||||
test.beforeAll(async () => {
|
||||
server = await serveRepos({
|
||||
sync: buildFixtureRepo({ 'compose.yaml': 'services:\n web:\n image: nginx\n' }),
|
||||
});
|
||||
});
|
||||
|
||||
test.afterAll(() => {
|
||||
server?.close();
|
||||
});
|
||||
|
||||
async function api(page: Page, path: string, init?: { method?: string; body?: unknown }) {
|
||||
return page.evaluate(async ({ p, opts }) => {
|
||||
const res = await fetch(p, {
|
||||
method: opts?.method ?? 'GET',
|
||||
headers: opts?.body !== undefined ? { 'Content-Type': 'application/json' } : undefined,
|
||||
body: opts?.body !== undefined ? JSON.stringify(opts.body) : undefined,
|
||||
credentials: 'include',
|
||||
});
|
||||
let json: unknown = null;
|
||||
try { json = await res.json(); } catch { /* empty or non-JSON body */ }
|
||||
return { status: res.status, json };
|
||||
}, { p: path, opts: init });
|
||||
}
|
||||
|
||||
/** History entries for this stack, newest page first, as { stage, trigger, outcome }. */
|
||||
async function historyEntries(page: Page): Promise<Array<{ stage: string; trigger: string; outcome: string }>> {
|
||||
const res = await api(page, `/api/stacks/${stackName}/git-source/history?limit=100`);
|
||||
const items = (res.json as { items?: Array<Record<string, unknown>> } | null)?.items ?? [];
|
||||
return items.map((i) => ({
|
||||
stage: String(i.stage ?? ''),
|
||||
trigger: String(i.trigger ?? ''),
|
||||
outcome: String(i.outcome ?? ''),
|
||||
}));
|
||||
}
|
||||
|
||||
test('suspension stops reconciliation and resume re-evaluates the source immediately', async ({ page }) => {
|
||||
await loginAs(page);
|
||||
stackName = `e2e-susp-${Date.now()}`;
|
||||
|
||||
// Park the unattended cadence on the slowest floor the settings allow
|
||||
// (60s effective) and restore whatever the instance had before, so the
|
||||
// scenario never races a background poll and never changes the box.
|
||||
previousPollIntervalMins = ((await api(page, '/api/git-sources/polling')).json as { poll_interval_mins?: number })?.poll_interval_mins ?? 0;
|
||||
await api(page, '/api/git-sources/polling', { method: 'PATCH', body: { poll_interval_mins: 1 } });
|
||||
|
||||
try {
|
||||
expect((await api(page, '/api/stacks', { method: 'POST', body: { stackName } })).status).toBe(200);
|
||||
|
||||
const saved = await api(page, `/api/stacks/${stackName}/git-source`, {
|
||||
method: 'PUT',
|
||||
body: {
|
||||
repo_url: `${server.url}/sync.git`,
|
||||
branch: 'main',
|
||||
compose_paths: ['compose.yaml'],
|
||||
context_dir: null,
|
||||
sync_env: false,
|
||||
auth_type: 'none',
|
||||
auto_apply_on_webhook: false,
|
||||
auto_deploy_on_apply: false,
|
||||
source_policy: 'automatic',
|
||||
},
|
||||
});
|
||||
expect(saved.status, JSON.stringify(saved.json)).toBe(200);
|
||||
|
||||
// A manual pull proves the source is live and records a manual-
|
||||
// trigger attempt before the suspension journey starts.
|
||||
const pulled = await api(page, `/api/stacks/${stackName}/git-source/pull`, { method: 'POST' });
|
||||
expect(pulled.status, JSON.stringify(pulled.json)).toBe(200);
|
||||
|
||||
const suspended = await api(page, `/api/stacks/${stackName}/git-source/suspend`, {
|
||||
method: 'POST',
|
||||
body: { reason: 'e2e maintenance hold' },
|
||||
});
|
||||
expect(suspended.status).toBe(200);
|
||||
expect((suspended.json as { outcome: string }).outcome).toBe('suspended');
|
||||
|
||||
// While suspended, a pull is refused outright, not silently skipped.
|
||||
const refused = await api(page, `/api/stacks/${stackName}/git-source/pull`, { method: 'POST' });
|
||||
expect(refused.status).toBe(409);
|
||||
expect((refused.json as { code: string }).code).toBe('OPERATION_IN_FLIGHT');
|
||||
// resume() itself stamps 'resume' on the suspension transition and
|
||||
// the armed poll cursor, so only a reconcile-start row can prove the
|
||||
// immediate re-evaluation actually fired.
|
||||
const reconcileStarts = (rows: Array<{ stage: string; trigger: string }>) =>
|
||||
rows.filter((e) => e.stage === 'source_reconcile_started' && e.trigger === 'resume').length;
|
||||
expect(reconcileStarts(await historyEntries(page))).toBe(0);
|
||||
|
||||
const resumed = await api(page, `/api/stacks/${stackName}/git-source/resume`, { method: 'POST' });
|
||||
expect(resumed.status, JSON.stringify(resumed.json)).toBe(200);
|
||||
expect((resumed.json as { outcome: string }).outcome).not.toBe('suspended');
|
||||
|
||||
// The response returns before the reconcile it kicked off can finish,
|
||||
// so poll the history until the resume-trigger reconcile lands: proof
|
||||
// the resumed source was re-evaluated now rather than at its next
|
||||
// scheduled cursor (which is a minute away).
|
||||
await expect.poll(async () => {
|
||||
return reconcileStarts(await historyEntries(page));
|
||||
}, { timeout: 30_000, intervals: [1_000] }).toBeGreaterThan(0);
|
||||
} finally {
|
||||
await api(page, `/api/stacks/${stackName}/git-source`, { method: 'DELETE' }).catch(() => {});
|
||||
await api(page, `/api/stacks/${stackName}`, { method: 'DELETE' }).catch(() => {});
|
||||
await api(page, '/api/git-sources/polling', {
|
||||
method: 'PATCH',
|
||||
body: { poll_interval_mins: previousPollIntervalMins },
|
||||
}).catch(() => {});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('Git Sources SSH deploy key (real backend)', () => {
|
||||
test.skip(!sshGitFixtureAvailable(), 'requires git and openssh-server');
|
||||
|
||||
|
||||
Reference in New Issue
Block a user