mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-21 07:36:40 +00:00
fix: untag reviewed image names during prune (#1836)
This commit is contained in:
@@ -1301,6 +1301,7 @@ describe('POST /api/fleet/labels/fleet-prune with dryRun: true', () => {
|
||||
executePrunePlan.mockResolvedValue({
|
||||
success: true,
|
||||
reclaimedBytes: 0,
|
||||
mutated: true,
|
||||
outcomes: [{ target: 'images', id: 'image', status: 'removed' }],
|
||||
});
|
||||
const res = await request(app)
|
||||
@@ -1337,7 +1338,12 @@ describe('POST /api/fleet/labels/fleet-prune with dryRun: true', () => {
|
||||
nodeId: local.id, scope: 'managed', targets: ['images'], items: [...testCase.items],
|
||||
reclaimableBytes: 0, fingerprint, createdAt: 1,
|
||||
});
|
||||
executePrunePlan.mockResolvedValue({ success: true, reclaimedBytes: 0, outcomes: [...testCase.outcomes] });
|
||||
executePrunePlan.mockResolvedValue({
|
||||
success: true,
|
||||
reclaimedBytes: 0,
|
||||
mutated: false,
|
||||
outcomes: [...testCase.outcomes],
|
||||
});
|
||||
invalidateNodeCaches.mockClear();
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
|
||||
@@ -3,6 +3,7 @@ import request from 'supertest';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { cleanupTestDb, setupTestDb, TEST_JWT_SECRET, TEST_USERNAME } from './helpers/setupTestDb';
|
||||
import type { PruneItemOutcome, PrunePlan, PrunePlanItem } from '../services/prunePlan';
|
||||
import { CacheService } from '../services/CacheService';
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
@@ -62,9 +63,11 @@ function mockLocal(planFactory: (nodeId: number) => PrunePlan = (nodeId) => plan
|
||||
success: boolean;
|
||||
reclaimedBytes: number;
|
||||
outcomes: PruneItemOutcome[];
|
||||
mutated: boolean;
|
||||
}> => ({
|
||||
success: true,
|
||||
reclaimedBytes: reviewedPlan.reclaimableBytes,
|
||||
mutated: reviewedPlan.items.length > 0,
|
||||
outcomes: reviewedPlan.items.map((entry) => ({
|
||||
id: entry.id,
|
||||
target: entry.target,
|
||||
@@ -212,6 +215,30 @@ describe('POST /api/fleet/labels/fleet-prune', () => {
|
||||
expect(activeBulkActions.size).toBe(0);
|
||||
});
|
||||
|
||||
it('invalidates local node caches when a failed image outcome is mutated', async () => {
|
||||
const fake = mockLocal();
|
||||
fake.executePrunePlan.mockResolvedValue({
|
||||
success: false,
|
||||
reclaimedBytes: 0,
|
||||
mutated: true,
|
||||
outcomes: [{ target: 'images', id: 'sha256:image', status: 'failed', error: 'tags remain' }],
|
||||
});
|
||||
const invalidate = vi.spyOn(CacheService.getInstance(), 'invalidate');
|
||||
const local = DatabaseService.getInstance().getNodes().find((node) => node.type === 'local')!;
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
targets: ['images'], scope: 'managed', dryRun: false,
|
||||
reviewedNodes: [{ nodeId: local.id, reachable: true }],
|
||||
plans: [{ nodeId: local.id, fingerprint: `fingerprint-${local.id}` }],
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.results[0].outcomes[0].status).toBe('failed');
|
||||
expect(invalidate).toHaveBeenCalledWith(`stats:${local.id}`);
|
||||
expect(invalidate).toHaveBeenCalledWith(`stack-statuses:${local.id}`);
|
||||
});
|
||||
|
||||
it('fails closed when a local prune lock is active', async () => {
|
||||
const fake = mockLocal();
|
||||
const { local, body } = localReview(`fingerprint-${DatabaseService.getInstance().getNodes()[0].id}`);
|
||||
@@ -440,6 +467,7 @@ describe('POST /api/fleet/labels/fleet-prune', () => {
|
||||
fake.executePrunePlan.mockResolvedValue({
|
||||
success: false,
|
||||
reclaimedBytes: 100,
|
||||
mutated: true,
|
||||
outcomes: [
|
||||
{ target: 'images', id: 'removed', status: 'removed', sizeBytes: 100 },
|
||||
{ target: 'images', id: 'skipped', status: 'skipped', reason: 'became active' },
|
||||
|
||||
@@ -659,9 +659,77 @@ describe('DockerController.buildPrunePlan', () => {
|
||||
|
||||
expect(plan.items.map((i) => i.id)).toEqual(['img-free']);
|
||||
});
|
||||
|
||||
it('excludes a fully synthetic sencho-rb hold image from the plan even without a DB hold', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{
|
||||
Id: 'img-orphan-hold',
|
||||
RepoTags: ['sencho-rb/abc123456789/web:hold', 'sencho-rb/abc123456789/api:hold'],
|
||||
Size: 50,
|
||||
Containers: 0,
|
||||
},
|
||||
{ Id: 'img-free', RepoTags: ['app:2'], Size: 100, Containers: 0 },
|
||||
]);
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const plan = await dc.buildPrunePlan(['images'], 'all', [], 1, () => false);
|
||||
|
||||
expect(plan.items.map((i) => i.id)).toEqual(['img-free']);
|
||||
});
|
||||
|
||||
it('still plans a dual-tagged image that carries a registry tag and a sencho-rb hold tag', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{
|
||||
Id: 'img-dual',
|
||||
RepoTags: ['myregistry/app:1.4', 'sencho-rb/abc123456789/app:hold'],
|
||||
Size: 100,
|
||||
Containers: 0,
|
||||
},
|
||||
]);
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const plan = await dc.buildPrunePlan(['images'], 'all', [], 1);
|
||||
|
||||
expect(plan.items.map((i) => i.id)).toEqual(['img-dual']);
|
||||
expect(plan.items[0]?.image?.references).toEqual([
|
||||
'myregistry/app:1.4',
|
||||
'sencho-rb/abc123456789/app:hold',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DockerController.executePrunePlan', () => {
|
||||
const multiTagImage = {
|
||||
Id: 'img-multi',
|
||||
RepoTags: ['qa1769/app:v0', 'qa1769-external/keep:v0'],
|
||||
Size: 50,
|
||||
Containers: 0,
|
||||
};
|
||||
|
||||
function mockImageRemove(
|
||||
onRemove?: (name: string, opts?: { force?: boolean }) => void | Promise<void>,
|
||||
): Array<{ name: string; force?: boolean }> {
|
||||
const removes: Array<{ name: string; force?: boolean }> = [];
|
||||
mockDocker.getImage.mockImplementation((name: string) => ({
|
||||
remove: vi.fn().mockImplementation(async (opts?: { force?: boolean }) => {
|
||||
removes.push({ name, force: opts?.force });
|
||||
await onRemove?.(name, opts);
|
||||
}),
|
||||
}));
|
||||
return removes;
|
||||
}
|
||||
|
||||
async function executeForcedFreshImagePlan(listings: Array<unknown[] | Error>) {
|
||||
for (const listing of listings) {
|
||||
if (listing instanceof Error) mockDocker.listImages.mockRejectedValueOnce(listing);
|
||||
else mockDocker.listImages.mockResolvedValueOnce(listing);
|
||||
}
|
||||
const dc = DockerController.getInstance(1);
|
||||
const plan = await dc.buildPrunePlan(['images'], 'all', [], 1);
|
||||
vi.spyOn(dc, 'assertPlanFresh').mockResolvedValue(plan);
|
||||
return dc.executePrunePlan(plan, []);
|
||||
}
|
||||
|
||||
it('throws PrunePlanStaleError when the fingerprint no longer matches', async () => {
|
||||
mockDocker.listContainers.mockResolvedValue([
|
||||
{
|
||||
@@ -852,6 +920,49 @@ describe('DockerController.executePrunePlan', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('skips a fully synthetic hold image even under a forced-fresh plan', async () => {
|
||||
const holdImg = {
|
||||
Id: 'img-orphan-hold',
|
||||
RepoTags: ['sencho-rb/abc123456789/web:hold', 'sencho-rb/abc123456789/api:hold'],
|
||||
Size: 50,
|
||||
Containers: 0,
|
||||
};
|
||||
mockDocker.listImages.mockResolvedValue([holdImg]);
|
||||
const removes = mockImageRemove();
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const plan = {
|
||||
scope: 'all' as const,
|
||||
targets: ['images' as const],
|
||||
items: [{
|
||||
target: 'images' as const,
|
||||
id: holdImg.Id,
|
||||
name: holdImg.RepoTags[0],
|
||||
sizeBytes: 50,
|
||||
managed: false,
|
||||
reason: 'Image is not used by any container',
|
||||
image: { references: holdImg.RepoTags },
|
||||
}],
|
||||
reclaimableBytes: 50,
|
||||
fingerprint: 'forced-hold',
|
||||
createdAt: Date.now(),
|
||||
nodeId: 1,
|
||||
};
|
||||
vi.spyOn(dc, 'assertPlanFresh').mockResolvedValue(plan);
|
||||
const result = await dc.executePrunePlan(plan, []);
|
||||
|
||||
expect(removes).toEqual([]);
|
||||
expect(result.mutated).toBe(false);
|
||||
expect(result.outcomes).toEqual([
|
||||
expect.objectContaining({
|
||||
id: holdImg.Id,
|
||||
target: 'images',
|
||||
status: 'skipped',
|
||||
reason: 'Sencho rollback-hold image',
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('marks the plan stale when a free image gains a new RepoTag between plan and rebuild', async () => {
|
||||
const img = {
|
||||
Id: 'img-retag',
|
||||
@@ -896,29 +1007,177 @@ describe('DockerController.executePrunePlan', () => {
|
||||
}));
|
||||
});
|
||||
|
||||
it('surfaces multi-repository refuse without a silent partial untag report', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{
|
||||
Id: 'img-multi',
|
||||
RepoTags: ['qa1769/app:v0', 'qa1769-external/keep:v0'],
|
||||
Size: 50,
|
||||
Containers: 0,
|
||||
},
|
||||
]);
|
||||
const imageRemove = vi.fn().mockRejectedValue(
|
||||
Object.assign(new Error('conflict: unable to delete (must be forced) - image is referenced in multiple repositories'), { statusCode: 409 }),
|
||||
);
|
||||
mockDocker.getImage.mockReturnValue({ remove: imageRemove });
|
||||
it('untags each reviewed name and reports removed once the image is gone', async () => {
|
||||
const removes = mockImageRemove();
|
||||
const result = await executeForcedFreshImagePlan([[multiTagImage], [multiTagImage], []]);
|
||||
|
||||
expect(removes.map((entry) => entry.name)).toEqual(['qa1769/app:v0', 'qa1769-external/keep:v0']);
|
||||
expect(result.outcomes[0]).toEqual(expect.objectContaining({
|
||||
id: 'img-multi',
|
||||
status: 'removed',
|
||||
}));
|
||||
expect(result.mutated).toBe(true);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('treats a first-ref 404 as absent and still removes when a later reviewed tag succeeds', async () => {
|
||||
const removes = mockImageRemove(async (name) => {
|
||||
if (name === 'qa1769/app:v0') {
|
||||
throw Object.assign(new Error('No such image: qa1769/app:v0'), { statusCode: 404 });
|
||||
}
|
||||
});
|
||||
const result = await executeForcedFreshImagePlan([[multiTagImage], [multiTagImage], []]);
|
||||
|
||||
expect(removes).toEqual([
|
||||
{ name: 'qa1769/app:v0', force: false },
|
||||
{ name: 'qa1769-external/keep:v0', force: false },
|
||||
]);
|
||||
expect(result.outcomes[0]).toEqual(expect.objectContaining({ status: 'removed' }));
|
||||
expect(result.mutated).toBe(true);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('does not report removed when a later 404 leaves another reviewed tag present', async () => {
|
||||
mockImageRemove(async (name) => {
|
||||
if (name === 'qa1769-external/keep:v0') {
|
||||
throw Object.assign(new Error('No such image'), { statusCode: 404 });
|
||||
}
|
||||
});
|
||||
const result = await executeForcedFreshImagePlan([
|
||||
[multiTagImage],
|
||||
[multiTagImage],
|
||||
[{ ...multiTagImage, RepoTags: ['qa1769/app:v0'] }],
|
||||
]);
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const plan = await dc.buildPrunePlan(['images'], 'all', [], 1);
|
||||
vi.spyOn(dc, 'assertPlanFresh').mockResolvedValue(plan);
|
||||
const result = await dc.executePrunePlan(plan, []);
|
||||
expect(imageRemove).toHaveBeenCalled();
|
||||
expect(result.outcomes[0]).toEqual(expect.objectContaining({
|
||||
status: 'failed',
|
||||
error: expect.stringMatching(/multiple repositories/i),
|
||||
error: expect.stringMatching(/qa1769\/app:v0/),
|
||||
}));
|
||||
expect(String(result.outcomes[0] && 'error' in result.outcomes[0] ? result.outcomes[0].error : '')).toMatch(/qa1769\/app:v0/);
|
||||
expect(result.outcomes[0]).not.toEqual(expect.objectContaining({ status: 'removed' }));
|
||||
expect(result.mutated).toBe(true);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('stops on a hard untag error, lists remaining tags, and marks mutated', async () => {
|
||||
const removes = mockImageRemove(async (name) => {
|
||||
if (name === 'qa1769-external/keep:v0') {
|
||||
throw Object.assign(new Error('conflict: unable to delete (must be forced)'), { statusCode: 409 });
|
||||
}
|
||||
});
|
||||
const result = await executeForcedFreshImagePlan([
|
||||
[multiTagImage],
|
||||
[multiTagImage],
|
||||
[{ ...multiTagImage, RepoTags: ['qa1769-external/keep:v0'] }],
|
||||
]);
|
||||
|
||||
expect(removes.map((entry) => entry.name)).toEqual(['qa1769/app:v0', 'qa1769-external/keep:v0']);
|
||||
expect(result.outcomes[0]).toEqual(expect.objectContaining({
|
||||
status: 'failed',
|
||||
error: expect.stringMatching(/qa1769-external\/keep:v0/),
|
||||
}));
|
||||
expect(result.mutated).toBe(true);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('never removes an unexpected tag that appears during execution', async () => {
|
||||
const img = { Id: 'img-multi', RepoTags: ['qa1769/app:v0'], Size: 50, Containers: 0 };
|
||||
const removes = mockImageRemove();
|
||||
const result = await executeForcedFreshImagePlan([
|
||||
[img],
|
||||
[img],
|
||||
[{ ...img, RepoTags: ['qa1769-external/keep:v0'] }],
|
||||
]);
|
||||
|
||||
expect(removes.map((entry) => entry.name)).toEqual(['qa1769/app:v0']);
|
||||
expect(removes.map((entry) => entry.name)).not.toContain('qa1769-external/keep:v0');
|
||||
expect(result.outcomes[0]).toEqual(expect.objectContaining({
|
||||
status: 'failed',
|
||||
error: expect.stringMatching(/not in the reviewed set/i),
|
||||
}));
|
||||
expect(result.mutated).toBe(true);
|
||||
});
|
||||
|
||||
it('removes a dangling planned image by id with force false', async () => {
|
||||
const img = { Id: 'img-dang', RepoTags: ['<none>:<none>'], Size: 20, Containers: 0 };
|
||||
const removes = mockImageRemove();
|
||||
const result = await executeForcedFreshImagePlan([[img], [img], [img]]);
|
||||
|
||||
expect(removes).toEqual([{ name: 'img-dang', force: false }]);
|
||||
expect(result.outcomes[0]).toEqual(expect.objectContaining({ status: 'removed' }));
|
||||
expect(result.mutated).toBe(true);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('id-removes leftover none tags after reviewed names are gone', async () => {
|
||||
const img = { Id: 'img-multi', RepoTags: ['qa1769/app:v0'], Size: 50, Containers: 0 };
|
||||
const removes = mockImageRemove();
|
||||
const result = await executeForcedFreshImagePlan([
|
||||
[img],
|
||||
[img],
|
||||
[{ ...img, RepoTags: ['<none>:<none>'] }],
|
||||
]);
|
||||
|
||||
expect(removes).toEqual([
|
||||
{ name: 'qa1769/app:v0', force: false },
|
||||
{ name: 'img-multi', force: false },
|
||||
]);
|
||||
expect(result.outcomes[0]).toEqual(expect.objectContaining({ status: 'removed' }));
|
||||
expect(result.mutated).toBe(true);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('does not treat a 409 as missing when the conflict text contains 404', async () => {
|
||||
const img = {
|
||||
Id: 'sha256:abc404def',
|
||||
RepoTags: ['qa1769/app:v0', 'qa1769/app:latest'],
|
||||
Size: 50,
|
||||
Containers: 0,
|
||||
};
|
||||
const removes = mockImageRemove(async () => {
|
||||
throw Object.assign(
|
||||
new Error('(HTTP code 409) conflict - image sha256:abc404def is being used by running container'),
|
||||
{ statusCode: 409 },
|
||||
);
|
||||
});
|
||||
const result = await executeForcedFreshImagePlan([[img], [img], [img]]);
|
||||
|
||||
expect(removes).toEqual([{ name: 'qa1769/app:v0', force: false }]);
|
||||
expect(result.outcomes[0]).toEqual(expect.objectContaining({
|
||||
status: 'failed',
|
||||
error: expect.stringMatching(/qa1769\/app:v0/),
|
||||
}));
|
||||
expect(result.mutated).toBe(false);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('does not report dangling id-remove 409 as removed when the message contains 404', async () => {
|
||||
const img = { Id: 'sha256:abc404def', RepoTags: ['<none>:<none>'], Size: 20, Containers: 0 };
|
||||
mockImageRemove(async () => {
|
||||
throw Object.assign(
|
||||
new Error('(HTTP code 409) conflict - unable to delete sha256:abc404def (must be forced)'),
|
||||
{ statusCode: 409 },
|
||||
);
|
||||
});
|
||||
const result = await executeForcedFreshImagePlan([[img], [img], [img]]);
|
||||
|
||||
expect(result.outcomes[0]).toEqual(expect.objectContaining({
|
||||
status: 'failed',
|
||||
error: expect.stringMatching(/dangling image could not be removed/i),
|
||||
}));
|
||||
expect(result.mutated).toBe(false);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('fails honestly when completion re-list throws after a successful untag', async () => {
|
||||
const img = { Id: 'img-multi', RepoTags: ['qa1769/app:v0'], Size: 50, Containers: 0 };
|
||||
mockImageRemove();
|
||||
const result = await executeForcedFreshImagePlan([[img], [img], new Error('daemon busy')]);
|
||||
|
||||
expect(result.outcomes[0]).toEqual(expect.objectContaining({
|
||||
status: 'failed',
|
||||
error: expect.stringMatching(/could not confirm remaining/i),
|
||||
}));
|
||||
expect(result.mutated).toBe(true);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -174,6 +174,7 @@ describe('Prune plan routes', () => {
|
||||
outcomes: [{ id: 'v1', target: 'volumes', status: 'removed', sizeBytes: 42 }],
|
||||
reclaimedBytes: 42,
|
||||
success: true,
|
||||
mutated: true,
|
||||
});
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
buildPrunePlan: vi.fn().mockResolvedValue(plan),
|
||||
@@ -190,6 +191,7 @@ describe('Prune plan routes', () => {
|
||||
expect(res.body.success).toBe(true);
|
||||
expect(res.body.reclaimedBytes).toBe(42);
|
||||
expect(res.body.outcomes).toHaveLength(1);
|
||||
expect(res.body).not.toHaveProperty('mutated');
|
||||
expect(executePrunePlan).toHaveBeenCalled();
|
||||
expect(invalidate).toHaveBeenCalledWith('stats:1');
|
||||
expect(invalidate).toHaveBeenCalledWith('stack-statuses:1');
|
||||
@@ -268,4 +270,80 @@ describe('Prune plan routes', () => {
|
||||
expect(res.body.items).toHaveLength(1);
|
||||
expect(executePrunePlan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('invalidates node caches when a failed image outcome is mutated', async () => {
|
||||
stubFsStacks();
|
||||
const plan = samplePlan('fp-partial');
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
buildPrunePlan: vi.fn().mockResolvedValue(plan),
|
||||
executePrunePlan: vi.fn().mockResolvedValue({
|
||||
outcomes: [{ id: 'img-multi', target: 'images', status: 'failed', error: 'tags remain' }],
|
||||
reclaimedBytes: 0,
|
||||
success: false,
|
||||
mutated: true,
|
||||
}),
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
const invalidate = vi.spyOn(CacheService.getInstance(), 'invalidate');
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/system/prune/system')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ target: 'volumes', scope: 'managed', planFingerprint: 'fp-partial' });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.success).toBe(false);
|
||||
expect(res.body.outcomes[0].status).toBe('failed');
|
||||
expect(invalidate).toHaveBeenCalledWith('stats:1');
|
||||
expect(invalidate).toHaveBeenCalledWith('stack-statuses:1');
|
||||
});
|
||||
|
||||
it('does not invalidate node caches when a failed outcome is not mutated', async () => {
|
||||
stubFsStacks();
|
||||
const plan = samplePlan('fp-nomut');
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
buildPrunePlan: vi.fn().mockResolvedValue(plan),
|
||||
executePrunePlan: vi.fn().mockResolvedValue({
|
||||
outcomes: [{ id: 'img-multi', target: 'images', status: 'failed', error: 'references changed' }],
|
||||
reclaimedBytes: 0,
|
||||
success: false,
|
||||
mutated: false,
|
||||
}),
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
const invalidate = vi.spyOn(CacheService.getInstance(), 'invalidate');
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/system/prune/system')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ target: 'volumes', scope: 'managed', planFingerprint: 'fp-nomut' });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(invalidate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('legacy no-fingerprint containers prune omits mutated from the JSON body', async () => {
|
||||
stubFsStacks();
|
||||
const executePrunePlan = vi.fn().mockResolvedValue({
|
||||
outcomes: [{ id: 'c1', target: 'containers', status: 'removed' }],
|
||||
reclaimedBytes: 0,
|
||||
success: true,
|
||||
mutated: true,
|
||||
});
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
buildPrunePlan: vi.fn().mockResolvedValue(samplePlan('unused')),
|
||||
executePrunePlan,
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/system/prune/system')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ target: 'containers', scope: 'managed' });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.success).toBe(true);
|
||||
expect(res.body.reclaimedBytes).toBe(0);
|
||||
expect(res.body.outcomes).toEqual([
|
||||
{ id: 'c1', target: 'containers', status: 'removed' },
|
||||
]);
|
||||
expect(res.body).not.toHaveProperty('mutated');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -427,7 +427,7 @@ async function executeLocal(entry: Preflight, targets: FleetPruneTarget[]): Prom
|
||||
const knownStacks = await FileSystemService.getInstance(entry.node.id).getStacks();
|
||||
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(entry.node.id);
|
||||
const result = await DockerController.getInstance(entry.node.id).executePrunePlan(plan, knownStacks, isImageHeld);
|
||||
if (result.outcomes.some((outcome) => outcome.status === 'removed')) {
|
||||
if (result.mutated) {
|
||||
try {
|
||||
invalidateNodeCaches(entry.node.id);
|
||||
} catch (error) {
|
||||
|
||||
@@ -6,7 +6,7 @@ import DockerController, {
|
||||
type PruneScope,
|
||||
type PruneTarget,
|
||||
} from '../services/DockerController';
|
||||
import { isPruneTarget } from '../services/prunePlan';
|
||||
import { isPruneTarget, type PruneItemOutcome } from '../services/prunePlan';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
import { StackUpdateRecoveryService, shortGenerationId } from '../services/StackUpdateRecoveryService';
|
||||
@@ -250,7 +250,7 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
|
||||
success: result.success,
|
||||
});
|
||||
}
|
||||
if (result.outcomes.some((outcome) => outcome.status === 'removed')) {
|
||||
if (result.mutated) {
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
}
|
||||
res.json({
|
||||
@@ -271,7 +271,7 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
|
||||
const target = targets[0];
|
||||
console.log(`[Resources] System prune: ${sanitizeForLog(target)} (scope: ${pruneScope})`);
|
||||
const pruneStartedAt = Date.now();
|
||||
let result: { success: boolean; reclaimedBytes: number };
|
||||
let result: { success: boolean; reclaimedBytes: number; outcomes?: PruneItemOutcome[] };
|
||||
if (pruneScope === 'managed' && target !== 'containers') {
|
||||
result = await dockerController.pruneManagedOnly(
|
||||
target as 'images' | 'volumes' | 'networks',
|
||||
@@ -296,7 +296,12 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
|
||||
if (target === 'containers') {
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
}
|
||||
res.json({ message: 'Prune completed', ...result });
|
||||
res.json({
|
||||
message: 'Prune completed',
|
||||
success: result.success,
|
||||
reclaimedBytes: result.reclaimedBytes,
|
||||
...(result.outcomes !== undefined ? { outcomes: result.outcomes } : {}),
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof TimeoutError) {
|
||||
console.warn('System prune: docker disk usage timed out');
|
||||
|
||||
@@ -12,6 +12,7 @@ import { FileSystemService } from './FileSystemService';
|
||||
import SelfIdentityService from './SelfIdentityService';
|
||||
import {
|
||||
fingerprintPrunePlan,
|
||||
normalizePruneImageReferences,
|
||||
normalizePruneTargets,
|
||||
projectPruneOwnershipLabels,
|
||||
pruneImageReferencesEqual,
|
||||
@@ -1019,7 +1020,8 @@ class DockerController {
|
||||
|
||||
/**
|
||||
* Build an itemized prune plan using the same eligibility predicates that
|
||||
* `executePrunePlan` revalidates before each delete. Never calls remove APIs.
|
||||
* `executePrunePlan` revalidates against live tags before mutation. Never
|
||||
* calls remove APIs.
|
||||
*/
|
||||
public async buildPrunePlan(
|
||||
targets: PruneTarget[],
|
||||
@@ -1214,6 +1216,7 @@ class DockerController {
|
||||
for (const img of rawImages) {
|
||||
if (selfIdentity.isOwnImage(img.Id)) continue;
|
||||
if (isImageHeld?.(img.Id)) continue;
|
||||
if (isFullySyntheticHoldImage(img.RepoTags ?? [])) continue;
|
||||
const refs = imageToContainerIds.get(img.Id) ?? [];
|
||||
const becomesFree = freeingImages
|
||||
&& refs.length > 0
|
||||
@@ -1311,14 +1314,16 @@ class DockerController {
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a previously previewed plan: revalidate fingerprint, then delete
|
||||
* each planned item serially with force:false. Never deletes unplanned items.
|
||||
* Execute a previously previewed plan: revalidate fingerprint, then apply each
|
||||
* planned item serially with force:false. Images untag reviewed names, then
|
||||
* re-list to decide removed, remaining-tag failure, or dangling ID-remove.
|
||||
* Never mutates unplanned items.
|
||||
*/
|
||||
public async executePrunePlan(
|
||||
plan: PrunePlan,
|
||||
knownStackNames: string[],
|
||||
isImageHeld?: (imageId: string) => boolean,
|
||||
): Promise<{ outcomes: PruneItemOutcome[]; reclaimedBytes: number; success: boolean }> {
|
||||
): Promise<{ outcomes: PruneItemOutcome[]; reclaimedBytes: number; success: boolean; mutated: boolean }> {
|
||||
const fresh = await this.assertPlanFresh(plan, knownStackNames, isImageHeld);
|
||||
if (!fresh) throw new PrunePlanStaleError();
|
||||
|
||||
@@ -1330,6 +1335,7 @@ class DockerController {
|
||||
|
||||
const outcomes: PruneItemOutcome[] = [];
|
||||
let reclaimedBytes = 0;
|
||||
let planMutated = false;
|
||||
/** Image IDs whose planned container removal failed or was skipped. */
|
||||
const blockedImageIds = new Set<string>();
|
||||
|
||||
@@ -1415,10 +1421,11 @@ class DockerController {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const outcome = await this.executePlannedImage(
|
||||
const { outcome, mutated: imageMutated } = await this.executePlannedImage(
|
||||
item, selfIdentity,
|
||||
);
|
||||
outcomes.push(outcome);
|
||||
if (imageMutated) planMutated = true;
|
||||
if (outcome.status === 'removed') reclaimedBytes += outcome.sizeBytes ?? item.sizeBytes ?? 0;
|
||||
}
|
||||
} catch (e) {
|
||||
@@ -1433,7 +1440,8 @@ class DockerController {
|
||||
}
|
||||
|
||||
const success = outcomes.every((o) => o.status !== 'failed');
|
||||
return { outcomes, reclaimedBytes, success };
|
||||
const mutated = planMutated || outcomes.some((outcome) => outcome.status === 'removed');
|
||||
return { outcomes, reclaimedBytes, success, mutated };
|
||||
}
|
||||
|
||||
private async lookupContainerImageId(containerId: string): Promise<string | null> {
|
||||
@@ -1588,35 +1596,66 @@ class DockerController {
|
||||
return { id: item.id, target: 'networks', status: 'removed' };
|
||||
}
|
||||
|
||||
private static dockerIsNotFound(error: unknown): boolean {
|
||||
const statusCode = (error as { statusCode?: number }).statusCode;
|
||||
if (statusCode === 404) return true;
|
||||
if (typeof statusCode === 'number') return false;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return /no such image/i.test(message);
|
||||
}
|
||||
|
||||
private static listedImageMatch<T extends { Id: string }>(images: T[], id: string): T | undefined {
|
||||
return images.find((image) => image.Id === id || image.Id.startsWith(id) || id.startsWith(image.Id));
|
||||
}
|
||||
|
||||
private static plannedImageResult(
|
||||
outcome: PruneItemOutcome,
|
||||
mutated = false,
|
||||
): { outcome: PruneItemOutcome; mutated: boolean } {
|
||||
return { outcome, mutated };
|
||||
}
|
||||
|
||||
private async listedImageById(id: string): Promise<{ Id: string; RepoTags?: string[] | null } | undefined> {
|
||||
const images = await this.docker.listImages({ all: false }) as Array<{
|
||||
Id: string;
|
||||
RepoTags?: string[] | null;
|
||||
}>;
|
||||
return DockerController.listedImageMatch(images, id);
|
||||
}
|
||||
|
||||
private async executePlannedImage(
|
||||
item: PrunePlanItem,
|
||||
selfIdentity: SelfIdentityService,
|
||||
): Promise<PruneItemOutcome> {
|
||||
): Promise<{ outcome: PruneItemOutcome; mutated: boolean }> {
|
||||
if (selfIdentity.isOwnImage(item.id)) {
|
||||
return { id: item.id, target: 'images', status: 'skipped', reason: 'Sencho self image' };
|
||||
return DockerController.plannedImageResult({
|
||||
id: item.id, target: 'images', status: 'skipped', reason: 'Sencho self image',
|
||||
});
|
||||
}
|
||||
if (item.target !== 'images') {
|
||||
return { id: item.id, target: 'images', status: 'failed', error: 'Plan item is not an image target' };
|
||||
return DockerController.plannedImageResult({
|
||||
id: item.id, target: 'images', status: 'failed', error: 'Plan item is not an image target',
|
||||
});
|
||||
}
|
||||
const plannedRefs = item.image.references;
|
||||
const rawImages = await this.docker.listImages({ all: false }) as Array<{
|
||||
Id: string;
|
||||
Size?: number;
|
||||
RepoTags?: string[] | null;
|
||||
}>;
|
||||
const img = rawImages.find((i) => i.Id === item.id || i.Id.startsWith(item.id) || item.id.startsWith(i.Id));
|
||||
const img = await this.listedImageById(item.id);
|
||||
if (!img) {
|
||||
return { id: item.id, target: 'images', status: 'skipped', reason: 'Image no longer exists' };
|
||||
return DockerController.plannedImageResult({
|
||||
id: item.id, target: 'images', status: 'skipped', reason: 'Image no longer exists',
|
||||
});
|
||||
}
|
||||
if (isFullySyntheticHoldImage(img.RepoTags ?? [])) {
|
||||
return DockerController.plannedImageResult({
|
||||
id: item.id, target: 'images', status: 'skipped', reason: 'Sencho rollback-hold image',
|
||||
});
|
||||
}
|
||||
// Defense in depth when fleet preflight/assertPlanFresh already matched:
|
||||
// refuse mutation if live tags drifted from the reviewed reference set.
|
||||
if (!pruneImageReferencesEqual(plannedRefs, img.RepoTags)) {
|
||||
return {
|
||||
return DockerController.plannedImageResult({
|
||||
id: item.id,
|
||||
target: 'images',
|
||||
status: 'failed',
|
||||
error: 'Image references changed since the plan was built; refresh and confirm again',
|
||||
};
|
||||
});
|
||||
}
|
||||
const allContainers = await this.docker.listContainers({ all: true }) as Array<{
|
||||
ImageID?: string;
|
||||
@@ -1626,51 +1665,87 @@ class DockerController {
|
||||
|| Boolean(container.ImageID?.startsWith(img.Id))
|
||||
|| img.Id.startsWith(container.ImageID ?? ''));
|
||||
if (containerRefs.length > 0) {
|
||||
return { id: item.id, target: 'images', status: 'skipped', reason: 'Image still has container references' };
|
||||
return DockerController.plannedImageResult({
|
||||
id: item.id, target: 'images', status: 'skipped', reason: 'Image still has container references',
|
||||
});
|
||||
}
|
||||
|
||||
let mutated = false;
|
||||
for (const ref of plannedRefs) {
|
||||
try {
|
||||
await this.docker.getImage(ref).remove({ force: false });
|
||||
mutated = true;
|
||||
} catch (error) {
|
||||
if (!DockerController.dockerIsNotFound(error)) break;
|
||||
}
|
||||
}
|
||||
return this.completePlannedImage(item, plannedRefs, mutated);
|
||||
}
|
||||
|
||||
private static remainingImageFailure(
|
||||
item: PrunePlanItem,
|
||||
plannedRefs: string[],
|
||||
remaining: string[],
|
||||
): PruneItemOutcome {
|
||||
const prefix = remaining.some((ref) => !plannedRefs.includes(ref))
|
||||
? 'Image still has repository tags that were not in the reviewed set: '
|
||||
: 'Image still has repository tags after prune: ';
|
||||
return {
|
||||
id: item.id,
|
||||
target: 'images',
|
||||
status: 'failed',
|
||||
error: prefix + remaining.join(', ')
|
||||
+ '. Refresh the plan and confirm remaining references before pruning again.',
|
||||
};
|
||||
}
|
||||
|
||||
private async completePlannedImage(
|
||||
item: PrunePlanItem,
|
||||
plannedRefs: string[],
|
||||
mutated: boolean,
|
||||
): Promise<{ outcome: PruneItemOutcome; mutated: boolean }> {
|
||||
const removed = DockerController.plannedImageResult(
|
||||
{ id: item.id, target: 'images', status: 'removed', sizeBytes: item.sizeBytes ?? 0 },
|
||||
true,
|
||||
);
|
||||
let listed: { Id: string; RepoTags?: string[] | null } | undefined;
|
||||
try {
|
||||
listed = await this.listedImageById(item.id);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(
|
||||
`[executePrunePlan] Could not confirm remaining tags for ${sanitizeForLog(item.id)}:`,
|
||||
sanitizeForLog(message),
|
||||
);
|
||||
return DockerController.plannedImageResult({
|
||||
id: item.id,
|
||||
target: 'images',
|
||||
status: 'failed',
|
||||
error: 'Could not confirm remaining image tags after prune. '
|
||||
+ 'Refresh the plan and confirm remaining references before pruning again.',
|
||||
}, mutated);
|
||||
}
|
||||
if (!listed) return removed;
|
||||
const remaining = normalizePruneImageReferences(listed.RepoTags);
|
||||
if (remaining.length > 0) {
|
||||
return DockerController.plannedImageResult(
|
||||
DockerController.remainingImageFailure(item, plannedRefs, remaining),
|
||||
mutated,
|
||||
);
|
||||
}
|
||||
try {
|
||||
await this.docker.getImage(item.id).remove({ force: false });
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : String(e);
|
||||
// Re-read tags so a multi-repository conflict that partially untagged is not
|
||||
// reported as a no-op when live names already diverged.
|
||||
let afterTags: string[] | null | undefined;
|
||||
try {
|
||||
const afterList = await this.docker.listImages({ all: false }) as Array<{
|
||||
Id: string;
|
||||
RepoTags?: string[] | null;
|
||||
}>;
|
||||
afterTags = afterList.find((i) => i.Id === img.Id || i.Id.startsWith(img.Id) || img.Id.startsWith(i.Id))
|
||||
?.RepoTags;
|
||||
} catch {
|
||||
afterTags = undefined;
|
||||
}
|
||||
if (afterTags !== undefined && !pruneImageReferencesEqual(plannedRefs, afterTags)) {
|
||||
return {
|
||||
id: item.id,
|
||||
target: 'images',
|
||||
status: 'failed',
|
||||
error: 'Image delete did not fully remove the reviewed image; some repository tags changed. '
|
||||
+ 'Refresh the plan and confirm remaining references before pruning again. '
|
||||
+ message,
|
||||
};
|
||||
}
|
||||
const lower = message.toLowerCase();
|
||||
if (lower.includes('multiple repositories') || lower.includes('must be forced') || lower.includes('conflict')) {
|
||||
return {
|
||||
id: item.id,
|
||||
target: 'images',
|
||||
status: 'failed',
|
||||
error: 'Docker refused to delete this image because it is tagged in multiple repositories. '
|
||||
+ 'Reviewed tags: ' + (plannedRefs.join(', ') || '(none)') + '. '
|
||||
+ 'Resolve tags on the host or re-scope the plan, then dry-run again. '
|
||||
+ message,
|
||||
};
|
||||
}
|
||||
throw e;
|
||||
return removed;
|
||||
} catch (error) {
|
||||
if (DockerController.dockerIsNotFound(error)) return removed;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return DockerController.plannedImageResult({
|
||||
id: item.id,
|
||||
target: 'images',
|
||||
status: 'failed',
|
||||
error: 'Dangling image could not be removed after reviewed tags were cleared. ' + message,
|
||||
}, mutated);
|
||||
}
|
||||
// Prefer plan unique-bytes; do not fall back to full Size (shared layers).
|
||||
return { id: item.id, target: 'images', status: 'removed', sizeBytes: item.sizeBytes ?? 0 };
|
||||
}
|
||||
|
||||
public async getDiskUsageClassified(knownStackNames: string[]): Promise<{
|
||||
|
||||
Reference in New Issue
Block a user