Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 36 additions & 2 deletions src/server/services/standalone-services/build-watch.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,14 @@ class BuildWatchService {
console.error('[BuildWatch] Failed to seed build statuses:', error);
}

try {
await this.seedProcessedJobs();
} catch (error) {
// Without a successful seed a restarted watch would replay every
// existing build job as "ADDED" and redeploy old builds.
console.error('[BuildWatch] Failed to seed existing build jobs:', error);
}

const kc = k3s.getKubeConfig();
const watch = new k8s.Watch(kc);

Expand All @@ -51,7 +59,7 @@ class BuildWatchService {
} catch (e) {
console.error('[BuildWatch] Status update failed:', e);
}
await this.handleJobEvent(job);
await this.handleJobEvent(type, job);
},
(err: unknown) => {
if (err) console.error('[BuildWatch] Watch error:', err);
Expand All @@ -62,7 +70,33 @@ class BuildWatchService {
);
}

private async handleJobEvent(job: V1Job) {
/**
* Marks already finished build jobs as processed before the watch starts.
*
* A watch without a resourceVersion first replays every existing job as a
* synthetic ADDED event ("Get State and Start at Most Recent"). Without this
* seed, a restart would redeploy old successful builds, and because jobs are
* replayed in list order an older build could win over the newest one.
*
* Running and pending jobs stay unseeded so they still deploy on completion.
*/
private async seedProcessedJobs() {
const jobs = await k3s.batch.listNamespacedJob({ namespace: BUILD_NAMESPACE });
for (const job of jobs.items ?? []) {
const jobName = job.metadata?.name;
if (!jobName) continue;

const status = buildService.getJobStatusString(job.status);
if (status === 'SUCCEEDED' || status === 'FAILED') {
this.processedJobs.add(jobName);
}
}
console.log('[BuildWatch] Seeded existing build jobs.');
}

private async handleJobEvent(type: string, job: V1Job) {
if (type === 'DELETED') return;

const jobName = job.metadata?.name;
if (!jobName || this.processedJobs.has(jobName)) return;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
const k8sMocks = vi.hoisted(() => ({
watch: vi.fn(),
abort: vi.fn(),
listNamespacedJob: vi.fn(),
}));

vi.mock('@kubernetes/client-node', async () => {
Expand All @@ -18,7 +19,7 @@ vi.mock('@/server/adapter/kubernetes-api.adapter', () => ({
default: {
getKubeConfig: vi.fn(),
batch: {
listNamespacedJob: vi.fn().mockResolvedValue({ body: { items: [] } }),
listNamespacedJob: k8sMocks.listNamespacedJob,
},
},
}));
Expand Down Expand Up @@ -71,6 +72,7 @@ describe('BuildWatchService', () => {
(buildWatchService as any).processedJobs.clear();
(buildWatchService as any).isWatchRunning = false;
k8sMocks.watch.mockResolvedValue({ abort: k8sMocks.abort });
k8sMocks.listNamespacedJob.mockResolvedValue({ items: [] });
});

it('seeds the build status service and forwards job events to it', async () => {
Expand Down Expand Up @@ -111,7 +113,7 @@ describe('BuildWatchService', () => {
it('ignores pending jobs and does not trigger deployment work', async () => {
vi.mocked(buildService.getJobStatusString).mockReturnValue('PENDING');

await (buildWatchService as any).handleJobEvent({
await (buildWatchService as any).handleJobEvent('MODIFIED', {
metadata: {
name: 'build-1',
annotations: {
Expand Down Expand Up @@ -173,4 +175,69 @@ describe('BuildWatchService', () => {
);
expect(appGitSshKeyService.deleteTemporaryBuildSecret).toHaveBeenCalledWith('git-ssh-build-1');
});

it('does not redeploy an old successful build replayed as ADDED after startup', async () => {
vi.mocked(buildService.getJobStatusString).mockReturnValue('SUCCEEDED');
k8sMocks.listNamespacedJob.mockResolvedValue({
items: [{ metadata: { name: 'old-build' }, status: { succeeded: 1 } }],
});
vi.mocked(appService.getExtendedById).mockResolvedValue({ buildMethod: 'RAILPACK' } as any);

await buildWatchService.startWatch();
const eventHandler = k8sMocks.watch.mock.calls[0][2] as (type: string, job: unknown) => Promise<void>;
await eventHandler('ADDED', {
metadata: {
name: 'old-build',
annotations: {
'qs-deplyoment-id': 'deployment-1',
'qs-app-id': 'app-1',
},
},
});

expect(deploymentService.createDeployment).not.toHaveBeenCalled();
});

it('still deploys a running build once it succeeds after startup', async () => {
vi.mocked(buildService.getJobStatusString).mockImplementation((status?: any) => {
if (status?.succeeded) return 'SUCCEEDED';
if (status?.active) return 'PENDING';
return 'UNKNOWN';
});
k8sMocks.listNamespacedJob.mockResolvedValue({
items: [{ metadata: { name: 'running-build' }, status: { active: 1 } }],
});
vi.mocked(appService.getExtendedById).mockResolvedValue({ buildMethod: 'RAILPACK' } as any);

await buildWatchService.startWatch();
const eventHandler = k8sMocks.watch.mock.calls[0][2] as (type: string, job: unknown) => Promise<void>;
await eventHandler('MODIFIED', {
metadata: {
name: 'running-build',
annotations: {
'qs-deplyoment-id': 'deployment-1',
'qs-app-id': 'app-1',
},
},
status: { succeeded: 1 },
});

expect(deploymentService.createDeployment).toHaveBeenCalledTimes(1);
});

it('ignores DELETED job events even when the job shows as succeeded', async () => {
vi.mocked(buildService.getJobStatusString).mockReturnValue('SUCCEEDED');

await (buildWatchService as any).handleJobEvent('DELETED', {
metadata: {
name: 'build-1',
annotations: {
'qs-deplyoment-id': 'deployment-1',
'qs-app-id': 'app-1',
},
},
});

expect(deploymentService.createDeployment).not.toHaveBeenCalled();
});
});
Loading