From b4abd23fa597a3ed03f75cfd66a33dc8faafddd7 Mon Sep 17 00:00:00 2001 From: aperolschpritz Date: Wed, 23 Sep 2026 18:00:40 +0000 Subject: [PATCH] fix: seed processed build jobs on watch start to prevent redeploy on restart --- .../build-watch.service.ts | 38 +++++++++- .../build-watch.service.unit.spec.ts | 71 ++++++++++++++++++- 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/src/server/services/standalone-services/build-watch.service.ts b/src/server/services/standalone-services/build-watch.service.ts index 86d21a81..642a151c 100644 --- a/src/server/services/standalone-services/build-watch.service.ts +++ b/src/server/services/standalone-services/build-watch.service.ts @@ -38,6 +38,14 @@ class BuildWatchService { console.error('[BuildWatch] Failed to seed build statuses:', error); } + try { + await this.seedProcessedJobs(); + } catch (error) { + // Without a successful seed a restarted watch would replay every + // existing build job as "ADDED" and redeploy old builds. + console.error('[BuildWatch] Failed to seed existing build jobs:', error); + } + const kc = k3s.getKubeConfig(); const watch = new k8s.Watch(kc); @@ -51,7 +59,7 @@ class BuildWatchService { } catch (e) { console.error('[BuildWatch] Status update failed:', e); } - await this.handleJobEvent(job); + await this.handleJobEvent(type, job); }, (err: unknown) => { if (err) console.error('[BuildWatch] Watch error:', err); @@ -62,7 +70,33 @@ class BuildWatchService { ); } - private async handleJobEvent(job: V1Job) { + /** + * Marks already finished build jobs as processed before the watch starts. + * + * A watch without a resourceVersion first replays every existing job as a + * synthetic ADDED event ("Get State and Start at Most Recent"). Without this + * seed, a restart would redeploy old successful builds, and because jobs are + * replayed in list order an older build could win over the newest one. + * + * Running and pending jobs stay unseeded so they still deploy on completion. + */ + private async seedProcessedJobs() { + const jobs = await k3s.batch.listNamespacedJob({ namespace: BUILD_NAMESPACE }); + for (const job of jobs.items ?? []) { + const jobName = job.metadata?.name; + if (!jobName) continue; + + const status = buildService.getJobStatusString(job.status); + if (status === 'SUCCEEDED' || status === 'FAILED') { + this.processedJobs.add(jobName); + } + } + console.log('[BuildWatch] Seeded existing build jobs.'); + } + + private async handleJobEvent(type: string, job: V1Job) { + if (type === 'DELETED') return; + const jobName = job.metadata?.name; if (!jobName || this.processedJobs.has(jobName)) return; diff --git a/src/server/services/standalone-services/build-watch.service.unit.spec.ts b/src/server/services/standalone-services/build-watch.service.unit.spec.ts index 92d346a8..e5be4d53 100644 --- a/src/server/services/standalone-services/build-watch.service.unit.spec.ts +++ b/src/server/services/standalone-services/build-watch.service.unit.spec.ts @@ -1,6 +1,7 @@ const k8sMocks = vi.hoisted(() => ({ watch: vi.fn(), abort: vi.fn(), + listNamespacedJob: vi.fn(), })); vi.mock('@kubernetes/client-node', async () => { @@ -18,7 +19,7 @@ vi.mock('@/server/adapter/kubernetes-api.adapter', () => ({ default: { getKubeConfig: vi.fn(), batch: { - listNamespacedJob: vi.fn().mockResolvedValue({ body: { items: [] } }), + listNamespacedJob: k8sMocks.listNamespacedJob, }, }, })); @@ -71,6 +72,7 @@ describe('BuildWatchService', () => { (buildWatchService as any).processedJobs.clear(); (buildWatchService as any).isWatchRunning = false; k8sMocks.watch.mockResolvedValue({ abort: k8sMocks.abort }); + k8sMocks.listNamespacedJob.mockResolvedValue({ items: [] }); }); it('seeds the build status service and forwards job events to it', async () => { @@ -111,7 +113,7 @@ describe('BuildWatchService', () => { it('ignores pending jobs and does not trigger deployment work', async () => { vi.mocked(buildService.getJobStatusString).mockReturnValue('PENDING'); - await (buildWatchService as any).handleJobEvent({ + await (buildWatchService as any).handleJobEvent('MODIFIED', { metadata: { name: 'build-1', annotations: { @@ -173,4 +175,69 @@ describe('BuildWatchService', () => { ); expect(appGitSshKeyService.deleteTemporaryBuildSecret).toHaveBeenCalledWith('git-ssh-build-1'); }); + + it('does not redeploy an old successful build replayed as ADDED after startup', async () => { + vi.mocked(buildService.getJobStatusString).mockReturnValue('SUCCEEDED'); + k8sMocks.listNamespacedJob.mockResolvedValue({ + items: [{ metadata: { name: 'old-build' }, status: { succeeded: 1 } }], + }); + vi.mocked(appService.getExtendedById).mockResolvedValue({ buildMethod: 'RAILPACK' } as any); + + await buildWatchService.startWatch(); + const eventHandler = k8sMocks.watch.mock.calls[0][2] as (type: string, job: unknown) => Promise; + await eventHandler('ADDED', { + metadata: { + name: 'old-build', + annotations: { + 'qs-deplyoment-id': 'deployment-1', + 'qs-app-id': 'app-1', + }, + }, + }); + + expect(deploymentService.createDeployment).not.toHaveBeenCalled(); + }); + + it('still deploys a running build once it succeeds after startup', async () => { + vi.mocked(buildService.getJobStatusString).mockImplementation((status?: any) => { + if (status?.succeeded) return 'SUCCEEDED'; + if (status?.active) return 'PENDING'; + return 'UNKNOWN'; + }); + k8sMocks.listNamespacedJob.mockResolvedValue({ + items: [{ metadata: { name: 'running-build' }, status: { active: 1 } }], + }); + vi.mocked(appService.getExtendedById).mockResolvedValue({ buildMethod: 'RAILPACK' } as any); + + await buildWatchService.startWatch(); + const eventHandler = k8sMocks.watch.mock.calls[0][2] as (type: string, job: unknown) => Promise; + await eventHandler('MODIFIED', { + metadata: { + name: 'running-build', + annotations: { + 'qs-deplyoment-id': 'deployment-1', + 'qs-app-id': 'app-1', + }, + }, + status: { succeeded: 1 }, + }); + + expect(deploymentService.createDeployment).toHaveBeenCalledTimes(1); + }); + + it('ignores DELETED job events even when the job shows as succeeded', async () => { + vi.mocked(buildService.getJobStatusString).mockReturnValue('SUCCEEDED'); + + await (buildWatchService as any).handleJobEvent('DELETED', { + metadata: { + name: 'build-1', + annotations: { + 'qs-deplyoment-id': 'deployment-1', + 'qs-app-id': 'app-1', + }, + }, + }); + + expect(deploymentService.createDeployment).not.toHaveBeenCalled(); + }); });