From de692fb1e51d43539d8ab6f1d3f7dd6bf9e4c007 Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Fri, 14 Aug 2026 02:52:00 +0200 Subject: [PATCH 1/6] feat(debezium)!: consolidate cdc capture on the outbox table --- .../debezium/configurations/entrypoint.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/development/debezium/configurations/entrypoint.sh b/src/development/debezium/configurations/entrypoint.sh index 4a61818f..32946a79 100755 --- a/src/development/debezium/configurations/entrypoint.sh +++ b/src/development/debezium/configurations/entrypoint.sh @@ -27,8 +27,18 @@ curl --fail --output /dev/null --silent --show-error \ "database.password": "'"$POSTGRES_PASSWORD"'", "database.user": "'"$POSTGRES_USER"'", "plugin.name": "pgoutput", - "table.include.list": "vibetype.event,vibetype.upload,vibetype_private.notification", - "topic.prefix" : "vibetype" + "table.include.list": "vibetype_private.outbox,vibetype.upload", + "topic.prefix" : "vibetype", + "transforms": "outbox", + "transforms.outbox.type": "io.debezium.transforms.outbox.EventRouter", + "transforms.outbox.route.by.field": "channel", + "transforms.outbox.route.topic.replacement": "vibetype.outbox.${routedByValue}", + "transforms.outbox.table.field.event.key": "id", + "transforms.outbox.table.field.event.payload": "payload", + "transforms.outbox.predicate": "isOutbox", + "predicates": "isOutbox", + "predicates.isOutbox.type": "org.apache.kafka.connect.transforms.predicates.TopicNameMatches", + "predicates.isOutbox.pattern": "vibetype\\.vibetype_private\\.outbox" }' echo "PostgreSQL connector '$CONNECTOR_NAME' is up to date." From ac230ce701a92029352c89c22406b5f215ea62fb Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Fri, 14 Aug 2026 03:29:00 +0200 Subject: [PATCH 2/6] feat(debezium)!: drop direct upload capture now that it routes through outbox --- src/development/debezium/configurations/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/development/debezium/configurations/entrypoint.sh b/src/development/debezium/configurations/entrypoint.sh index 32946a79..caa79593 100755 --- a/src/development/debezium/configurations/entrypoint.sh +++ b/src/development/debezium/configurations/entrypoint.sh @@ -27,7 +27,7 @@ curl --fail --output /dev/null --silent --show-error \ "database.password": "'"$POSTGRES_PASSWORD"'", "database.user": "'"$POSTGRES_USER"'", "plugin.name": "pgoutput", - "table.include.list": "vibetype_private.outbox,vibetype.upload", + "table.include.list": "vibetype_private.outbox", "topic.prefix" : "vibetype", "transforms": "outbox", "transforms.outbox.type": "io.debezium.transforms.outbox.EventRouter", From f146435f8abba1e825a6514b2526db028cc305bb Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Fri, 14 Aug 2026 04:07:00 +0200 Subject: [PATCH 3/6] feat(debezium)!: partition outbox topics by aggregate_id --- src/development/debezium/configurations/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/development/debezium/configurations/entrypoint.sh b/src/development/debezium/configurations/entrypoint.sh index caa79593..f15caea6 100755 --- a/src/development/debezium/configurations/entrypoint.sh +++ b/src/development/debezium/configurations/entrypoint.sh @@ -33,7 +33,7 @@ curl --fail --output /dev/null --silent --show-error \ "transforms.outbox.type": "io.debezium.transforms.outbox.EventRouter", "transforms.outbox.route.by.field": "channel", "transforms.outbox.route.topic.replacement": "vibetype.outbox.${routedByValue}", - "transforms.outbox.table.field.event.key": "id", + "transforms.outbox.table.field.event.key": "aggregate_id", "transforms.outbox.table.field.event.payload": "payload", "transforms.outbox.predicate": "isOutbox", "predicates": "isOutbox", From 7393b60d7d46d0636787846396760335ddf62175 Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Fri, 14 Aug 2026 04:44:00 +0200 Subject: [PATCH 4/6] feat(debezium)!: route outbox topics by aggregate_type instead of type --- src/development/debezium/configurations/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/development/debezium/configurations/entrypoint.sh b/src/development/debezium/configurations/entrypoint.sh index f15caea6..d9c6643e 100755 --- a/src/development/debezium/configurations/entrypoint.sh +++ b/src/development/debezium/configurations/entrypoint.sh @@ -31,7 +31,7 @@ curl --fail --output /dev/null --silent --show-error \ "topic.prefix" : "vibetype", "transforms": "outbox", "transforms.outbox.type": "io.debezium.transforms.outbox.EventRouter", - "transforms.outbox.route.by.field": "channel", + "transforms.outbox.route.by.field": "aggregate_type", "transforms.outbox.route.topic.replacement": "vibetype.outbox.${routedByValue}", "transforms.outbox.table.field.event.key": "aggregate_id", "transforms.outbox.table.field.event.payload": "payload", From c5a505c500987c68ee549bda6d1d55a8ec984d93 Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Tue, 18 Aug 2026 18:52:00 +0200 Subject: [PATCH 5/6] chore(debezium): remove outbox payload field config matching Debezium default --- src/development/debezium/configurations/entrypoint.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/development/debezium/configurations/entrypoint.sh b/src/development/debezium/configurations/entrypoint.sh index d9c6643e..45973796 100755 --- a/src/development/debezium/configurations/entrypoint.sh +++ b/src/development/debezium/configurations/entrypoint.sh @@ -34,7 +34,6 @@ curl --fail --output /dev/null --silent --show-error \ "transforms.outbox.route.by.field": "aggregate_type", "transforms.outbox.route.topic.replacement": "vibetype.outbox.${routedByValue}", "transforms.outbox.table.field.event.key": "aggregate_id", - "transforms.outbox.table.field.event.payload": "payload", "transforms.outbox.predicate": "isOutbox", "predicates": "isOutbox", "predicates.isOutbox.type": "org.apache.kafka.connect.transforms.predicates.TopicNameMatches", From 83de788c2acec5e5605d2f5440014613f6c3e0e6 Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Tue, 18 Aug 2026 19:00:00 +0200 Subject: [PATCH 6/6] feat(jobber): add outbox purge job --- src/development/jobber/compose.yaml | 5 +++++ src/development/jobber/configurations/.jobber | 9 +++++++++ .../sinks/sentry-outbox-purge/error.sh | 3 +++ .../sinks/sentry-outbox-purge/in-progress.sh | 3 +++ .../configurations/sinks/sentry-outbox-purge/ok.sh | 3 +++ src/development/postgres/compose.yaml | 11 +++++++++++ src/development/sqitch/compose.yaml | 2 ++ src/production/.env.template | 1 + src/production/jobber/compose.yaml | 1 + src/production/jobber/configurations/.jobber | 14 ++++++++++++++ 10 files changed, 52 insertions(+) create mode 100755 src/development/jobber/configurations/sinks/sentry-outbox-purge/error.sh create mode 100755 src/development/jobber/configurations/sinks/sentry-outbox-purge/in-progress.sh create mode 100755 src/development/jobber/configurations/sinks/sentry-outbox-purge/ok.sh diff --git a/src/development/jobber/compose.yaml b/src/development/jobber/compose.yaml index 8b0ff062..e9d131a7 100644 --- a/src/development/jobber/compose.yaml +++ b/src/development/jobber/compose.yaml @@ -19,6 +19,8 @@ services: # You cannot access the jobber via a web interface. environment: AWS_SHARED_CREDENTIALS_FILE: /run/secrets/jobber-aws-credentials + POSTGRES_DB_FILE: /run/secrets/postgres-db + POSTGRES_HOST: postgres image: ghcr.io/dargmuesli/jobber-aws-msmtp:1.3.0 secrets: - source: jobber-aliases @@ -29,6 +31,9 @@ services: target: /home/jobberuser/.aws/config - source: jobber-msmtprc target: /etc/msmtprc + - postgres-db + - postgres-role-service-jobber-password + - postgres-role-service-jobber-username volumes: - ./configurations/.jobber:/home/jobberuser/.jobber:ro - ./configurations/sinks:/srv/sinks:ro diff --git a/src/development/jobber/configurations/.jobber b/src/development/jobber/configurations/.jobber index 906ff841..a824435b 100644 --- a/src/development/jobber/configurations/.jobber +++ b/src/development/jobber/configurations/.jobber @@ -10,3 +10,12 @@ jobs: - type: system-email notifyOnFailure: - type: system-email + OutboxPurge: + cmd: PGPASSWORD="$(cat /run/secrets/postgres-role-service-jobber-password)" psql -v ON_ERROR_STOP=1 -h "$POSTGRES_HOST" -U "$(cat /run/secrets/postgres-role-service-jobber-username)" -d "$(cat "$POSTGRES_DB_FILE")" -c "DELETE FROM vibetype_private.outbox WHERE created_at < now() - interval '24 hours'" + time: 0 0 */2 * + notifyOnSuccess: + - type: system-email + notifyOnError: + - type: system-email + notifyOnFailure: + - type: system-email diff --git a/src/development/jobber/configurations/sinks/sentry-outbox-purge/error.sh b/src/development/jobber/configurations/sinks/sentry-outbox-purge/error.sh new file mode 100755 index 00000000..e769052f --- /dev/null +++ b/src/development/jobber/configurations/sinks/sentry-outbox-purge/error.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +curl "${SENTRY_CRONS_OUTBOX_PURGE}?status=error" diff --git a/src/development/jobber/configurations/sinks/sentry-outbox-purge/in-progress.sh b/src/development/jobber/configurations/sinks/sentry-outbox-purge/in-progress.sh new file mode 100755 index 00000000..8658bb7b --- /dev/null +++ b/src/development/jobber/configurations/sinks/sentry-outbox-purge/in-progress.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +curl "${SENTRY_CRONS_OUTBOX_PURGE}?status=in_progress" diff --git a/src/development/jobber/configurations/sinks/sentry-outbox-purge/ok.sh b/src/development/jobber/configurations/sinks/sentry-outbox-purge/ok.sh new file mode 100755 index 00000000..d17c2407 --- /dev/null +++ b/src/development/jobber/configurations/sinks/sentry-outbox-purge/ok.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +curl "${SENTRY_CRONS_OUTBOX_PURGE}?status=ok" diff --git a/src/development/postgres/compose.yaml b/src/development/postgres/compose.yaml index 546cb761..251be610 100644 --- a/src/development/postgres/compose.yaml +++ b/src/development/postgres/compose.yaml @@ -11,6 +11,12 @@ secrets: postgres-role-service-grafana-username: # The username of the observation dashboard's database role. file: ~~/artifacts/secrets/postgres-role-service-grafana-username.secret + postgres-role-service-jobber-password: + # The password of the job scheduler's database role. + file: ~~/artifacts/secrets/postgres-role-service-jobber-password.secret + postgres-role-service-jobber-username: + # The username of the job scheduler's database role. + file: ~~/artifacts/secrets/postgres-role-service-jobber-username.secret postgres-role-service-postgraphile-password: # The password of the GraphQL API database wrapper's database role. file: ~~/artifacts/secrets/postgres-role-service-postgraphile-password.secret @@ -90,6 +96,11 @@ x-dargstack: type: random_string postgres-role-service-grafana-username: type: wordlist_word + postgres-role-service-jobber-password: + special_characters: false + type: random_string + postgres-role-service-jobber-username: + type: wordlist_word postgres-role-service-postgraphile-password: special_characters: false type: random_string diff --git a/src/development/sqitch/compose.yaml b/src/development/sqitch/compose.yaml index 12499b7b..aab6ca6e 100644 --- a/src/development/sqitch/compose.yaml +++ b/src/development/sqitch/compose.yaml @@ -16,6 +16,8 @@ services: secrets: - postgres-role-service-grafana-password - postgres-role-service-grafana-username + - postgres-role-service-jobber-password + - postgres-role-service-jobber-username - postgres-role-service-postgraphile-password - postgres-role-service-postgraphile-username - postgres-role-service-reccoom-password diff --git a/src/production/.env.template b/src/production/.env.template index 8ffc032b..8934c80b 100644 --- a/src/production/.env.template +++ b/src/production/.env.template @@ -1,5 +1,6 @@ CLOUDFLARED_TUNNEL_TOKEN= SENTRY_CRONS= +SENTRY_CRONS_OUTBOX_PURGE= STACK_DOMAIN= TRAEFIK_ACME_EMAIL= TRAEFIK_ACME_PROVIDER= \ No newline at end of file diff --git a/src/production/jobber/compose.yaml b/src/production/jobber/compose.yaml index 063fbc2f..764b0aa8 100644 --- a/src/production/jobber/compose.yaml +++ b/src/production/jobber/compose.yaml @@ -5,6 +5,7 @@ services: order: stop-first environment: SENTRY_CRONS: ${SENTRY_CRONS} + SENTRY_CRONS_OUTBOX_PURGE: ${SENTRY_CRONS_OUTBOX_PURGE} volumes: - (( append )) - postgres-backup-data:/backups/ diff --git a/src/production/jobber/configurations/.jobber b/src/production/jobber/configurations/.jobber index 162ba232..69266354 100644 --- a/src/production/jobber/configurations/.jobber +++ b/src/production/jobber/configurations/.jobber @@ -18,3 +18,17 @@ jobs: notifyOnFailure: - type: program path: /srv/sinks/sentry/error.sh + OutboxPurge: + cmd: | + /srv/sinks/sentry-outbox-purge/in-progress.sh + PGPASSWORD="$(cat /run/secrets/postgres-role-service-jobber-password)" psql -v ON_ERROR_STOP=1 -h "$POSTGRES_HOST" -U "$(cat /run/secrets/postgres-role-service-jobber-username)" -d "$(cat "$POSTGRES_DB_FILE")" -c "DELETE FROM vibetype_private.outbox WHERE created_at < now() - interval '24 hours'" + time: 0 0 */2 * + notifyOnSuccess: + - type: program + path: /srv/sinks/sentry-outbox-purge/ok.sh + notifyOnError: + - type: program + path: /srv/sinks/sentry-outbox-purge/error.sh + notifyOnFailure: + - type: program + path: /srv/sinks/sentry-outbox-purge/error.sh