feat(debezium)!: consolidate cdc capture on the outbox table

src/development/debezium/configurations/entrypoint.sh

@@ -27,8 +27,17 @@ curl --fail --output /dev/null --silent --show-error \
     "database.password": "'"$POSTGRES_PASSWORD"'",
     "database.user": "'"$POSTGRES_USER"'",
     "plugin.name": "pgoutput",
-    "table.include.list": "vibetype.event,vibetype.upload,vibetype_private.notification",
-    "topic.prefix" : "vibetype"
+    "table.include.list": "vibetype_private.outbox",
+    "topic.prefix" : "vibetype",
+    "transforms": "outbox",
+    "transforms.outbox.type": "io.debezium.transforms.outbox.EventRouter",
+    "transforms.outbox.route.by.field": "aggregate_type",
+    "transforms.outbox.route.topic.replacement": "vibetype.outbox.${routedByValue}",
+    "transforms.outbox.table.field.event.key": "aggregate_id",
+    "transforms.outbox.predicate": "isOutbox",
+    "predicates": "isOutbox",
+    "predicates.isOutbox.type": "org.apache.kafka.connect.transforms.predicates.TopicNameMatches",
+    "predicates.isOutbox.pattern": "vibetype\\.vibetype_private\\.outbox"
 }'
 
 echo "PostgreSQL connector '$CONNECTOR_NAME' is up to date."

src/development/jobber/compose.yaml

@@ -19,6 +19,8 @@ services:
     # You cannot access the jobber via a web interface.
     environment:
       AWS_SHARED_CREDENTIALS_FILE: /run/secrets/jobber-aws-credentials
+      POSTGRES_DB_FILE: /run/secrets/postgres-db
+      POSTGRES_HOST: postgres
     image: ghcr.io/dargmuesli/jobber-aws-msmtp:1.3.0
     secrets:
       - source: jobber-aliases
@@ -29,6 +31,9 @@ services:
         target: /home/jobberuser/.aws/config
       - source: jobber-msmtprc
         target: /etc/msmtprc
+      - postgres-db
+      - postgres-role-service-jobber-password
+      - postgres-role-service-jobber-username
     volumes:
       - ./configurations/.jobber:/home/jobberuser/.jobber:ro
       - ./configurations/sinks:/srv/sinks:ro

src/development/jobber/configurations/.jobber

@@ -10,3 +10,12 @@ jobs:
       - type: system-email
     notifyOnFailure:
       - type: system-email
+  OutboxPurge:
+    cmd: PGPASSWORD="$(cat /run/secrets/postgres-role-service-jobber-password)" psql -v ON_ERROR_STOP=1 -h "$POSTGRES_HOST" -U "$(cat /run/secrets/postgres-role-service-jobber-username)" -d "$(cat "$POSTGRES_DB_FILE")" -c "DELETE FROM vibetype_private.outbox WHERE created_at < now() - interval '24 hours'"
+    time: 0 0 */2 *
+    notifyOnSuccess:
+      - type: system-email
+    notifyOnError:
+      - type: system-email
+    notifyOnFailure:
+      - type: system-email

src/development/jobber/configurations/sinks/sentry-outbox-purge/error.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+curl "${SENTRY_CRONS_OUTBOX_PURGE}?status=error"

src/development/jobber/configurations/sinks/sentry-outbox-purge/in-progress.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+curl "${SENTRY_CRONS_OUTBOX_PURGE}?status=in_progress"

src/development/jobber/configurations/sinks/sentry-outbox-purge/ok.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+curl "${SENTRY_CRONS_OUTBOX_PURGE}?status=ok"

src/development/postgres/compose.yaml

@@ -11,6 +11,12 @@ secrets:
   postgres-role-service-grafana-username:
     # The username of the observation dashboard's database role.
     file: ~~/artifacts/secrets/postgres-role-service-grafana-username.secret
+  postgres-role-service-jobber-password:
+    # The password of the job scheduler's database role.
+    file: ~~/artifacts/secrets/postgres-role-service-jobber-password.secret
+  postgres-role-service-jobber-username:
+    # The username of the job scheduler's database role.
+    file: ~~/artifacts/secrets/postgres-role-service-jobber-username.secret
   postgres-role-service-postgraphile-password:
     # The password of the GraphQL API database wrapper's database role.
     file: ~~/artifacts/secrets/postgres-role-service-postgraphile-password.secret
@@ -90,6 +96,11 @@ x-dargstack:
       type: random_string
     postgres-role-service-grafana-username:
       type: wordlist_word
+    postgres-role-service-jobber-password:
+      special_characters: false
+      type: random_string
+    postgres-role-service-jobber-username:
+      type: wordlist_word
     postgres-role-service-postgraphile-password:
       special_characters: false
       type: random_string

src/development/sqitch/compose.yaml

@@ -16,6 +16,8 @@ services:
     secrets:
       - postgres-role-service-grafana-password
       - postgres-role-service-grafana-username
+      - postgres-role-service-jobber-password
+      - postgres-role-service-jobber-username
       - postgres-role-service-postgraphile-password
       - postgres-role-service-postgraphile-username
       - postgres-role-service-reccoom-password

src/production/.env.template

@@ -1,5 +1,6 @@
 CLOUDFLARED_TUNNEL_TOKEN=
 SENTRY_CRONS=
+SENTRY_CRONS_OUTBOX_PURGE=
 STACK_DOMAIN=
 TRAEFIK_ACME_EMAIL=
 TRAEFIK_ACME_PROVIDER=

src/production/jobber/compose.yaml

@@ -5,6 +5,7 @@ services:
         order: stop-first
     environment:
       SENTRY_CRONS: ${SENTRY_CRONS}
+      SENTRY_CRONS_OUTBOX_PURGE: ${SENTRY_CRONS_OUTBOX_PURGE}
     volumes:
       - (( append ))
       - postgres-backup-data:/backups/

src/production/jobber/configurations/.jobber

@@ -18,3 +18,17 @@ jobs:
     notifyOnFailure:
       - type: program
         path: /srv/sinks/sentry/error.sh
+  OutboxPurge:
+    cmd: |
+      /srv/sinks/sentry-outbox-purge/in-progress.sh
+      PGPASSWORD="$(cat /run/secrets/postgres-role-service-jobber-password)" psql -v ON_ERROR_STOP=1 -h "$POSTGRES_HOST" -U "$(cat /run/secrets/postgres-role-service-jobber-username)" -d "$(cat "$POSTGRES_DB_FILE")" -c "DELETE FROM vibetype_private.outbox WHERE created_at < now() - interval '24 hours'"
+    time: 0 0 */2 *
+    notifyOnSuccess:
+      - type: program
+        path: /srv/sinks/sentry-outbox-purge/ok.sh
+    notifyOnError:
+      - type: program
+        path: /srv/sinks/sentry-outbox-purge/error.sh
+    notifyOnFailure:
+      - type: program
+        path: /srv/sinks/sentry-outbox-purge/error.sh