diff --git a/src/development/debezium/configurations/entrypoint.sh b/src/development/debezium/configurations/entrypoint.sh index 4a61818f..45973796 100755 --- a/src/development/debezium/configurations/entrypoint.sh +++ b/src/development/debezium/configurations/entrypoint.sh @@ -27,8 +27,17 @@ curl --fail --output /dev/null --silent --show-error \ "database.password": "'"$POSTGRES_PASSWORD"'", "database.user": "'"$POSTGRES_USER"'", "plugin.name": "pgoutput", - "table.include.list": "vibetype.event,vibetype.upload,vibetype_private.notification", - "topic.prefix" : "vibetype" + "table.include.list": "vibetype_private.outbox", + "topic.prefix" : "vibetype", + "transforms": "outbox", + "transforms.outbox.type": "io.debezium.transforms.outbox.EventRouter", + "transforms.outbox.route.by.field": "aggregate_type", + "transforms.outbox.route.topic.replacement": "vibetype.outbox.${routedByValue}", + "transforms.outbox.table.field.event.key": "aggregate_id", + "transforms.outbox.predicate": "isOutbox", + "predicates": "isOutbox", + "predicates.isOutbox.type": "org.apache.kafka.connect.transforms.predicates.TopicNameMatches", + "predicates.isOutbox.pattern": "vibetype\\.vibetype_private\\.outbox" }' echo "PostgreSQL connector '$CONNECTOR_NAME' is up to date." diff --git a/src/development/jobber/compose.yaml b/src/development/jobber/compose.yaml index 8b0ff062..e9d131a7 100644 --- a/src/development/jobber/compose.yaml +++ b/src/development/jobber/compose.yaml @@ -19,6 +19,8 @@ services: # You cannot access the jobber via a web interface. environment: AWS_SHARED_CREDENTIALS_FILE: /run/secrets/jobber-aws-credentials + POSTGRES_DB_FILE: /run/secrets/postgres-db + POSTGRES_HOST: postgres image: ghcr.io/dargmuesli/jobber-aws-msmtp:1.3.0 secrets: - source: jobber-aliases @@ -29,6 +31,9 @@ services: target: /home/jobberuser/.aws/config - source: jobber-msmtprc target: /etc/msmtprc + - postgres-db + - postgres-role-service-jobber-password + - postgres-role-service-jobber-username volumes: - ./configurations/.jobber:/home/jobberuser/.jobber:ro - ./configurations/sinks:/srv/sinks:ro diff --git a/src/development/jobber/configurations/.jobber b/src/development/jobber/configurations/.jobber index 906ff841..a824435b 100644 --- a/src/development/jobber/configurations/.jobber +++ b/src/development/jobber/configurations/.jobber @@ -10,3 +10,12 @@ jobs: - type: system-email notifyOnFailure: - type: system-email + OutboxPurge: + cmd: PGPASSWORD="$(cat /run/secrets/postgres-role-service-jobber-password)" psql -v ON_ERROR_STOP=1 -h "$POSTGRES_HOST" -U "$(cat /run/secrets/postgres-role-service-jobber-username)" -d "$(cat "$POSTGRES_DB_FILE")" -c "DELETE FROM vibetype_private.outbox WHERE created_at < now() - interval '24 hours'" + time: 0 0 */2 * + notifyOnSuccess: + - type: system-email + notifyOnError: + - type: system-email + notifyOnFailure: + - type: system-email diff --git a/src/development/jobber/configurations/sinks/sentry-outbox-purge/error.sh b/src/development/jobber/configurations/sinks/sentry-outbox-purge/error.sh new file mode 100755 index 00000000..e769052f --- /dev/null +++ b/src/development/jobber/configurations/sinks/sentry-outbox-purge/error.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +curl "${SENTRY_CRONS_OUTBOX_PURGE}?status=error" diff --git a/src/development/jobber/configurations/sinks/sentry-outbox-purge/in-progress.sh b/src/development/jobber/configurations/sinks/sentry-outbox-purge/in-progress.sh new file mode 100755 index 00000000..8658bb7b --- /dev/null +++ b/src/development/jobber/configurations/sinks/sentry-outbox-purge/in-progress.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +curl "${SENTRY_CRONS_OUTBOX_PURGE}?status=in_progress" diff --git a/src/development/jobber/configurations/sinks/sentry-outbox-purge/ok.sh b/src/development/jobber/configurations/sinks/sentry-outbox-purge/ok.sh new file mode 100755 index 00000000..d17c2407 --- /dev/null +++ b/src/development/jobber/configurations/sinks/sentry-outbox-purge/ok.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +curl "${SENTRY_CRONS_OUTBOX_PURGE}?status=ok" diff --git a/src/development/postgres/compose.yaml b/src/development/postgres/compose.yaml index 546cb761..251be610 100644 --- a/src/development/postgres/compose.yaml +++ b/src/development/postgres/compose.yaml @@ -11,6 +11,12 @@ secrets: postgres-role-service-grafana-username: # The username of the observation dashboard's database role. file: ~~/artifacts/secrets/postgres-role-service-grafana-username.secret + postgres-role-service-jobber-password: + # The password of the job scheduler's database role. + file: ~~/artifacts/secrets/postgres-role-service-jobber-password.secret + postgres-role-service-jobber-username: + # The username of the job scheduler's database role. + file: ~~/artifacts/secrets/postgres-role-service-jobber-username.secret postgres-role-service-postgraphile-password: # The password of the GraphQL API database wrapper's database role. file: ~~/artifacts/secrets/postgres-role-service-postgraphile-password.secret @@ -90,6 +96,11 @@ x-dargstack: type: random_string postgres-role-service-grafana-username: type: wordlist_word + postgres-role-service-jobber-password: + special_characters: false + type: random_string + postgres-role-service-jobber-username: + type: wordlist_word postgres-role-service-postgraphile-password: special_characters: false type: random_string diff --git a/src/development/sqitch/compose.yaml b/src/development/sqitch/compose.yaml index 12499b7b..aab6ca6e 100644 --- a/src/development/sqitch/compose.yaml +++ b/src/development/sqitch/compose.yaml @@ -16,6 +16,8 @@ services: secrets: - postgres-role-service-grafana-password - postgres-role-service-grafana-username + - postgres-role-service-jobber-password + - postgres-role-service-jobber-username - postgres-role-service-postgraphile-password - postgres-role-service-postgraphile-username - postgres-role-service-reccoom-password diff --git a/src/production/.env.template b/src/production/.env.template index 8ffc032b..8934c80b 100644 --- a/src/production/.env.template +++ b/src/production/.env.template @@ -1,5 +1,6 @@ CLOUDFLARED_TUNNEL_TOKEN= SENTRY_CRONS= +SENTRY_CRONS_OUTBOX_PURGE= STACK_DOMAIN= TRAEFIK_ACME_EMAIL= TRAEFIK_ACME_PROVIDER= \ No newline at end of file diff --git a/src/production/jobber/compose.yaml b/src/production/jobber/compose.yaml index 063fbc2f..764b0aa8 100644 --- a/src/production/jobber/compose.yaml +++ b/src/production/jobber/compose.yaml @@ -5,6 +5,7 @@ services: order: stop-first environment: SENTRY_CRONS: ${SENTRY_CRONS} + SENTRY_CRONS_OUTBOX_PURGE: ${SENTRY_CRONS_OUTBOX_PURGE} volumes: - (( append )) - postgres-backup-data:/backups/ diff --git a/src/production/jobber/configurations/.jobber b/src/production/jobber/configurations/.jobber index 162ba232..69266354 100644 --- a/src/production/jobber/configurations/.jobber +++ b/src/production/jobber/configurations/.jobber @@ -18,3 +18,17 @@ jobs: notifyOnFailure: - type: program path: /srv/sinks/sentry/error.sh + OutboxPurge: + cmd: | + /srv/sinks/sentry-outbox-purge/in-progress.sh + PGPASSWORD="$(cat /run/secrets/postgres-role-service-jobber-password)" psql -v ON_ERROR_STOP=1 -h "$POSTGRES_HOST" -U "$(cat /run/secrets/postgres-role-service-jobber-username)" -d "$(cat "$POSTGRES_DB_FILE")" -c "DELETE FROM vibetype_private.outbox WHERE created_at < now() - interval '24 hours'" + time: 0 0 */2 * + notifyOnSuccess: + - type: program + path: /srv/sinks/sentry-outbox-purge/ok.sh + notifyOnError: + - type: program + path: /srv/sinks/sentry-outbox-purge/error.sh + notifyOnFailure: + - type: program + path: /srv/sinks/sentry-outbox-purge/error.sh