feat: ERPNext kill switch for referral reward payouts

package.json

@@ -34,7 +34,8 @@
     "bridge-webhook": ". ./.env && ts-node --transpile-only -r tsconfig-paths/register src/servers/bridge-webhook-server.ts --configPath dev/config/base-config.yaml",
     "fygaro-webhook": ". ./.env && ts-node --transpile-only -r tsconfig-paths/register src/servers/fygaro-webhook-server.ts --configPath dev/config/base-config.yaml",
     "replay-bridge-events": "yarn build && node lib/scripts/replay-bridge-events.js",
-    "reconcile-bridge-ibex-deposits": "yarn build && node lib/scripts/reconcile-bridge-ibex-deposits.js"
+    "reconcile-bridge-ibex-deposits": "yarn build && node lib/scripts/reconcile-bridge-ibex-deposits.js",
+    "retry-deferred-referral-rewards": "yarn build && node lib/scripts/retry-deferred-referral-rewards.js"
   },
   "engines": {
     "node": ">=24.0.0 <25"

src/app/invite/award-referral-reward.ts

@@ -7,9 +7,12 @@ import { WalletCurrency } from "@domain/shared"
 
 import { AccountsRepository, WalletsRepository } from "@services/mongoose"
 import { InviteRepository } from "@services/mongoose/models/invite"
+
 import { nextReferralRewardSeq } from "@services/mongoose/models/referral-reward-counter"
 import { baseLogger } from "@services/logger"
 
+import { referralRewardsEnabledInErp } from "./referral-settings"
+
 import { sendReferralRewardNotificationBestEffort } from "./send-referral-notifications"
 
 const REWARDS_ROLE = "rewards"
@@ -69,6 +72,24 @@ export const awardReferralRewardOnKycApproval = async ({
     })
     if (!pending) return // not a referred user, or already claimed
 
+    // Operator kill switch (ERPNext "Referral Settings" single). Checked
+    // AFTER the invite lookups (so non-referred KYC approvals never touch the
+    // ERP reader or log a bogus deferral — this hook fires on EVERY Bridge KYC
+    // approval) but BEFORE the atomic reward claim on purpose: returning here
+    // leaves the invite ACCEPTED and unrewarded — a DEFERRED payout, not a
+    // failed one. It pays on any later trigger (KYC status re-fire, or an ops
+    // sweep of ACCEPTED+unrewarded invites) once the switch is re-enabled.
+    // Disabled and unreadable are treated identically: no affirmative yes, no
+    // money.
+    const erpEnabled = await referralRewardsEnabledInErp()
+    if (!erpEnabled) {
+      baseLogger.info(
+        { accountId },
+        "referral reward deferred: ERPNext kill switch is off or unreadable",
+      )
+      return
+    }
+
     // Atomic claim — only one caller flips absent -> "processing".
     const invite = await InviteRepository.findOneAndUpdate(
       { _id: pending._id, rewardStatus: { $exists: false } },

src/app/invite/referral-settings.ts

@@ -0,0 +1,51 @@
+/**
+ * Cached reader for the ERPNext "Referral Settings" Single doctype — the
+ * operator kill switch for referral reward payouts (requested after the
+ * 2026-09-01 referral-farming wave: 87 fresh accounts, 288 invites in a day).
+ *
+ * Polarity is the whole design: a payout may proceed ONLY on an affirmative,
+ * readable `rewards_enabled = 1`. "Disabled", "unreadable", "missing", and
+ * "malformed" all mean the same thing — DO NOT PAY. The caller defers by
+ * returning before the reward claim, so nothing is marked failed and nothing
+ * is lost: the invite stays ACCEPTED + unrewarded and pays on a later trigger
+ * once the switch is back on. That makes fail-closed safe here in a way it
+ * would not be if deferral were destructive.
+ *
+ * Memoised for ~60s (successes AND failures, same as the Fygaro settings
+ * reader) so a KYC-approval burst cannot become an ERPNext fetch storm, while
+ * an operator flip still takes effect within a minute.
+ */
+import { toBoolean } from "@services/frappe/coerce"
+import ErpNext from "@services/frappe/ErpNext"
+import { baseLogger } from "@services/logger"
+
+const CACHE_TTL_MS = 60_000
+
+let cache: { enabled: boolean; at: number } | undefined
+
+export const resetReferralSettingsCache = () => {
+  cache = undefined
+}
+
+/** True only on an affirmative, readable rewards_enabled=1. */
+export const referralRewardsEnabledInErp = async (): Promise<boolean> => {
+  const now = Date.now()
+  if (cache && now - cache.at < CACHE_TTL_MS) return cache.enabled
+
+  const doc = ErpNext?.getReferralSettings
+    ? await ErpNext.getReferralSettings()
+    : undefined
+
+  let enabled = false
+  if (doc instanceof Error || doc === undefined) {
+    baseLogger.warn(
+      { error: doc },
+      "Referral Settings unreadable — treating reward payouts as DISABLED (deferred)",
+    )
+  } else {
+    enabled = toBoolean(doc.rewards_enabled)
+  }
+
+  cache = { enabled, at: now }
+  return enabled
+}

src/app/invite/retry-deferred-rewards.ts

@@ -0,0 +1,65 @@
+/**
+ * Operator reconciliation for referral rewards deferred by the ERPNext kill
+ * switch (see referral-settings.ts and award-referral-reward.ts). While the
+ * switch is off (or unreadable), a matching Bridge KYC approval leaves its
+ * invite ACCEPTED and unrewarded rather than paying or marking it failed —
+ * a deferred payout, not a lost one. This sweep finds that backlog and,
+ * unless dryRun, replays the award hook for every distinct account in it.
+ *
+ * Also usable in dry-run alone: an operator about to flip the switch back on
+ * can see the backlog size up front instead of relying on grepping logs for
+ * "deferred" lines.
+ */
+import { InviteStatus } from "@domain/invite"
+
+import { InviteRepository } from "@services/mongoose/models/invite"
+import { baseLogger } from "@services/logger"
+
+import { awardReferralRewardOnKycApproval } from "./award-referral-reward"
+
+export type RetryDeferredReferralRewardsResult = {
+  backlogCount: number
+  accountsRetried: number
+}
+
+/**
+ * awardReferralRewardOnKycApproval is itself idempotent and fail-closed — it
+ * re-checks the ERPNext switch, the already-processed guard, and the atomic
+ * reward claim — so re-invoking it here is always safe: a still-disabled
+ * switch or an already-claimed invite is a silent no-op, never a double-pay.
+ */
+export const retryDeferredReferralRewards = async ({
+  dryRun,
+}: {
+  dryRun: boolean
+}): Promise<RetryDeferredReferralRewardsResult> => {
+  const deferred = await InviteRepository.find({
+    status: InviteStatus.ACCEPTED,
+    rewardStatus: { $exists: false },
+    redeemedById: { $exists: true },
+  })
+
+  // Redemption history can hold more than one accepted invite for the same
+  // account, but the award hook only ever pays one — dedupe before retrying
+  // so the same account isn't looked up twice for nothing.
+  const accountIds = [
+    ...new Set(
+      deferred
+        .map((invite) => invite.redeemedById?.toString())
+        .filter((id): id is string => Boolean(id)),
+    ),
+  ] as AccountId[]
+
+  baseLogger.info(
+    { backlogCount: deferred.length, accounts: accountIds.length, dryRun },
+    "referral reward backlog: accepted + unrewarded invites found",
+  )
+
+  if (dryRun) return { backlogCount: deferred.length, accountsRetried: 0 }
+
+  for (const accountId of accountIds) {
+    await awardReferralRewardOnKycApproval({ accountId })
+  }
+
+  return { backlogCount: deferred.length, accountsRetried: accountIds.length }
+}

src/scripts/retry-deferred-referral-rewards.ts

@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+
+/**
+ * Operator tool: replay referral rewards deferred by the ERPNext kill switch
+ * (src/app/invite/referral-settings.ts). Run with --dry-run to see the
+ * backlog size without paying anyone (e.g. right before flipping the switch
+ * back on); omit --dry-run (default: false) to actually replay it. See
+ * src/app/invite/retry-deferred-rewards.ts for why replaying is always safe
+ * (the award hook is idempotent and fail-closed).
+ *
+ * Usage:
+ *   node lib/scripts/retry-deferred-referral-rewards.js \
+ *     --configPath dev/config/base-config.yaml \
+ *     [--dry-run]
+ */
+
+import yargs from "yargs"
+import { hideBin } from "yargs/helpers"
+import { setupMongoConnection } from "@services/mongodb"
+import { baseLogger } from "@services/logger"
+import { retryDeferredReferralRewards } from "@app/invite/retry-deferred-rewards"
+
+export const buildArgs = (argv: string[]) =>
+  yargs(argv)
+    .option("dry-run", {
+      type: "boolean" as const,
+      default: false,
+      describe: "Report the backlog size without paying anyone (default: false)",
+    })
+    .option("configPath", { type: "string" as const, demandOption: true })
+    .parseSync()
+
+const main = async () => {
+  const args = buildArgs(hideBin(process.argv))
+  const dryRun = args["dry-run"]
+  const result = await retryDeferredReferralRewards({ dryRun })
+  baseLogger.info(
+    { ...result, dryRun },
+    dryRun
+      ? "Referral reward backlog check finished (dry run, nothing paid)"
+      : "Referral reward backlog replay finished",
+  )
+}
+
+if (require.main === module) {
+  setupMongoConnection()
+    .then(async (mongoose) => {
+      await main()
+      await mongoose?.connection.close()
+      process.exit(0)
+    })
+    .catch((error) => {
+      baseLogger.error({ error }, "Referral reward backlog replay failed")
+      process.exit(1)
+    })
+}

src/services/frappe/ErpNext.ts

@@ -17,6 +17,7 @@ import {
   FeeDiscountQueryError,
   AllowedCountryQueryError,
   FygaroSettingsQueryError,
+  ReferralSettingsQueryError,
   FygaroTopupHistoryQueryError,
   JournalEntryDeleteError,
   SetDocTypeValueError,
@@ -138,6 +139,12 @@ export type FygaroSettingsDoc = {
   l3_daily_limit?: number | string
 }
 
+// The ERPNext "Referral Settings" Single doctype: the operator-facing kill
+// switch for referral reward payouts. One Check field by design.
+export type ReferralSettingsDoc = {
+  rewards_enabled?: unknown
+}
+
 // Raw "Fee Discount" doctype row as ERPNext returns it (one row per username,
 // operator-managed at /app/fee-discount). Numeric/check fields may arrive as
 // numbers or strings; the caller (fee-discounts.ts) coerces and validates
@@ -503,6 +510,34 @@ export class ErpNext {
     }
   }
 
+  // Reads the "Referral Settings" Single doctype — the operator kill switch
+  // for referral reward payouts. Consulted before every payout (through a 60s
+  // cache in @app/invite/referral-settings); a read failure means "defer the
+  // payout", so this must return the error rather than a default.
+  async getReferralSettings(): Promise<ReferralSettingsDoc | ReferralSettingsQueryError> {
+    try {
+      const resp = await axios.get(
+        `${this.url}/api/resource/${encodeURIComponent("Referral Settings")}/${encodeURIComponent("Referral Settings")}`,
+        { headers: this.headers },
+      )
+      const data = resp.data?.data
+      if (!data)
+        return new ReferralSettingsQueryError("No data in Referral Settings response")
+      return data as ReferralSettingsDoc
+    } catch (err) {
+      const responseData = isAxiosError(err) ? err.response?.data : undefined
+      baseLogger.error(
+        { err, responseData },
+        "Error querying Referral Settings from ERPNext",
+      )
+      recordExceptionInCurrentSpan({
+        error: err,
+        attributes: { "erpnext.exception": responseData?.exception },
+      })
+      return new ReferralSettingsQueryError(err)
+    }
+  }
+
   // Countries whose residents Bridge can issue Flash a USD virtual account
   // for: the ERPNext "Allowed Country" doctype rows ops have ticked
   // `flash_allowed`. Read by the bridgeInitiateKyc country gate. Only the

src/services/frappe/coerce.ts

@@ -9,7 +9,8 @@
  * soft the divergence would be silent.
  *
  * Consumers: fee-discounts.ts, fygaro/webhook-server/fygaro-settings.ts,
- * allowed-countries.ts (and, for the same alpha-2 rule, app/bridge/kyc-gate.ts).
+ * app/invite/referral-settings.ts, allowed-countries.ts (and, for the same
+ * alpha-2 rule, app/bridge/kyc-gate.ts).
  */
 
 /**

src/services/frappe/errors.ts

@@ -14,6 +14,7 @@ export class BankAccountUpdateRequestQueryError extends ErpNextError {}
 export class ExchangeRateQueryError extends ErpNextError {}
 export class BridgeTransferRequestUpsertError extends ErpNextError {}
 export class FygaroSettingsQueryError extends ErpNextError {}
+export class ReferralSettingsQueryError extends ErpNextError {}
 export class FeeDiscountQueryError extends ErpNextError {}
 export class AllowedCountryQueryError extends ErpNextError {}
 export class FygaroTopupHistoryQueryError extends ErpNextError {}

test/flash/unit/app/invite/award-referral-reward.spec.ts

@@ -43,6 +43,11 @@ jest.mock("@app/payments/send-intraledger", () => ({
   intraledgerPaymentSendWalletIdForUsdWallet: (...a: unknown[]) => mockPay(...a),
 }))
 
+const mockErpEnabled = jest.fn()
+jest.mock("@app/invite/referral-settings", () => ({
+  referralRewardsEnabledInErp: (...a: unknown[]) => mockErpEnabled(...a),
+}))
+
 const mockRewardPush = jest.fn()
 jest.mock("@app/invite/send-referral-notifications", () => ({
   sendReferralRewardNotificationBestEffort: (...a: unknown[]) => mockRewardPush(...a),
@@ -89,6 +94,7 @@ const lastSet = () =>
 
 describe("awardReferralRewardOnKycApproval", () => {
   beforeEach(() => {
+    mockErpEnabled.mockResolvedValue(true)
     jest.clearAllMocks()
     mockGetConfig.mockReturnValue({ enabled: true, tiers: DEFAULT_TIERS })
     mockExists.mockResolvedValue(null) // no prior processed invite for the account
@@ -112,6 +118,50 @@ describe("awardReferralRewardOnKycApproval", () => {
     expect(mockPay).not.toHaveBeenCalled()
   })
 
+  it("defers (no claim, no failure mark) when the ERPNext kill switch is off", async () => {
+    mockErpEnabled.mockResolvedValue(false)
+
+    await awardReferralRewardOnKycApproval({ accountId: INVITEE })
+
+    // Deferral means the invite is left exactly as found: unclaimed and
+    // retryable. Nothing is claimed, paid, or marked failed — and the log
+    // names a genuinely deferred reward (a pending invite was found first).
+    expect(mockFindOne).toHaveBeenCalled()
+    expect(mockFindOneAndUpdate).not.toHaveBeenCalled()
+    expect(mockPay).not.toHaveBeenCalled()
+    expect(mockUpdateOne).not.toHaveBeenCalled()
+    expect(baseLogger.info).toHaveBeenCalledWith(
+      expect.objectContaining({ accountId: INVITEE }),
+      expect.stringContaining("deferred"),
+    )
+  })
+
+  it("never consults ERP or logs a deferral for a non-referred KYC approval", async () => {
+    // This hook fires on EVERY Bridge KYC approval. With the switch off, an
+    // ops grep for "deferred" must return only genuinely deferred rewards —
+    // accounts with no invite must not touch the ERP reader or log at all.
+    mockErpEnabled.mockResolvedValue(false)
+    mockFindOne.mockResolvedValue(null) // not a referred user
+
+    await awardReferralRewardOnKycApproval({ accountId: INVITEE })
+
+    expect(mockErpEnabled).not.toHaveBeenCalled()
+    expect(baseLogger.info).not.toHaveBeenCalled()
+    expect(mockFindOneAndUpdate).not.toHaveBeenCalled()
+    expect(mockPay).not.toHaveBeenCalled()
+  })
+
+  it("never consults ERP when the account's reward was already processed", async () => {
+    mockErpEnabled.mockResolvedValue(false)
+    mockExists.mockResolvedValue({ _id: "earlier-invite" })
+
+    await awardReferralRewardOnKycApproval({ accountId: INVITEE })
+
+    expect(mockErpEnabled).not.toHaveBeenCalled()
+    expect(baseLogger.info).not.toHaveBeenCalled()
+    expect(mockPay).not.toHaveBeenCalled()
+  })
+
   it("never pays a second reward for the same account (KYC re-approval flap)", async () => {
     // A prior invite for this account was already claimed/processed.
     mockExists.mockResolvedValue({ _id: "earlier-invite" })