From abfd48232a55464c32dd1538a10f6469db1ca75d Mon Sep 17 00:00:00 2001 From: Dread Date: Tue, 1 Sep 2026 14:23:35 -0700 Subject: [PATCH 1/4] feat: ERPNext kill switch for referral reward payouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reads the new 'Referral Settings' Single (frappe-flash-admin#75) before every payout, through a 60s cache. Polarity is the feature: money moves only on an affirmative, readable rewards_enabled=1 — off, unreadable, missing, and malformed all mean do-not-pay. Fail-closed is safe here because the gate runs BEFORE the atomic reward claim: returning leaves the invite ACCEPTED and unrewarded, a DEFERRED payout that pays on any later trigger once the switch is back on. Nothing is marked failed, nothing is lost, and an ERP outage cannot strand or double-pay a reward. Context: 2026-09-01 referral-farming wave (87 fresh accounts, 288 invites in a day, invitee->inviter chains). The deployments-level referralReward.enabled flag still exists but needs a deploy; this gives the operator a ~60s toggle in ERPNext. --- src/app/invite/award-referral-reward.ts | 18 +++++ src/app/invite/referral-settings.ts | 51 ++++++++++++++ src/services/frappe/ErpNext.ts | 32 +++++++++ src/services/frappe/errors.ts | 1 + .../app/invite/award-referral-reward.spec.ts | 29 ++++++++ .../unit/app/invite/referral-settings.spec.ts | 70 +++++++++++++++++++ 6 files changed, 201 insertions(+) create mode 100644 src/app/invite/referral-settings.ts create mode 100644 test/flash/unit/app/invite/referral-settings.spec.ts diff --git a/src/app/invite/award-referral-reward.ts b/src/app/invite/award-referral-reward.ts index d635294ef..3b5248902 100644 --- a/src/app/invite/award-referral-reward.ts +++ b/src/app/invite/award-referral-reward.ts @@ -7,9 +7,12 @@ import { WalletCurrency } from "@domain/shared" import { AccountsRepository, WalletsRepository } from "@services/mongoose" import { InviteRepository } from "@services/mongoose/models/invite" + import { nextReferralRewardSeq } from "@services/mongoose/models/referral-reward-counter" import { baseLogger } from "@services/logger" +import { referralRewardsEnabledInErp } from "./referral-settings" + import { sendReferralRewardNotificationBestEffort } from "./send-referral-notifications" const REWARDS_ROLE = "rewards" @@ -51,6 +54,21 @@ export const awardReferralRewardOnKycApproval = async ({ const config = getReferralRewardConfig() if (!config.enabled) return + // Operator kill switch (ERPNext "Referral Settings" single). Checked + // BEFORE the atomic reward claim on purpose: returning here leaves the + // invite ACCEPTED and unrewarded — a DEFERRED payout, not a failed one. + // It pays on any later trigger (KYC status re-fire, or an ops sweep of + // ACCEPTED+unrewarded invites) once the switch is re-enabled. Disabled + // and unreadable are treated identically: no affirmative yes, no money. + const erpEnabled = await referralRewardsEnabledInErp() + if (!erpEnabled) { + baseLogger.info( + { accountId }, + "referral reward deferred: ERPNext kill switch is off or unreadable", + ) + return + } + // One reward per invitee, ever. Bridge KYC can flap back to "approved" // (approved -> under_review -> approved re-fires this hook), and redemption // history may hold more than one accepted invite — if ANY invite for this diff --git a/src/app/invite/referral-settings.ts b/src/app/invite/referral-settings.ts new file mode 100644 index 000000000..c720aa044 --- /dev/null +++ b/src/app/invite/referral-settings.ts @@ -0,0 +1,51 @@ +/** + * Cached reader for the ERPNext "Referral Settings" Single doctype — the + * operator kill switch for referral reward payouts (requested after the + * 2026-09-01 referral-farming wave: 87 fresh accounts, 288 invites in a day). + * + * Polarity is the whole design: a payout may proceed ONLY on an affirmative, + * readable `rewards_enabled = 1`. "Disabled", "unreadable", "missing", and + * "malformed" all mean the same thing — DO NOT PAY. The caller defers by + * returning before the reward claim, so nothing is marked failed and nothing + * is lost: the invite stays ACCEPTED + unrewarded and pays on a later trigger + * once the switch is back on. That makes fail-closed safe here in a way it + * would not be if deferral were destructive. + * + * Memoised for ~60s (successes AND failures, same as the Fygaro settings + * reader) so a KYC-approval burst cannot become an ERPNext fetch storm, while + * an operator flip still takes effect within a minute. + */ +import { toBoolean } from "@services/frappe/coerce" +import ErpNext from "@services/frappe/ErpNext" +import { baseLogger } from "@services/logger" + +const CACHE_TTL_MS = 60_000 + +let cache: { enabled: boolean; at: number } | undefined + +export const resetReferralSettingsCache = () => { + cache = undefined +} + +/** True only on an affirmative, readable rewards_enabled=1. */ +export const referralRewardsEnabledInErp = async (): Promise => { + const now = Date.now() + if (cache && now - cache.at < CACHE_TTL_MS) return cache.enabled + + const doc = ErpNext?.getReferralSettings + ? await ErpNext.getReferralSettings() + : undefined + + let enabled = false + if (doc instanceof Error || doc === undefined) { + baseLogger.warn( + { error: doc }, + "Referral Settings unreadable — treating reward payouts as DISABLED (deferred)", + ) + } else { + enabled = toBoolean(doc.rewards_enabled) + } + + cache = { enabled, at: now } + return enabled +} diff --git a/src/services/frappe/ErpNext.ts b/src/services/frappe/ErpNext.ts index f5d44aad2..21e74f021 100644 --- a/src/services/frappe/ErpNext.ts +++ b/src/services/frappe/ErpNext.ts @@ -16,6 +16,7 @@ import { ExchangeRateQueryError, FeeDiscountQueryError, FygaroSettingsQueryError, + ReferralSettingsQueryError, FygaroTopupHistoryQueryError, JournalEntryDeleteError, SetDocTypeValueError, @@ -137,6 +138,12 @@ export type FygaroSettingsDoc = { l3_daily_limit?: number | string } +// The ERPNext "Referral Settings" Single doctype: the operator-facing kill +// switch for referral reward payouts. One Check field by design. +export type ReferralSettingsDoc = { + rewards_enabled?: unknown +} + // Raw "Fee Discount" doctype row as ERPNext returns it (one row per username, // operator-managed at /app/fee-discount). Numeric/check fields may arrive as // numbers or strings; the caller (fee-discounts.ts) coerces and validates @@ -460,6 +467,31 @@ export class ErpNext { // cashouts). The consumers read this through a 60s cache and fail open to a // 0% discount, so an ERPNext blip can never block a credit or an offer — // it just charges the standard fee. + + // Reads the "Referral Settings" Single doctype — the operator kill switch + // for referral reward payouts. Consulted before every payout (through a 60s + // cache in @app/invite/referral-settings); a read failure means "defer the + // payout", so this must return the error rather than a default. + async getReferralSettings(): Promise { + try { + const resp = await axios.get( + `${this.url}/api/resource/${encodeURIComponent("Referral Settings")}/${encodeURIComponent("Referral Settings")}`, + { headers: this.headers }, + ) + const data = resp.data?.data + if (!data) + return new ReferralSettingsQueryError("No data in Referral Settings response") + return data as ReferralSettingsDoc + } catch (err) { + const responseData = isAxiosError(err) ? err.response?.data : undefined + baseLogger.error( + { err, responseData }, + "Error querying Referral Settings from ERPNext", + ) + return new ReferralSettingsQueryError(err) + } + } + async getFeeDiscounts(): Promise { try { // Serialized through axios `params` (not hand-interpolated into the URL) diff --git a/src/services/frappe/errors.ts b/src/services/frappe/errors.ts index 030ebd7ad..3da6c939d 100644 --- a/src/services/frappe/errors.ts +++ b/src/services/frappe/errors.ts @@ -14,5 +14,6 @@ export class BankAccountUpdateRequestQueryError extends ErpNextError {} export class ExchangeRateQueryError extends ErpNextError {} export class BridgeTransferRequestUpsertError extends ErpNextError {} export class FygaroSettingsQueryError extends ErpNextError {} +export class ReferralSettingsQueryError extends ErpNextError {} export class FeeDiscountQueryError extends ErpNextError {} export class FygaroTopupHistoryQueryError extends ErpNextError {} diff --git a/test/flash/unit/app/invite/award-referral-reward.spec.ts b/test/flash/unit/app/invite/award-referral-reward.spec.ts index 2d026791a..1f2f90493 100644 --- a/test/flash/unit/app/invite/award-referral-reward.spec.ts +++ b/test/flash/unit/app/invite/award-referral-reward.spec.ts @@ -43,6 +43,11 @@ jest.mock("@app/payments/send-intraledger", () => ({ intraledgerPaymentSendWalletIdForUsdWallet: (...a: unknown[]) => mockPay(...a), })) +const mockErpEnabled = jest.fn() +jest.mock("@app/invite/referral-settings", () => ({ + referralRewardsEnabledInErp: (...a: unknown[]) => mockErpEnabled(...a), +})) + const mockRewardPush = jest.fn() jest.mock("@app/invite/send-referral-notifications", () => ({ sendReferralRewardNotificationBestEffort: (...a: unknown[]) => mockRewardPush(...a), @@ -89,6 +94,7 @@ const lastSet = () => describe("awardReferralRewardOnKycApproval", () => { beforeEach(() => { + mockErpEnabled.mockResolvedValue(true) jest.clearAllMocks() mockGetConfig.mockReturnValue({ enabled: true, tiers: DEFAULT_TIERS }) mockExists.mockResolvedValue(null) // no prior processed invite for the account @@ -112,6 +118,29 @@ describe("awardReferralRewardOnKycApproval", () => { expect(mockPay).not.toHaveBeenCalled() }) + it("defers (no claim, no failure mark) when the ERPNext kill switch is off", async () => { + mockErpEnabled.mockResolvedValue(false) + + await awardReferralRewardOnKycApproval({ accountId: INVITEE }) + + // Deferral means the invite is left exactly as found: unclaimed and + // retryable. Nothing is queried, claimed, paid, or marked failed. + expect(mockFindOneAndUpdate).not.toHaveBeenCalled() + expect(mockPay).not.toHaveBeenCalled() + expect(mockUpdateOne).not.toHaveBeenCalled() + }) + + it("treats an unreadable kill switch the same as off — no affirmative yes, no money", async () => { + // referralRewardsEnabledInErp itself maps errors to false; the award path + // must not distinguish. This pins the calling contract. + mockErpEnabled.mockResolvedValue(false) + + await awardReferralRewardOnKycApproval({ accountId: INVITEE }) + + expect(mockPay).not.toHaveBeenCalled() + expect(mockFindOneAndUpdate).not.toHaveBeenCalled() + }) + it("never pays a second reward for the same account (KYC re-approval flap)", async () => { // A prior invite for this account was already claimed/processed. mockExists.mockResolvedValue({ _id: "earlier-invite" }) diff --git a/test/flash/unit/app/invite/referral-settings.spec.ts b/test/flash/unit/app/invite/referral-settings.spec.ts new file mode 100644 index 000000000..623b87aaf --- /dev/null +++ b/test/flash/unit/app/invite/referral-settings.spec.ts @@ -0,0 +1,70 @@ +/** + * The ERPNext referral-rewards kill switch. The polarity IS the feature: + * money moves only on an affirmative, readable rewards_enabled=1 — disabled, + * unreadable, missing, and malformed must all read as "do not pay". + */ +const mockGetReferralSettings = jest.fn() +jest.mock("@services/frappe/ErpNext", () => ({ + __esModule: true, + default: { + getReferralSettings: (...a: unknown[]) => mockGetReferralSettings(...a), + }, +})) + +jest.mock("@services/logger", () => ({ + baseLogger: { info: jest.fn(), error: jest.fn(), warn: jest.fn() }, +})) + +import { + referralRewardsEnabledInErp, + resetReferralSettingsCache, +} from "@app/invite/referral-settings" +import { ReferralSettingsQueryError } from "@services/frappe/errors" + +beforeEach(() => { + jest.clearAllMocks() + resetReferralSettingsCache() +}) + +describe("referralRewardsEnabledInErp", () => { + it("returns true only on an affirmative rewards_enabled", async () => { + mockGetReferralSettings.mockResolvedValue({ rewards_enabled: 1 }) + + await expect(referralRewardsEnabledInErp()).resolves.toBe(true) + }) + + it.each([ + ["switch off", { rewards_enabled: 0 }], + ["field missing (pre-migration ERP row)", {}], + ["malformed field", { rewards_enabled: "banana" }], + ])("returns false when %s", async (_label, doc) => { + mockGetReferralSettings.mockResolvedValue(doc) + + await expect(referralRewardsEnabledInErp()).resolves.toBe(false) + }) + + it("fails CLOSED on a read error — an outage must not pay rewards", async () => { + mockGetReferralSettings.mockResolvedValue(new ReferralSettingsQueryError("erp down")) + + await expect(referralRewardsEnabledInErp()).resolves.toBe(false) + }) + + it("memoises for the TTL — a KYC burst is one ERP read, not a fetch storm", async () => { + mockGetReferralSettings.mockResolvedValue({ rewards_enabled: 1 }) + + await referralRewardsEnabledInErp() + await referralRewardsEnabledInErp() + await referralRewardsEnabledInErp() + + expect(mockGetReferralSettings).toHaveBeenCalledTimes(1) + }) + + it("caches failures too, so an ERP outage cannot become a retry storm", async () => { + mockGetReferralSettings.mockResolvedValue(new ReferralSettingsQueryError("erp down")) + + await expect(referralRewardsEnabledInErp()).resolves.toBe(false) + await expect(referralRewardsEnabledInErp()).resolves.toBe(false) + + expect(mockGetReferralSettings).toHaveBeenCalledTimes(1) + }) +}) From 016a7eac3f4924e6941fc5436d8e96b7e8c61d7b Mon Sep 17 00:00:00 2001 From: Dread Date: Tue, 1 Sep 2026 14:27:23 -0700 Subject: [PATCH 2/4] =?UTF-8?q?fix:=20apply=20review=20fixes=20=E2=80=94?= =?UTF-8?q?=20gate=20ERP=20check=20behind=20invite=20lookups?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move the ERPNext kill-switch check after the mongo invite lookups (still before the atomic claim) so non-referred KYC approvals never touch the ERP reader or log a bogus 'referral reward deferred' line; the deferral log now names only genuinely deferred rewards. Pinned by new tests asserting ERP is never consulted for non-referred / already-processed accounts. - Drop the duplicate 'unreadable kill switch' unit test; the calling contract is fully pinned by the preceding case and the error->false mapping is pinned in referral-settings.spec.ts. - Update the consumers list in src/services/frappe/coerce.ts to include app/invite/referral-settings.ts. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NEoz7nBtdtsHyuYG5wNPQV --- src/app/invite/award-referral-reward.ts | 33 ++++++++++--------- src/services/frappe/coerce.ts | 3 +- .../app/invite/award-referral-reward.spec.ts | 31 ++++++++++++++--- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/src/app/invite/award-referral-reward.ts b/src/app/invite/award-referral-reward.ts index 3b5248902..40971f121 100644 --- a/src/app/invite/award-referral-reward.ts +++ b/src/app/invite/award-referral-reward.ts @@ -54,21 +54,6 @@ export const awardReferralRewardOnKycApproval = async ({ const config = getReferralRewardConfig() if (!config.enabled) return - // Operator kill switch (ERPNext "Referral Settings" single). Checked - // BEFORE the atomic reward claim on purpose: returning here leaves the - // invite ACCEPTED and unrewarded — a DEFERRED payout, not a failed one. - // It pays on any later trigger (KYC status re-fire, or an ops sweep of - // ACCEPTED+unrewarded invites) once the switch is re-enabled. Disabled - // and unreadable are treated identically: no affirmative yes, no money. - const erpEnabled = await referralRewardsEnabledInErp() - if (!erpEnabled) { - baseLogger.info( - { accountId }, - "referral reward deferred: ERPNext kill switch is off or unreadable", - ) - return - } - // One reward per invitee, ever. Bridge KYC can flap back to "approved" // (approved -> under_review -> approved re-fires this hook), and redemption // history may hold more than one accepted invite — if ANY invite for this @@ -87,6 +72,24 @@ export const awardReferralRewardOnKycApproval = async ({ }) if (!pending) return // not a referred user, or already claimed + // Operator kill switch (ERPNext "Referral Settings" single). Checked + // AFTER the invite lookups (so non-referred KYC approvals never touch the + // ERP reader or log a bogus deferral — this hook fires on EVERY Bridge KYC + // approval) but BEFORE the atomic reward claim on purpose: returning here + // leaves the invite ACCEPTED and unrewarded — a DEFERRED payout, not a + // failed one. It pays on any later trigger (KYC status re-fire, or an ops + // sweep of ACCEPTED+unrewarded invites) once the switch is re-enabled. + // Disabled and unreadable are treated identically: no affirmative yes, no + // money. + const erpEnabled = await referralRewardsEnabledInErp() + if (!erpEnabled) { + baseLogger.info( + { accountId }, + "referral reward deferred: ERPNext kill switch is off or unreadable", + ) + return + } + // Atomic claim — only one caller flips absent -> "processing". const invite = await InviteRepository.findOneAndUpdate( { _id: pending._id, rewardStatus: { $exists: false } }, diff --git a/src/services/frappe/coerce.ts b/src/services/frappe/coerce.ts index 2d62be5e6..381a3ab17 100644 --- a/src/services/frappe/coerce.ts +++ b/src/services/frappe/coerce.ts @@ -8,7 +8,8 @@ * someone teaches one of them a new encoding, and because both coercions fail * soft the divergence would be silent. * - * Consumers: fee-discounts.ts, fygaro/webhook-server/fygaro-settings.ts. + * Consumers: fee-discounts.ts, fygaro/webhook-server/fygaro-settings.ts, + * app/invite/referral-settings.ts. */ /** diff --git a/test/flash/unit/app/invite/award-referral-reward.spec.ts b/test/flash/unit/app/invite/award-referral-reward.spec.ts index 1f2f90493..5bb7603b8 100644 --- a/test/flash/unit/app/invite/award-referral-reward.spec.ts +++ b/test/flash/unit/app/invite/award-referral-reward.spec.ts @@ -124,21 +124,42 @@ describe("awardReferralRewardOnKycApproval", () => { await awardReferralRewardOnKycApproval({ accountId: INVITEE }) // Deferral means the invite is left exactly as found: unclaimed and - // retryable. Nothing is queried, claimed, paid, or marked failed. + // retryable. Nothing is claimed, paid, or marked failed — and the log + // names a genuinely deferred reward (a pending invite was found first). + expect(mockFindOne).toHaveBeenCalled() expect(mockFindOneAndUpdate).not.toHaveBeenCalled() expect(mockPay).not.toHaveBeenCalled() expect(mockUpdateOne).not.toHaveBeenCalled() + expect(baseLogger.info).toHaveBeenCalledWith( + expect.objectContaining({ accountId: INVITEE }), + expect.stringContaining("deferred"), + ) }) - it("treats an unreadable kill switch the same as off — no affirmative yes, no money", async () => { - // referralRewardsEnabledInErp itself maps errors to false; the award path - // must not distinguish. This pins the calling contract. + it("never consults ERP or logs a deferral for a non-referred KYC approval", async () => { + // This hook fires on EVERY Bridge KYC approval. With the switch off, an + // ops grep for "deferred" must return only genuinely deferred rewards — + // accounts with no invite must not touch the ERP reader or log at all. mockErpEnabled.mockResolvedValue(false) + mockFindOne.mockResolvedValue(null) // not a referred user await awardReferralRewardOnKycApproval({ accountId: INVITEE }) - expect(mockPay).not.toHaveBeenCalled() + expect(mockErpEnabled).not.toHaveBeenCalled() + expect(baseLogger.info).not.toHaveBeenCalled() expect(mockFindOneAndUpdate).not.toHaveBeenCalled() + expect(mockPay).not.toHaveBeenCalled() + }) + + it("never consults ERP when the account's reward was already processed", async () => { + mockErpEnabled.mockResolvedValue(false) + mockExists.mockResolvedValue({ _id: "earlier-invite" }) + + await awardReferralRewardOnKycApproval({ accountId: INVITEE }) + + expect(mockErpEnabled).not.toHaveBeenCalled() + expect(baseLogger.info).not.toHaveBeenCalled() + expect(mockPay).not.toHaveBeenCalled() }) it("never pays a second reward for the same account (KYC re-approval flap)", async () => { From 7fba6afaae511120e48291171227e76180a487fb Mon Sep 17 00:00:00 2001 From: Dread Date: Wed, 2 Sep 2026 14:08:56 -0700 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20apply=20review=20fixes=20=E2=80=94?= =?UTF-8?q?=20comment=20placement=20+=20deferred-reward=20reconciliation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - src/services/frappe/ErpNext.ts: move getReferralSettings back below its own doc comment; it had drifted above getFeeDiscounts, leaving the Fee Discount explanation orphaned over the wrong method and getFeeDiscounts with no comment at all. - Add the retry-deferred-referral-rewards sweep promised in the original PR's ops notes but never shipped: while the ERPNext kill switch is off, KYC approvals leave their invite ACCEPTED + unrewarded (deferred, not failed). retryDeferredReferralRewards() finds that backlog and, unless --dry-run, replays awardReferralRewardOnKycApproval per distinct account — safe because that hook is itself idempotent and fail-closed. Dry-run alone also answers the review's minimum ask: an operator flipping the switch back on can see the backlog size instead of grepping logs. - New src/scripts/retry-deferred-referral-rewards.ts CLI wrapper (defaults to --dry-run) plus a yarn script entry, mirroring the existing reconcile-bridge-ibex-deposits / replay-bridge-events scripts. - Tests: retry-deferred-rewards.spec.ts covers the exact query shape, dry-run reporting, per-account dedup on replay, empty backlog, and invites with no redeemedById. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EtQzyzQ38thfC2F8BLK4u8 --- package.json | 3 +- src/app/invite/retry-deferred-rewards.ts | 65 ++++++++++++++ .../retry-deferred-referral-rewards.ts | 51 +++++++++++ src/services/frappe/ErpNext.ts | 49 +++++------ .../app/invite/retry-deferred-rewards.spec.ts | 87 +++++++++++++++++++ 5 files changed, 229 insertions(+), 26 deletions(-) create mode 100644 src/app/invite/retry-deferred-rewards.ts create mode 100644 src/scripts/retry-deferred-referral-rewards.ts create mode 100644 test/flash/unit/app/invite/retry-deferred-rewards.spec.ts diff --git a/package.json b/package.json index 23ffb2123..4aed9fd81 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "bridge-webhook": ". ./.env && ts-node --transpile-only -r tsconfig-paths/register src/servers/bridge-webhook-server.ts --configPath dev/config/base-config.yaml", "fygaro-webhook": ". ./.env && ts-node --transpile-only -r tsconfig-paths/register src/servers/fygaro-webhook-server.ts --configPath dev/config/base-config.yaml", "replay-bridge-events": "yarn build && node lib/scripts/replay-bridge-events.js", - "reconcile-bridge-ibex-deposits": "yarn build && node lib/scripts/reconcile-bridge-ibex-deposits.js" + "reconcile-bridge-ibex-deposits": "yarn build && node lib/scripts/reconcile-bridge-ibex-deposits.js", + "retry-deferred-referral-rewards": "yarn build && node lib/scripts/retry-deferred-referral-rewards.js" }, "engines": { "node": ">=24.0.0 <25" diff --git a/src/app/invite/retry-deferred-rewards.ts b/src/app/invite/retry-deferred-rewards.ts new file mode 100644 index 000000000..35209ec2f --- /dev/null +++ b/src/app/invite/retry-deferred-rewards.ts @@ -0,0 +1,65 @@ +/** + * Operator reconciliation for referral rewards deferred by the ERPNext kill + * switch (see referral-settings.ts and award-referral-reward.ts). While the + * switch is off (or unreadable), a matching Bridge KYC approval leaves its + * invite ACCEPTED and unrewarded rather than paying or marking it failed — + * a deferred payout, not a lost one. This sweep finds that backlog and, + * unless dryRun, replays the award hook for every distinct account in it. + * + * Also usable in dry-run alone: an operator about to flip the switch back on + * can see the backlog size up front instead of relying on grepping logs for + * "deferred" lines. + */ +import { InviteStatus } from "@domain/invite" + +import { InviteRepository } from "@services/mongoose/models/invite" +import { baseLogger } from "@services/logger" + +import { awardReferralRewardOnKycApproval } from "./award-referral-reward" + +export type RetryDeferredReferralRewardsResult = { + backlogCount: number + accountsRetried: number +} + +/** + * awardReferralRewardOnKycApproval is itself idempotent and fail-closed — it + * re-checks the ERPNext switch, the already-processed guard, and the atomic + * reward claim — so re-invoking it here is always safe: a still-disabled + * switch or an already-claimed invite is a silent no-op, never a double-pay. + */ +export const retryDeferredReferralRewards = async ({ + dryRun, +}: { + dryRun: boolean +}): Promise => { + const deferred = await InviteRepository.find({ + status: InviteStatus.ACCEPTED, + rewardStatus: { $exists: false }, + redeemedById: { $exists: true }, + }) + + // Redemption history can hold more than one accepted invite for the same + // account, but the award hook only ever pays one — dedupe before retrying + // so the same account isn't looked up twice for nothing. + const accountIds = [ + ...new Set( + deferred + .map((invite) => invite.redeemedById?.toString()) + .filter((id): id is string => Boolean(id)), + ), + ] as AccountId[] + + baseLogger.info( + { backlogCount: deferred.length, accounts: accountIds.length, dryRun }, + "referral reward backlog: accepted + unrewarded invites found", + ) + + if (dryRun) return { backlogCount: deferred.length, accountsRetried: 0 } + + for (const accountId of accountIds) { + await awardReferralRewardOnKycApproval({ accountId }) + } + + return { backlogCount: deferred.length, accountsRetried: accountIds.length } +} diff --git a/src/scripts/retry-deferred-referral-rewards.ts b/src/scripts/retry-deferred-referral-rewards.ts new file mode 100644 index 000000000..89c3f5f01 --- /dev/null +++ b/src/scripts/retry-deferred-referral-rewards.ts @@ -0,0 +1,51 @@ +#!/usr/bin/env node + +/** + * Operator tool: replay referral rewards deferred by the ERPNext kill switch + * (src/app/invite/referral-settings.ts). Run with --dry-run right before + * flipping the switch back on to see the backlog size; drop --dry-run to + * actually replay it. See src/app/invite/retry-deferred-rewards.ts for why + * replaying is always safe (the award hook is idempotent and fail-closed). + * + * Usage: + * node lib/scripts/retry-deferred-referral-rewards.js \ + * --configPath dev/config/base-config.yaml \ + * [--dry-run] + */ + +import yargs from "yargs" +import { hideBin } from "yargs/helpers" +import { setupMongoConnection } from "@services/mongodb" +import { baseLogger } from "@services/logger" +import { retryDeferredReferralRewards } from "@app/invite/retry-deferred-rewards" + +const args = yargs(hideBin(process.argv)) + .option("dry-run", { + type: "boolean", + default: true, + describe: "Report the backlog size without paying anyone (default: true)", + }) + .option("configPath", { type: "string", demandOption: true }) + .parseSync() + +const main = async () => { + const dryRun = args["dry-run"] + const result = await retryDeferredReferralRewards({ dryRun }) + baseLogger.info( + { ...result, dryRun }, + dryRun + ? "Referral reward backlog check finished (dry run, nothing paid)" + : "Referral reward backlog replay finished", + ) +} + +setupMongoConnection() + .then(async (mongoose) => { + await main() + await mongoose?.connection.close() + process.exit(0) + }) + .catch((error) => { + baseLogger.error({ error }, "Referral reward backlog replay failed") + process.exit(1) + }) diff --git a/src/services/frappe/ErpNext.ts b/src/services/frappe/ErpNext.ts index 21e74f021..395f0cc24 100644 --- a/src/services/frappe/ErpNext.ts +++ b/src/services/frappe/ErpNext.ts @@ -467,31 +467,6 @@ export class ErpNext { // cashouts). The consumers read this through a 60s cache and fail open to a // 0% discount, so an ERPNext blip can never block a credit or an offer — // it just charges the standard fee. - - // Reads the "Referral Settings" Single doctype — the operator kill switch - // for referral reward payouts. Consulted before every payout (through a 60s - // cache in @app/invite/referral-settings); a read failure means "defer the - // payout", so this must return the error rather than a default. - async getReferralSettings(): Promise { - try { - const resp = await axios.get( - `${this.url}/api/resource/${encodeURIComponent("Referral Settings")}/${encodeURIComponent("Referral Settings")}`, - { headers: this.headers }, - ) - const data = resp.data?.data - if (!data) - return new ReferralSettingsQueryError("No data in Referral Settings response") - return data as ReferralSettingsDoc - } catch (err) { - const responseData = isAxiosError(err) ? err.response?.data : undefined - baseLogger.error( - { err, responseData }, - "Error querying Referral Settings from ERPNext", - ) - return new ReferralSettingsQueryError(err) - } - } - async getFeeDiscounts(): Promise { try { // Serialized through axios `params` (not hand-interpolated into the URL) @@ -528,6 +503,30 @@ export class ErpNext { } } + // Reads the "Referral Settings" Single doctype — the operator kill switch + // for referral reward payouts. Consulted before every payout (through a 60s + // cache in @app/invite/referral-settings); a read failure means "defer the + // payout", so this must return the error rather than a default. + async getReferralSettings(): Promise { + try { + const resp = await axios.get( + `${this.url}/api/resource/${encodeURIComponent("Referral Settings")}/${encodeURIComponent("Referral Settings")}`, + { headers: this.headers }, + ) + const data = resp.data?.data + if (!data) + return new ReferralSettingsQueryError("No data in Referral Settings response") + return data as ReferralSettingsDoc + } catch (err) { + const responseData = isAxiosError(err) ? err.response?.data : undefined + baseLogger.error( + { err, responseData }, + "Error querying Referral Settings from ERPNext", + ) + return new ReferralSettingsQueryError(err) + } + } + // Sums the GROSS USD cents of one account's Fygaro card top-ups over a // trailing window, for the per-level daily top-up limit gate. Counts every // captured USD payment (Fiat Received or Completed — i.e. the card was diff --git a/test/flash/unit/app/invite/retry-deferred-rewards.spec.ts b/test/flash/unit/app/invite/retry-deferred-rewards.spec.ts new file mode 100644 index 000000000..25dbe0e5a --- /dev/null +++ b/test/flash/unit/app/invite/retry-deferred-rewards.spec.ts @@ -0,0 +1,87 @@ +/** + * Reconciliation sweep for referral rewards deferred by the ERPNext kill + * switch. Covers the finding that no such sweep (nor an operator-visible + * backlog count) existed despite the original PR's ops notes claiming one. + */ +const mockFind = jest.fn() +jest.mock("@services/mongoose/models/invite", () => { + const actual = jest.requireActual("@services/mongoose/models/invite") + return { + InviteStatus: actual.InviteStatus, + InviteRepository: { + find: (...a: unknown[]) => mockFind(...a), + }, + } +}) + +const mockAward = jest.fn() +jest.mock("@app/invite/award-referral-reward", () => ({ + awardReferralRewardOnKycApproval: (...a: unknown[]) => mockAward(...a), +})) + +jest.mock("@services/logger", () => ({ + baseLogger: { info: jest.fn(), error: jest.fn(), warn: jest.fn() }, +})) + +import { InviteStatus } from "@domain/invite" +import { retryDeferredReferralRewards } from "@app/invite/retry-deferred-rewards" +import { baseLogger } from "@services/logger" + +const deferredInvite = (redeemedById: string) => ({ + _id: `invite-${redeemedById}`, + redeemedById: { toString: () => redeemedById }, +}) + +describe("retryDeferredReferralRewards", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + it("queries the exact deferred-payout shape: ACCEPTED, unrewarded, redeemed", async () => { + mockFind.mockResolvedValue([]) + await retryDeferredReferralRewards({ dryRun: true }) + expect(mockFind).toHaveBeenCalledWith({ + status: InviteStatus.ACCEPTED, + rewardStatus: { $exists: false }, + redeemedById: { $exists: true }, + }) + }) + + it("dry-run reports the backlog size without paying anyone", async () => { + mockFind.mockResolvedValue([deferredInvite("acct-1"), deferredInvite("acct-2")]) + const result = await retryDeferredReferralRewards({ dryRun: true }) + expect(result).toEqual({ backlogCount: 2, accountsRetried: 0 }) + expect(mockAward).not.toHaveBeenCalled() + expect(baseLogger.info).toHaveBeenCalledWith( + expect.objectContaining({ backlogCount: 2, accounts: 2, dryRun: true }), + expect.stringContaining("backlog"), + ) + }) + + it("replays the award hook once per distinct account, deduping repeat invites", async () => { + mockFind.mockResolvedValue([ + deferredInvite("acct-1"), + deferredInvite("acct-1"), // same account, second accepted invite + deferredInvite("acct-2"), + ]) + const result = await retryDeferredReferralRewards({ dryRun: false }) + expect(mockAward).toHaveBeenCalledTimes(2) + expect(mockAward).toHaveBeenCalledWith({ accountId: "acct-1" }) + expect(mockAward).toHaveBeenCalledWith({ accountId: "acct-2" }) + expect(result).toEqual({ backlogCount: 3, accountsRetried: 2 }) + }) + + it("no-ops cleanly when there is no backlog", async () => { + mockFind.mockResolvedValue([]) + const result = await retryDeferredReferralRewards({ dryRun: false }) + expect(mockAward).not.toHaveBeenCalled() + expect(result).toEqual({ backlogCount: 0, accountsRetried: 0 }) + }) + + it("skips invites with no redeemedById rather than retrying undefined", async () => { + mockFind.mockResolvedValue([{ _id: "invite-x", redeemedById: undefined }]) + const result = await retryDeferredReferralRewards({ dryRun: false }) + expect(mockAward).not.toHaveBeenCalled() + expect(result).toEqual({ backlogCount: 1, accountsRetried: 0 }) + }) +}) From 783865ce17cd1ccd3f416e5f0cd932cd9834e09a Mon Sep 17 00:00:00 2001 From: Dread Date: Wed, 2 Sep 2026 14:23:16 -0700 Subject: [PATCH 4/4] fix: correct dry-run default and add tracing to referral kill-switch reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code-review fixes for PR #501: - retry-deferred-referral-rewards.ts defaulted --dry-run to true while its own usage doc said "drop --dry-run to actually replay it" — omitting the flag left an operator's documented command paying nobody. Flip the default to false, matching every other operator sweep script in this repo (replay-bridge-events.ts, cash-wallet-cutover.ts) and the fact that replaying is always safe (award hook is idempotent/fail-closed). Extract buildArgs() and guard execution behind require.main so the default is unit-testable against the real yargs parser. - ErpNext.getReferralSettings()'s catch block logged the error but never called recordExceptionInCurrentSpan, unlike every other query method in the class. This is the kill-switch reader consulted on every KYC approval, so losing APM visibility into its failures is a real gap during an ERP outage. Add the missing call, matching getFygaroSettings/getFeeDiscounts. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01EtQzyzQ38thfC2F8BLK4u8 --- .../retry-deferred-referral-rewards.ts | 49 ++++++++++-------- src/services/frappe/ErpNext.ts | 4 ++ .../retry-deferred-referral-rewards.spec.ts | 42 +++++++++++++++ .../unit/services/frappe/ErpNext.spec.ts | 51 ++++++++++++++++++- 4 files changed, 123 insertions(+), 23 deletions(-) create mode 100644 test/flash/unit/scripts/retry-deferred-referral-rewards.spec.ts diff --git a/src/scripts/retry-deferred-referral-rewards.ts b/src/scripts/retry-deferred-referral-rewards.ts index 89c3f5f01..a052bc8cb 100644 --- a/src/scripts/retry-deferred-referral-rewards.ts +++ b/src/scripts/retry-deferred-referral-rewards.ts @@ -2,10 +2,11 @@ /** * Operator tool: replay referral rewards deferred by the ERPNext kill switch - * (src/app/invite/referral-settings.ts). Run with --dry-run right before - * flipping the switch back on to see the backlog size; drop --dry-run to - * actually replay it. See src/app/invite/retry-deferred-rewards.ts for why - * replaying is always safe (the award hook is idempotent and fail-closed). + * (src/app/invite/referral-settings.ts). Run with --dry-run to see the + * backlog size without paying anyone (e.g. right before flipping the switch + * back on); omit --dry-run (default: false) to actually replay it. See + * src/app/invite/retry-deferred-rewards.ts for why replaying is always safe + * (the award hook is idempotent and fail-closed). * * Usage: * node lib/scripts/retry-deferred-referral-rewards.js \ @@ -19,16 +20,18 @@ import { setupMongoConnection } from "@services/mongodb" import { baseLogger } from "@services/logger" import { retryDeferredReferralRewards } from "@app/invite/retry-deferred-rewards" -const args = yargs(hideBin(process.argv)) - .option("dry-run", { - type: "boolean", - default: true, - describe: "Report the backlog size without paying anyone (default: true)", - }) - .option("configPath", { type: "string", demandOption: true }) - .parseSync() +export const buildArgs = (argv: string[]) => + yargs(argv) + .option("dry-run", { + type: "boolean" as const, + default: false, + describe: "Report the backlog size without paying anyone (default: false)", + }) + .option("configPath", { type: "string" as const, demandOption: true }) + .parseSync() const main = async () => { + const args = buildArgs(hideBin(process.argv)) const dryRun = args["dry-run"] const result = await retryDeferredReferralRewards({ dryRun }) baseLogger.info( @@ -39,13 +42,15 @@ const main = async () => { ) } -setupMongoConnection() - .then(async (mongoose) => { - await main() - await mongoose?.connection.close() - process.exit(0) - }) - .catch((error) => { - baseLogger.error({ error }, "Referral reward backlog replay failed") - process.exit(1) - }) +if (require.main === module) { + setupMongoConnection() + .then(async (mongoose) => { + await main() + await mongoose?.connection.close() + process.exit(0) + }) + .catch((error) => { + baseLogger.error({ error }, "Referral reward backlog replay failed") + process.exit(1) + }) +} diff --git a/src/services/frappe/ErpNext.ts b/src/services/frappe/ErpNext.ts index 395f0cc24..fb721338f 100644 --- a/src/services/frappe/ErpNext.ts +++ b/src/services/frappe/ErpNext.ts @@ -523,6 +523,10 @@ export class ErpNext { { err, responseData }, "Error querying Referral Settings from ERPNext", ) + recordExceptionInCurrentSpan({ + error: err, + attributes: { "erpnext.exception": responseData?.exception }, + }) return new ReferralSettingsQueryError(err) } } diff --git a/test/flash/unit/scripts/retry-deferred-referral-rewards.spec.ts b/test/flash/unit/scripts/retry-deferred-referral-rewards.spec.ts new file mode 100644 index 000000000..8eef8ed63 --- /dev/null +++ b/test/flash/unit/scripts/retry-deferred-referral-rewards.spec.ts @@ -0,0 +1,42 @@ +/** + * CLI arg-parsing coverage for the referral-reward backlog replay script. + * + * Covers the finding that --dry-run defaulted to true while the script's own + * usage doc said "drop --dry-run to actually replay it": an operator running + * the documented command with no flags paid nobody and had to notice a log + * line to find out. + * + * yargs is globally mocked in test/flash/unit/jest.setup.ts (for config + * loading in other suites); this file un-mocks it back to the real library + * so buildArgs is exercised against real option-default resolution instead + * of a stub. + */ +jest.mock("yargs", () => jest.requireActual("yargs")) + +jest.mock("@services/mongodb", () => ({ + setupMongoConnection: jest.fn(), +})) + +jest.mock("@services/logger", () => ({ + baseLogger: { info: jest.fn(), error: jest.fn(), warn: jest.fn() }, +})) + +jest.mock("@app/invite/retry-deferred-rewards", () => ({ + retryDeferredReferralRewards: jest.fn(), +})) + +import { buildArgs } from "../../../../src/scripts/retry-deferred-referral-rewards" + +describe("retry-deferred-referral-rewards buildArgs", () => { + it("defaults dry-run to false when the flag is omitted, matching the usage doc", () => { + const args = buildArgs(["--configPath", "dev/config/base-config.yaml"]) + + expect(args["dry-run"]).toBe(false) + }) + + it("still honors --dry-run to report the backlog without paying anyone", () => { + const args = buildArgs(["--configPath", "dev/config/base-config.yaml", "--dry-run"]) + + expect(args["dry-run"]).toBe(true) + }) +}) diff --git a/test/flash/unit/services/frappe/ErpNext.spec.ts b/test/flash/unit/services/frappe/ErpNext.spec.ts index d5b467069..b1de4b5df 100644 --- a/test/flash/unit/services/frappe/ErpNext.spec.ts +++ b/test/flash/unit/services/frappe/ErpNext.spec.ts @@ -18,8 +18,12 @@ jest.mock("@config", () => ({ })) import axios from "axios" +import { recordExceptionInCurrentSpan } from "@services/tracing" import { ErpNext } from "@services/frappe/ErpNext" -import { FeeDiscountQueryError } from "@services/frappe/errors" +import { + FeeDiscountQueryError, + ReferralSettingsQueryError, +} from "@services/frappe/errors" import { BridgeTransferRequest, BridgeTransferRequestStatus, @@ -34,6 +38,8 @@ const mockedAxios = axios as unknown as { put: jest.Mock } +const mockedRecordExceptionInCurrentSpan = recordExceptionInCurrentSpan as jest.Mock + const client = new ErpNext("https://erp.example", "erp.example", { apiKey: "key", apiSecret: "secret", @@ -954,3 +960,46 @@ describe("ErpNext.getFeeDiscounts", () => { expect(await client.getFeeDiscounts()).toBeInstanceOf(FeeDiscountQueryError) }) }) + +describe("ErpNext.getReferralSettings", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + it("returns the doc as-is for the caller to coerce", async () => { + mockedAxios.get.mockResolvedValue({ data: { data: { rewards_enabled: 1 } } }) + + expect(await client.getReferralSettings()).toEqual({ rewards_enabled: 1 }) + }) + + it("returns an error when the response has no data", async () => { + mockedAxios.get.mockResolvedValue({ data: {} }) + + expect(await client.getReferralSettings()).toBeInstanceOf(ReferralSettingsQueryError) + }) + + it("returns the error rather than throwing when the request rejects", async () => { + mockedAxios.get.mockRejectedValue(new Error("erpnext down")) + + expect(await client.getReferralSettings()).toBeInstanceOf(ReferralSettingsQueryError) + }) + + // Consulted before every payout — the kill-switch reader must feed the + // same APM/tracing visibility as every other query method in this class + // on failure, not just a log line, since it's exactly what's read during + // an ERP outage. + it("records the exception in the current span when the request rejects", async () => { + const err = { + isAxiosError: true, + response: { status: 500, data: { exception: "InternalServerError" } }, + } + mockedAxios.get.mockRejectedValue(err) + + await client.getReferralSettings() + + expect(mockedRecordExceptionInCurrentSpan).toHaveBeenCalledWith({ + error: err, + attributes: { "erpnext.exception": "InternalServerError" }, + }) + }) +})