diff --git a/src/graphql/error-map.ts b/src/graphql/error-map.ts index 76c08a923..cb702470b 100644 --- a/src/graphql/error-map.ts +++ b/src/graphql/error-map.ts @@ -108,6 +108,16 @@ export const mapError = (error: ApplicationError): CustomApolloError => { message = `Account does not exist for id ${error.message}` return new NotFoundError({ message, logger: baseLogger }) + // Same shape as the id/uuid/username siblings: a not-found for whichever + // user the caller looked up. Admin lookups (accountDetailsByUserPhone, + // accountDetailsByUserEmail) resolve OTHER users through findByUserId, so + // this must never read as "your session is unauthenticated". The one site + // that knows the missing account is the caller's own is the session + // middleware, which raises AuthenticationError itself. + case "CouldNotFindAccountFromKratosIdError": + message = `Account does not exist for user id ${error.message}` + return new NotFoundError({ message, logger: baseLogger }) + case "CouldNotFindAccountFromUuidError": message = `Account does not exist for uuid ${error.message}` return new NotFoundError({ message, logger: baseLogger }) @@ -855,7 +865,6 @@ export const mapError = (error: ApplicationError): CustomApolloError => { case "CouldNotFindTransactionMetadataError": case "CouldNotFindExpectedTransactionMetadataError": case "InvalidDocumentIdForDbError": - case "DuplicateKeyForPersistError": case "MismatchedResultForTransactionMetadataQuery": case "InvalidLedgerTransactionId": case "MultiplePendingPaymentsForHashError": @@ -894,7 +903,6 @@ export const mapError = (error: ApplicationError): CustomApolloError => { case "InvalidCurrencyBaseAmountError": case "NoTransactionToUpdateError": case "BalanceLessThanZeroError": - case "CouldNotFindAccountFromKratosIdError": case "MissingPhoneError": case "InvalidUserId": case "InvalidLightningPaymentFlowBuilderStateError": @@ -975,6 +983,13 @@ export const mapError = (error: ApplicationError): CustomApolloError => { }${error.message ? ": " + error.message : ""})` return new UnexpectedClientError({ message, logger: baseLogger }) + // parseRepositoryError keeps the Mongo driver text on this error for logs + // and spans. It names internal collections, indexes and the colliding + // value, so unlike the catch-all above the message is not interpolated. + case "DuplicateKeyForPersistError": + message = `Unexpected error occurred, please try again or contact support if it persists (code: ${error.name})` + return new UnexpectedClientError({ message, logger: baseLogger }) + case "MissingSessionIdError": case "AuthTokenUserIdMismatchError": return new UnexpectedClientError({ diff --git a/src/servers/graphql-main-server.ts b/src/servers/graphql-main-server.ts index 697fe0bf6..61c76e080 100644 --- a/src/servers/graphql-main-server.ts +++ b/src/servers/graphql-main-server.ts @@ -1,6 +1,6 @@ import { applyMiddleware } from "graphql-middleware" -import { GALOY_API_PORT, UNSECURE_IP_FROM_REQUEST_OBJECT } from "@config" +import { GALOY_API_PORT } from "@config" import { AuthorizationError } from "@graphql/error" import { gqlMainSchema, mutationFields, queryFields } from "@graphql/public" @@ -10,17 +10,10 @@ import { baseLogger } from "@services/logger" import { setupMongoConnection } from "@services/mongodb" import { and, shield } from "graphql-shield" import { ShieldRule } from "graphql-shield/typings/types" -import { - ACCOUNT_USERNAME, - SemanticAttributes, - addAttributesToCurrentSpanAndPropagate, -} from "@services/tracing" - -import { NextFunction, Request, Response } from "express" +import { recordExceptionInCurrentSpan } from "@services/tracing" -import { parseIps } from "@domain/accounts-ips" import { apiKeyNestedFieldScopes } from "@domain/api-keys" -import { parseCashWalletClientCapabilities } from "@app/cash-wallet-cutover/client-capability" +import { ErrorLevel } from "@domain/shared" import { startApiKeyMetricsServer } from "./api-key-metrics" import { startApolloServerForAdminSchema } from "./graphql-admin-server" @@ -30,53 +23,9 @@ import { scopedApiKeyTypeField, startApolloServer, } from "./graphql-server" +import { setGqlContext } from "./middlewares/gql-context" import { walletIdMiddleware } from "./middlewares/wallet-id" -import { sessionPublicContext } from "./middlewares/session" - -const setGqlContext = async ( - req: Request, - res: Response, - next: NextFunction, -): Promise => { - const tokenPayload = req.token - - const ipString = UNSECURE_IP_FROM_REQUEST_OBJECT - ? req.ip - : req.headers["x-real-ip"] || req.headers["x-forwarded-for"] - - const ip = parseIps(ipString) - - const gqlContext = await sessionPublicContext({ - tokenPayload, - ip, - }) - const cashWalletClientCapabilities = parseCashWalletClientCapabilities(req.headers) - - req.gqlContext = { - ...gqlContext, - cashWalletClientCapabilities, - } - - return addAttributesToCurrentSpanAndPropagate( - { - "token.iss": tokenPayload?.iss, - "token.session_id": tokenPayload?.session_id, - "token.expires_at": tokenPayload?.expires_at, - [SemanticAttributes.HTTP_CLIENT_IP]: ip, - [SemanticAttributes.HTTP_USER_AGENT]: req.headers["user-agent"], - [ACCOUNT_USERNAME]: gqlContext?.domainAccount?.username, - [SemanticAttributes.ENDUSER_ID]: tokenPayload?.sub, - "cash_wallet.client_presentation": - cashWalletClientCapabilities.cashWalletPresentation, - "cash_wallet.client_usdt_supported": String( - cashWalletClientCapabilities.hasUsdtCashWalletSupport, - ), - }, - next, - ) -} - export async function startApolloServerForCoreSchema() { const authedQueryFields: { [key: string]: ShieldRule } = {} for (const key of Object.keys({ @@ -135,6 +84,18 @@ export async function startApolloServerForCoreSchema() { } if (require.main === module) { + // A rejected promise nobody awaits must be logged, not fatal: Node's default + // `--unhandled-rejections=throw` exits the whole api replica on one stray + // rejection (see setGqlContext for the 2026-09-01 crash loop). + process.on("unhandledRejection", (reason) => { + baseLogger.error({ reason }, "unhandledRejection") + recordExceptionInCurrentSpan({ + error: reason, + level: ErrorLevel.Critical, + fallbackMsg: "unhandledRejection", + }) + }) + setupMongoConnection(true) .then(async () => { // activateLndHealthCheck() diff --git a/src/servers/middlewares/gql-context.ts b/src/servers/middlewares/gql-context.ts new file mode 100644 index 000000000..eb6796111 --- /dev/null +++ b/src/servers/middlewares/gql-context.ts @@ -0,0 +1,110 @@ +import { NextFunction, Request, Response } from "express" + +import { UNSECURE_IP_FROM_REQUEST_OBJECT } from "@config" + +import { parseCashWalletClientCapabilities } from "@app/cash-wallet-cutover/client-capability" +import { parseIps } from "@domain/accounts-ips" +import { ErrorLevel } from "@domain/shared" +import { AuthenticationError } from "@graphql/error" +import { baseLogger } from "@services/logger" +import { + ACCOUNT_USERNAME, + SemanticAttributes, + addAttributesToCurrentSpanAndPropagate, + recordExceptionInCurrentSpan, +} from "@services/tracing" + +import { sessionPublicContext } from "./session" + +// Express 4 does not catch a rejected async middleware. A throw while +// resolving the session used to surface as an unhandled rejection, and Node +// exits the process on those (`--unhandled-rejections=throw` is the default). +// On 2026-09-01 a single Kratos identity whose account write had failed was +// enough to crash every api replica on each request it made +// (CouldNotFindAccountFromKratosIdError out of sessionPublicContext). +// +// Nothing that happens while building the context may take the process down. +// A session that cannot be resolved is answered, not thrown. +// +// Response shape: same constraint as apiKeyRateLimitMiddleware — this /graphql +// server is a federation subgraph behind the Apollo router, which swallows any +// non-2xx subgraph response into an opaque SUBREQUEST_HTTP_ERROR. An +// unauthenticated session is therefore answered as HTTP 200 with a GraphQL +// error carrying a code the client can act on. Anything else is a genuine +// server fault and stays a 500. +export const setGqlContext = async ( + req: Request, + res: Response, + next: NextFunction, +): Promise => { + const tokenPayload = req.token + + const ipString = UNSECURE_IP_FROM_REQUEST_OBJECT + ? req.ip + : req.headers["x-real-ip"] || req.headers["x-forwarded-for"] + + const ip = parseIps(ipString) + + let gqlContext: Awaited> + try { + gqlContext = await sessionPublicContext({ + tokenPayload, + ip, + }) + } catch (err) { + const kratosUserId = tokenPayload?.sub + + if (err instanceof AuthenticationError) { + // The session resolved; it just has no usable account. Not a server + // fault — whatever made it unrepairable was recorded where the repair + // ran — and an orphan's pollers hit this on every request, so no + // Critical span here. + baseLogger.warn({ err, kratosUserId }, "unauthenticated session") + res.status(200).json({ + data: null, + errors: [ + { + message: err.message, + extensions: { code: err.extensions.code }, + }, + ], + }) + return + } + + baseLogger.error({ err, kratosUserId }, "failed to build graphql context") + recordExceptionInCurrentSpan({ + error: err, + level: ErrorLevel.Critical, + attributes: { kratosUserId }, + fallbackMsg: "failed to build graphql context", + }) + res.status(500).json({ error: "failed to build graphql context" }) + return + } + + const cashWalletClientCapabilities = parseCashWalletClientCapabilities(req.headers) + + req.gqlContext = { + ...gqlContext, + cashWalletClientCapabilities, + } + + return addAttributesToCurrentSpanAndPropagate( + { + "token.iss": tokenPayload?.iss, + "token.session_id": tokenPayload?.session_id, + "token.expires_at": tokenPayload?.expires_at, + [SemanticAttributes.HTTP_CLIENT_IP]: ip, + [SemanticAttributes.HTTP_USER_AGENT]: req.headers["user-agent"], + [ACCOUNT_USERNAME]: gqlContext?.domainAccount?.username, + [SemanticAttributes.ENDUSER_ID]: tokenPayload?.sub, + "cash_wallet.client_presentation": + cashWalletClientCapabilities.cashWalletPresentation, + "cash_wallet.client_usdt_supported": String( + cashWalletClientCapabilities.hasUsdtCashWalletSupport, + ), + }, + next, + ) +} diff --git a/src/servers/middlewares/session.ts b/src/servers/middlewares/session.ts index a7e7266db..17c6466f6 100644 --- a/src/servers/middlewares/session.ts +++ b/src/servers/middlewares/session.ts @@ -5,15 +5,229 @@ import { DEFAULT_CASH_WALLET_CLIENT_CAPABILITIES } from "@app/cash-wallet-cutove import { recordExceptionInCurrentSpan } from "@services/tracing" import jsonwebtoken from "jsonwebtoken" +import { getDefaultAccountsConfig } from "@config" + +import { AuthenticationError } from "@graphql/error" import { mapError } from "@graphql/error-map" +import { createAccountWithPhoneIdentifier } from "@app/accounts" import { maybeExtendSession } from "@app/authentication" import { checkedToUserId } from "@domain/accounts" -import { ValidationError } from "@domain/shared" +import { + CouldNotFindAccountFromKratosIdError, + DuplicateKeyForPersistError, +} from "@domain/errors" +import { ErrorLevel, ValidationError } from "@domain/shared" +import { IdentityRepository } from "@services/kratos" import { baseLogger } from "@services/logger" import { UsersRepository } from "@services/mongoose" +import { TwilioClient } from "@services/twilio" import { IbexError } from "@services/ibex/errors" +type SessionLogger = typeof baseLogger + +type OrphanRepairArgs = { + userId: UserId + orphanError: CouldNotFindAccountFromKratosIdError + logger: SessionLogger +} + +// The Kratos after-registration web_hook runs with `response.parse: false`: +// the identity is committed whether or not the api managed to write its +// account. A failed write (duplicate key, wallet provider down) therefore +// leaves a logged-in identity with no account, and every request it makes +// lands here. Re-run the registration write from the identity's own phone +// trait, including the Twilio carrier lookup the webhook path stores as +// `users.phoneMetadata`. If that is not possible the original not-found error +// stands and the caller answers it as "not authenticated" — it must never +// crash on it. +// +// Two pieces of replica-local state keep that from becoming a storm: +// +// - `inFlightRepairs` single-flights the repair per identity. An orphan +// opening the app fires several queries in parallel; `accounts.kratosUserId` +// is unique, so without this every request but one would lose the persistNew +// race and be answered NOT_AUTHENTICATED (with a Critical span) on the very +// launch that fixed the account. Requests landing mid-repair also never see +// the half-initialised account (no defaultWalletId yet) — they wait for the +// winner. A loser on ANOTHER replica is handled by re-reading the account +// after a duplicate-key failure. +// - `failedRepairs` negative-caches an identity that could not be repaired +// (phone collides on `users.phone`, no phone trait, Kratos unreachable). +// Without it every request from that identity costs a Kratos admin read plus +// two Mongo writes and emits a Critical span exception, and the mobile app's +// pollers keep firing on NOT_AUTHENTICATED. The window doubles on every +// consecutive failure (60s, 2m, 4m, ... capped at an hour) so an identity +// that will never repair costs one attempt an hour per replica, not one a +// minute. The first failure is an error log with a Critical span, later +// ones warn. +// - `carrierLookups` remembers the Twilio carrier lookup per identity so a +// retried repair does not bill Twilio again for the same number. +export const REPAIR_RETRY_WINDOW_MS = 60_000 +export const REPAIR_RETRY_MAX_MS = 60 * 60_000 + +const inFlightRepairs = new Map>() +const failedRepairs = new Map() +const carrierLookups = new Map() + +export const repairRetryDelayMs = (failures: number): number => + Math.min(REPAIR_RETRY_MAX_MS, REPAIR_RETRY_WINDOW_MS * 2 ** Math.max(0, failures - 1)) + +// The maps above live for the process; specs reset them between cases. +export const clearOrphanRepairState = (): void => { + inFlightRepairs.clear() + failedRepairs.clear() + carrierLookups.clear() +} + +const recordRepairFailure = ({ + userId, + cause, + reason, + orphanError, + logger, +}: OrphanRepairArgs & { + cause: Error | undefined + reason: string +}): CouldNotFindAccountFromKratosIdError => { + const failures = (failedRepairs.get(userId)?.failures ?? 0) + 1 + const retryAfterMs = repairRetryDelayMs(failures) + failedRepairs.set(userId, { + retryAfter: Date.now() + retryAfterMs, + failures, + }) + + const isFirstFailure = failures === 1 + const attributes = { kratosUserId: userId, repairFailures: failures } + recordExceptionInCurrentSpan({ + error: cause ?? orphanError, + level: isFirstFailure ? ErrorLevel.Critical : ErrorLevel.Warn, + attributes, + }) + + const details = { err: cause, ...attributes, retryAfterMs } + const msg = `orphaned kratos identity: ${reason}` + if (isFirstFailure) { + logger.error(details, msg) + } else { + logger.warn(details, msg) + } + + return orphanError +} + +const attemptRepair = async ({ + userId, + orphanError, + logger, +}: OrphanRepairArgs): Promise => { + const identity = await IdentityRepository().getIdentity(userId) + if (identity instanceof Error) { + return recordRepairFailure({ + userId, + cause: identity, + reason: "could not load identity", + orphanError, + logger, + }) + } + + if (!identity.phone) { + return recordRepairFailure({ + userId, + cause: undefined, + reason: "no phone trait to repair from", + orphanError, + logger, + }) + } + + // The webhook path stores the Twilio carrier lookup on the users row + // (login.ts → transient_payload → RegistrationPayloadValidator). Rewards + // (add-earn) fail closed on a missing phoneMetadata, so a repair without it + // would leave the account permanently ineligible with nothing pointing at + // why. Best effort, as on the webhook path: a lookup failure must not fail + // the repair, it just gets the warn line below so the gap is attributable. + const cachedCarrier = carrierLookups.get(userId) + const carrier = cachedCarrier ?? (await TwilioClient().getCarrier(identity.phone)) + if (carrier instanceof Error) { + logger.warn( + { err: carrier, kratosUserId: userId }, + "orphaned kratos identity: carrier lookup failed, repairing without phone metadata", + ) + } else if (!cachedCarrier) { + carrierLookups.set(userId, carrier) + } + + const created = await createAccountWithPhoneIdentifier({ + newAccountInfo: { kratosUserId: userId, phone: identity.phone }, + config: getDefaultAccountsConfig(), + phoneMetadata: carrier instanceof Error ? undefined : carrier, + }) + + if (created instanceof DuplicateKeyForPersistError) { + // Lost the persistNew race to a request on another replica: its account + // is the one to use. If the account is still missing the collision was + // on users.phone instead, and the failure stands. + const existing = await Accounts.getAccountFromUserId(userId) + if (!(existing instanceof Error)) { + failedRepairs.delete(userId) + carrierLookups.delete(userId) + logger.warn( + { kratosUserId: userId, accountId: existing.id }, + "orphaned kratos identity repaired by a concurrent request", + ) + return existing + } + } + + if (created instanceof Error) { + return recordRepairFailure({ + userId, + cause: created, + reason: "repair failed", + orphanError, + logger, + }) + } + + failedRepairs.delete(userId) + carrierLookups.delete(userId) + logger.warn( + { kratosUserId: userId, accountId: created.id }, + "orphaned kratos identity repaired", + ) + return created +} + +const repairOrphanedIdentity = ({ + userId, + orphanError, + logger, +}: OrphanRepairArgs): Promise => { + const failed = failedRepairs.get(userId) + if (failed && Date.now() < failed.retryAfter) { + logger.debug( + { + kratosUserId: userId, + repairFailures: failed.failures, + retryAfter: new Date(failed.retryAfter), + }, + "orphaned kratos identity: repair skipped, last attempt failed recently", + ) + return Promise.resolve(orphanError) + } + + const inFlight = inFlightRepairs.get(userId) + if (inFlight) return inFlight + + const repair = attemptRepair({ userId, orphanError, logger }).finally(() => { + inFlightRepairs.delete(userId) + }) + inFlightRepairs.set(userId, repair) + return repair +} + export const sessionPublicContext = async ({ tokenPayload, ip, @@ -39,27 +253,41 @@ export const sessionPublicContext = async ({ if (!(maybeUserId instanceof ValidationError)) { const userId = maybeUserId - const account = await Accounts.getAccountFromUserId(userId) - if (account instanceof Error) { - throw mapError(account) - } else { - domainAccount = account - // not awaiting on purpose. just updating metadata - // TODO: look if this can be a source of memory leaks - Accounts.updateAccountIPsInfo({ - accountId: account.id, - ip, + let account = await Accounts.getAccountFromUserId(userId) + if (account instanceof CouldNotFindAccountFromKratosIdError) { + account = await repairOrphanedIdentity({ userId, orphanError: account, logger }) + } + if (account instanceof CouldNotFindAccountFromKratosIdError) { + // Only here is the missing account known to be the caller's own. The + // session is unusable until one exists, which the client reads as + // NOT_AUTHENTICATED rather than "unexpected error, please try again". + // Anywhere else (admin lookups of other users) the same domain error + // is a plain not-found — see mapError. + throw new AuthenticationError({ + message: "No account is linked to this session", logger, }) + } + if (account instanceof Error) { + throw mapError(account) + } - if (sessionId && expiresAt) { - maybeExtendSession({ sessionId, expiresAt }) - } + domainAccount = account + // not awaiting on purpose. just updating metadata + // TODO: look if this can be a source of memory leaks + Accounts.updateAccountIPsInfo({ + accountId: account.id, + ip, + logger, + }) - const userRes = await UsersRepository().findById(account.kratosUserId) - if (userRes instanceof Error) throw mapError(userRes) - user = userRes + if (sessionId && expiresAt) { + maybeExtendSession({ sessionId, expiresAt }) } + + const userRes = await UsersRepository().findById(account.kratosUserId) + if (userRes instanceof Error) throw mapError(userRes) + user = userRes } const loaders = { diff --git a/src/services/mongoose/utils.ts b/src/services/mongoose/utils.ts index 66ad1db3d..2410716b4 100644 --- a/src/services/mongoose/utils.ts +++ b/src/services/mongoose/utils.ts @@ -40,7 +40,10 @@ export const parseRepositoryError = (err: Error | string | unknown) => { return new InvalidDocumentIdForDbError() case match(KnownRepositoryErrorMessages.MongoDuplicateKeyForPersist): - return new DuplicateKeyForPersistError() + // Keep the driver message: it names the collection and index that + // collided, which is the only way to attribute a failed write after + // the fact (e.g. accounts.kratosUserId vs users.phone). + return new DuplicateKeyForPersistError(errMsg) default: return new UnknownRepositoryError(errMsg) diff --git a/test/flash/unit/graphql/admin/account-details-by-phone.spec.ts b/test/flash/unit/graphql/admin/account-details-by-phone.spec.ts new file mode 100644 index 000000000..2bb437d5b --- /dev/null +++ b/test/flash/unit/graphql/admin/account-details-by-phone.spec.ts @@ -0,0 +1,108 @@ +/** + * Schema-execution spec for the admin `accountDetailsByUserPhone` query, built + * the same way as account-details-by-npub.spec.ts: real field definition, real + * `Phone` scalar, real resolver, real error map; only the app layer mocked. + * + * The case that matters: an orphaned identity — `users` row written, `accounts` + * row missing — is exactly what an operator looks up by phone when reconciling + * a failed registration. The repository answers that lookup with + * CouldNotFindAccountFromKratosIdError, the same domain error the session + * middleware sees for the caller's OWN session. Here it must read as a + * not-found for the looked-up user, never as the operator's session being + * unauthenticated. + */ +const mockGetAccountByUserPhone = jest.fn() +const mockGetUser = jest.fn() + +jest.mock("@app", () => ({ + Admin: { + getAccountByUserPhone: (...args: unknown[]) => mockGetAccountByUserPhone(...args), + }, + Users: { + getUser: (...args: unknown[]) => mockGetUser(...args), + }, + Accounts: { getAccountCapabilities: jest.fn() }, + Wallets: { listWalletsByAccountId: jest.fn() }, + Merchants: { getMerchantsByUsername: jest.fn() }, +})) + +import { graphql, GraphQLSchema, GraphQLObjectType, GraphQLFieldConfig } from "graphql" + +import { CouldNotFindAccountFromKratosIdError } from "@domain/errors" +import AccountDetailsByUserPhoneQuery from "@graphql/admin/root/query/account-details-by-phone" + +const PHONE = "+18765550100" +const KRATOS_USER_ID = "ebbe2b32-9a2e-4c77-80e4-5d7347c024bb" + +const LOOKUP_QUERY = ` + query accountDetailsByUserPhone($phone: Phone!) { + accountDetailsByUserPhone(phone: $phone) { + username + level + } + } +` + +const adminSchema = () => + new GraphQLSchema({ + query: new GraphQLObjectType({ + name: "Query", + fields: { + accountDetailsByUserPhone: + AccountDetailsByUserPhoneQuery as unknown as GraphQLFieldConfig< + unknown, + unknown + >, + }, + }), + }) + +const account = { + id: "account-id", + uuid: "5a9f6f45-0a3a-4b0a-9f3e-1e0f9b1b1b1b", + username: "jaceth2009", + level: 1, + kratosUserId: KRATOS_USER_ID, +} + +describe("admin accountDetailsByUserPhone", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + it("resolves an account by phone", async () => { + mockGetAccountByUserPhone.mockResolvedValue(account) + + const result = await graphql({ + schema: adminSchema(), + source: LOOKUP_QUERY, + variableValues: { phone: PHONE }, + }) + + expect(result.errors).toBeUndefined() + expect(mockGetAccountByUserPhone).toHaveBeenCalledWith(PHONE) + expect(result.data?.accountDetailsByUserPhone).toEqual({ + username: "jaceth2009", + level: "ONE", + }) + }) + + it("reports an orphaned identity as a not-found for that user, not as the operator's session", async () => { + mockGetAccountByUserPhone.mockResolvedValue( + new CouldNotFindAccountFromKratosIdError(KRATOS_USER_ID), + ) + + const result = await graphql({ + schema: adminSchema(), + source: LOOKUP_QUERY, + variableValues: { phone: PHONE }, + }) + + expect(result.errors?.[0].extensions?.code).toBe("NOT_FOUND") + expect(result.errors?.[0].extensions?.code).not.toBe("NOT_AUTHENTICATED") + expect(result.errors?.[0].message).toBe( + `Account does not exist for user id ${KRATOS_USER_ID}`, + ) + expect(result.errors?.[0].message).not.toContain("session") + }) +}) diff --git a/test/flash/unit/graphql/error-map.spec.ts b/test/flash/unit/graphql/error-map.spec.ts index 85e8d37b2..cd2f4d025 100644 --- a/test/flash/unit/graphql/error-map.spec.ts +++ b/test/flash/unit/graphql/error-map.spec.ts @@ -8,9 +8,47 @@ import { } from "@services/bridge/errors" import { IbexError, InsufficientIbexBalance } from "@services/ibex/errors" import { PhoneCountryNotAllowedError } from "@domain/users/errors" -import { InvalidPhoneNumber } from "@domain/errors" +import { + CouldNotFindAccountFromKratosIdError, + DuplicateKeyForPersistError, + InvalidPhoneNumber, +} from "@domain/errors" describe("error-map", () => { + // The same domain error reaches mapError from admin lookups of OTHER users + // (accountDetailsByUserPhone / accountDetailsByUserEmail → findByUserId) and + // from the caller's own session. Here it is a plain not-found like its + // id/uuid siblings. The session middleware is the one site that answers it + // as NOT_AUTHENTICATED, because only it knows the account is the caller's + // own — an operator reconciling an orphan must not be told their session is + // broken. + it("maps CouldNotFindAccountFromKratosIdError to NOT_FOUND, never NOT_AUTHENTICATED", () => { + const kratosUserId = "ebbe2b32-9a2e-4c77-80e4-5d7347c024bb" + const result = mapError(new CouldNotFindAccountFromKratosIdError(kratosUserId)) + + expect(result.extensions.code).toBe("NOT_FOUND") + expect(result.message).toBe(`Account does not exist for user id ${kratosUserId}`) + expect(result.extensions.code).not.toBe("NOT_AUTHENTICATED") + expect(result.extensions.code).not.toBe("UNEXPECTED_CLIENT_ERROR") + }) + + // parseRepositoryError keeps the Mongo driver text on the domain error for + // logs and spans. It names internal collections, indexes and the colliding + // value (a setUsername that loses the race after its availability check is + // the everyday case), so it must not reach the client. + it("maps DuplicateKeyForPersistError without leaking the driver message", () => { + const driverMessage = + 'E11000 duplicate key error collection: galoy.accounts index: username_1 dup key: { username: "alice" }' + const result = mapError(new DuplicateKeyForPersistError(driverMessage)) + + expect(result.extensions.code).toBe("UNEXPECTED_CLIENT_ERROR") + expect(result.message).toContain("code: DuplicateKeyForPersistError") + expect(result.message).not.toContain("E11000") + expect(result.message).not.toContain("galoy.accounts") + expect(result.message).not.toContain("username_1") + expect(result.message).not.toContain("alice") + }) + it("maps BridgeWithdrawalNotFoundError to BRIDGE_WITHDRAWAL_NOT_FOUND", () => { const result = mapError(new BridgeWithdrawalNotFoundError()) diff --git a/test/flash/unit/servers/middlewares/gql-context.spec.ts b/test/flash/unit/servers/middlewares/gql-context.spec.ts new file mode 100644 index 000000000..9454a82a7 --- /dev/null +++ b/test/flash/unit/servers/middlewares/gql-context.spec.ts @@ -0,0 +1,146 @@ +import { NextFunction, Request, Response } from "express" + +import { AuthenticationError } from "@graphql/error" +import { setGqlContext } from "@servers/middlewares/gql-context" +import { sessionPublicContext } from "@servers/middlewares/session" +import { baseLogger } from "@services/logger" +import { recordExceptionInCurrentSpan } from "@services/tracing" + +jest.mock("@servers/middlewares/session", () => ({ + sessionPublicContext: jest.fn(), +})) + +jest.mock("@services/logger", () => ({ + baseLogger: { error: jest.fn(), warn: jest.fn(), info: jest.fn() }, +})) + +jest.mock("@services/tracing", () => ({ + ACCOUNT_USERNAME: "account.username", + SemanticAttributes: { + HTTP_CLIENT_IP: "http.client_ip", + HTTP_USER_AGENT: "http.user_agent", + ENDUSER_ID: "enduser.id", + }, + addAttributesToCurrentSpanAndPropagate: jest.fn((_attributes, fn) => fn()), + recordExceptionInCurrentSpan: jest.fn(), +})) + +const mockedSessionPublicContext = sessionPublicContext as jest.MockedFunction< + typeof sessionPublicContext +> +const mockedRecordException = recordExceptionInCurrentSpan as jest.MockedFunction< + typeof recordExceptionInCurrentSpan +> +const mockedLogger = baseLogger as unknown as { error: jest.Mock; warn: jest.Mock } + +const kratosUserId = "ebbe2b32-9a2e-4c77-80e4-5d7347c024bb" + +const makeReq = () => + ({ + token: { sub: kratosUserId, iss: "galoy.io" }, + headers: { "x-real-ip": "203.0.113.7", "user-agent": "jest" }, + }) as unknown as Request & { gqlContext?: unknown } + +const makeRes = () => { + const res = { + status: jest.fn(), + json: jest.fn(), + } + res.status.mockReturnValue(res) + return res as unknown as Response & { status: jest.Mock; json: jest.Mock } +} + +describe("setGqlContext", () => { + let next: jest.MockedFunction + + beforeEach(() => { + mockedSessionPublicContext.mockReset() + mockedRecordException.mockReset() + mockedLogger.error.mockReset() + mockedLogger.warn.mockReset() + next = jest.fn() + }) + + it("attaches the resolved context and continues the chain", async () => { + const context = { + domainAccount: { username: "alice" }, + user: { id: kratosUserId }, + } + mockedSessionPublicContext.mockResolvedValue( + context as unknown as Awaited>, + ) + const req = makeReq() + const res = makeRes() + + await setGqlContext(req, res, next) + + expect(next).toHaveBeenCalled() + expect(req.gqlContext).toEqual( + expect.objectContaining({ + domainAccount: context.domainAccount, + cashWalletClientCapabilities: expect.any(Object), + }), + ) + expect(res.status).not.toHaveBeenCalled() + }) + + // The 2026-09-01 crash loop: one Kratos identity with no account rejected + // out of the async middleware, Express never caught it, and Node exited the + // replica on the unhandled rejection. The middleware must resolve — never + // reject — and answer the session as unauthenticated. + it("answers an unauthenticated session as a GraphQL error and does not reject", async () => { + const authError = new AuthenticationError({ + message: "No account is linked to this session", + logger: baseLogger, + }) + mockedSessionPublicContext.mockRejectedValue(authError) + const req = makeReq() + const res = makeRes() + + await expect(setGqlContext(req, res, next)).resolves.toBeUndefined() + + expect(next).not.toHaveBeenCalled() + expect(req.gqlContext).toBeUndefined() + // HTTP 200 + GraphQL error: the federation router swallows non-2xx + // subgraph responses into an opaque SUBREQUEST_HTTP_ERROR. + expect(res.status).toHaveBeenCalledWith(200) + expect(res.json).toHaveBeenCalledWith({ + data: null, + errors: [ + { + message: "No account is linked to this session", + extensions: { code: "NOT_AUTHENTICATED" }, + }, + ], + }) + // Not a server fault: the repair site already recorded why the session + // has no account, and an orphan's pollers land here on every request. A + // warn line, no error line, no Critical span exception. + expect(mockedLogger.warn).toHaveBeenCalledWith( + expect.objectContaining({ err: authError, kratosUserId }), + "unauthenticated session", + ) + expect(mockedLogger.error).not.toHaveBeenCalled() + expect(mockedRecordException).not.toHaveBeenCalled() + }) + + it("answers any other failure as a 500 and does not reject", async () => { + const boom = new Error("kratos unreachable") + mockedSessionPublicContext.mockRejectedValue(boom) + const req = makeReq() + const res = makeRes() + + await expect(setGqlContext(req, res, next)).resolves.toBeUndefined() + + expect(next).not.toHaveBeenCalled() + expect(res.status).toHaveBeenCalledWith(500) + expect(res.json).toHaveBeenCalledWith({ error: "failed to build graphql context" }) + expect(mockedLogger.error).toHaveBeenCalledWith( + expect.objectContaining({ err: boom, kratosUserId }), + "failed to build graphql context", + ) + expect(mockedRecordException).toHaveBeenCalledWith( + expect.objectContaining({ error: boom, level: "critical" }), + ) + }) +}) diff --git a/test/flash/unit/servers/middlewares/session.spec.ts b/test/flash/unit/servers/middlewares/session.spec.ts new file mode 100644 index 000000000..3abfa85e8 --- /dev/null +++ b/test/flash/unit/servers/middlewares/session.spec.ts @@ -0,0 +1,526 @@ +import { Accounts } from "@app" +import { createAccountWithPhoneIdentifier } from "@app/accounts" +import { + CouldNotFindAccountFromKratosIdError, + DuplicateKeyForPersistError, + UnknownRepositoryError, +} from "@domain/errors" +import { UnknownPhoneProviderServiceError } from "@domain/phone-provider" +import { ErrorLevel } from "@domain/shared" +import { AuthenticationError } from "@graphql/error" +import { + clearOrphanRepairState, + REPAIR_RETRY_MAX_MS, + REPAIR_RETRY_WINDOW_MS, + repairRetryDelayMs, + sessionPublicContext, +} from "@servers/middlewares/session" +import { IdentityRepository, UnknownKratosError } from "@services/kratos" +import { baseLogger } from "@services/logger" +import { UsersRepository } from "@services/mongoose" +import { recordExceptionInCurrentSpan } from "@services/tracing" +import { TwilioClient } from "@services/twilio" + +jest.mock("@app", () => ({ + Accounts: { + getAccountFromUserId: jest.fn(), + updateAccountIPsInfo: jest.fn(), + }, + Transactions: { + getTransactionsMetadataByIds: jest.fn(), + }, +})) + +jest.mock("@app/accounts", () => ({ + createAccountWithPhoneIdentifier: jest.fn(), +})) + +jest.mock("@app/authentication", () => ({ + maybeExtendSession: jest.fn(), +})) + +jest.mock("@app/cash-wallet-cutover", () => ({ + DEFAULT_CASH_WALLET_CLIENT_CAPABILITIES: { + cashWalletPresentation: "legacy_compat", + hasUsdtCashWalletSupport: false, + }, +})) + +jest.mock("@services/kratos", () => ({ + IdentityRepository: jest.fn(), + UnknownKratosError: class UnknownKratosError extends Error {}, +})) + +jest.mock("@services/mongoose", () => ({ + UsersRepository: jest.fn(), +})) + +jest.mock("@services/tracing", () => ({ + recordExceptionInCurrentSpan: jest.fn(), +})) + +jest.mock("@services/twilio", () => ({ + TwilioClient: jest.fn(), +})) + +// The real error map runs (no mock): what the client is answered with is part +// of what is under test. CustomApolloError binds `logger[level]`, so every +// pino level it can name has to exist on the mock. +jest.mock("@services/logger", () => { + const child = { + fatal: jest.fn(), + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + trace: jest.fn(), + } + return { + baseLogger: { ...child, child: jest.fn(() => child) }, + } +}) + +const mockedGetAccount = Accounts.getAccountFromUserId as jest.MockedFunction< + typeof Accounts.getAccountFromUserId +> +const mockedCreateAccount = createAccountWithPhoneIdentifier as jest.MockedFunction< + typeof createAccountWithPhoneIdentifier +> +const mockedIdentityRepository = IdentityRepository as jest.MockedFunction< + typeof IdentityRepository +> +const mockedUsersRepository = UsersRepository as jest.MockedFunction< + typeof UsersRepository +> +const mockedRecordException = recordExceptionInCurrentSpan as jest.MockedFunction< + typeof recordExceptionInCurrentSpan +> +const mockedTwilioClient = TwilioClient as jest.MockedFunction +const childLogger = ( + baseLogger as unknown as { + child: () => { error: jest.Mock; warn: jest.Mock; debug: jest.Mock } + } +).child() + +const kratosUserId = "ebbe2b32-9a2e-4c77-80e4-5d7347c024bb" as UserId +const phone = "+2348012345678" as PhoneNumber +const ip = "203.0.113.7" as IpAddress + +const account = { + id: "6a972befef5ea964adc2548f", + kratosUserId, + username: "alice", +} as unknown as Account +const user = { id: kratosUserId, phone } as unknown as User + +// What the registration webhook stores on the users row from Twilio's carrier +// lookup (login.ts → transient_payload → RegistrationPayloadValidator). +const phoneMetadata = { + carrier: { + error_code: "", + mobile_country_code: "621", + mobile_network_code: "30", + name: "MTN Nigeria", + type: "mobile", + }, + countryCode: "NG", +} as PhoneMetadata + +const getIdentity = jest.fn() +const getCarrier = jest.fn() + +const tokenPayload = { sub: kratosUserId, session_id: "sess", expires_at: "later" } + +const flushMicrotasks = () => new Promise((resolve) => setImmediate(resolve)) + +// The caller's own session with no account is answered as NOT_AUTHENTICATED: +// the client must not retry it as a transient fault, and must not be told the +// account "does not exist for user id …" as an admin looking up someone else +// would be. +const expectNotAuthenticated = async (attempt: Promise) => { + await expect(attempt).rejects.toBeInstanceOf(AuthenticationError) + await expect(attempt).rejects.toMatchObject({ + message: "No account is linked to this session", + extensions: { code: "NOT_AUTHENTICATED" }, + }) +} + +describe("sessionPublicContext", () => { + beforeEach(() => { + clearOrphanRepairState() + mockedGetAccount.mockReset() + mockedCreateAccount.mockReset() + mockedRecordException.mockReset() + mockedIdentityRepository.mockReset() + mockedTwilioClient.mockReset() + getIdentity.mockReset() + getCarrier.mockReset() + childLogger.error.mockReset() + childLogger.warn.mockReset() + childLogger.debug.mockReset() + mockedIdentityRepository.mockReturnValue({ + getIdentity, + } as unknown as ReturnType) + mockedTwilioClient.mockReturnValue({ + getCarrier, + } as unknown as ReturnType) + mockedUsersRepository.mockReturnValue({ + findById: jest.fn().mockResolvedValue(user), + } as unknown as ReturnType) + }) + + afterEach(() => { + jest.useRealTimers() + }) + + it("resolves an existing account without touching kratos", async () => { + mockedGetAccount.mockResolvedValue(account) + + const context = await sessionPublicContext({ tokenPayload, ip }) + + expect(context.domainAccount).toBe(account) + expect(context.user).toBe(user) + expect(mockedIdentityRepository).not.toHaveBeenCalled() + expect(mockedCreateAccount).not.toHaveBeenCalled() + }) + + it("leaves an anonymous subject without an account and without lookups", async () => { + const context = await sessionPublicContext({ + tokenPayload: { sub: "anon" }, + ip, + }) + + expect(context.domainAccount).toBeUndefined() + expect(context.user).toBeUndefined() + expect(mockedGetAccount).not.toHaveBeenCalled() + }) + + it("maps any other lookup failure through the error map and does not try to repair", async () => { + mockedGetAccount.mockResolvedValue(new UnknownRepositoryError("mongo unreachable")) + + const attempt = sessionPublicContext({ tokenPayload, ip }) + + await expect(attempt).rejects.not.toBeInstanceOf(AuthenticationError) + await expect(attempt).rejects.toMatchObject({ extensions: { code: "DB_ERROR" } }) + expect(mockedIdentityRepository).not.toHaveBeenCalled() + expect(mockedCreateAccount).not.toHaveBeenCalled() + }) + + describe("orphaned kratos identity (identity committed, account write failed)", () => { + const orphan = new CouldNotFindAccountFromKratosIdError(kratosUserId) + + beforeEach(() => { + mockedGetAccount.mockResolvedValue(orphan) + getCarrier.mockResolvedValue(phoneMetadata) + }) + + it("repairs it from the identity's phone trait and continues as that account", async () => { + getIdentity.mockResolvedValue({ id: kratosUserId, phone }) + mockedCreateAccount.mockResolvedValue(account) + + const context = await sessionPublicContext({ tokenPayload, ip }) + + expect(mockedCreateAccount).toHaveBeenCalledWith({ + newAccountInfo: { kratosUserId, phone }, + config: expect.any(Object), + phoneMetadata, + }) + expect(context.domainAccount).toBe(account) + expect(context.user).toBe(user) + expect(childLogger.warn).toHaveBeenCalledWith( + expect.objectContaining({ kratosUserId, accountId: account.id }), + "orphaned kratos identity repaired", + ) + expect(mockedRecordException).not.toHaveBeenCalled() + }) + + // The webhook path stores Twilio's carrier lookup as users.phoneMetadata. + // Quiz rewards (add-earn → PhoneMetadataAuthorizer) fail closed when it is + // missing, so a repair that skipped the lookup would leave the account + // permanently ineligible for rewards with nothing pointing at why. + it("looks up the carrier and stores it with the repair, as the registration webhook would", async () => { + getIdentity.mockResolvedValue({ id: kratosUserId, phone }) + mockedCreateAccount.mockResolvedValue(account) + + await sessionPublicContext({ tokenPayload, ip }) + + expect(getCarrier).toHaveBeenCalledTimes(1) + expect(getCarrier).toHaveBeenCalledWith(phone) + expect(mockedCreateAccount).toHaveBeenCalledWith( + expect.objectContaining({ phoneMetadata }), + ) + }) + + it("still repairs when the carrier lookup fails, just without phone metadata", async () => { + getIdentity.mockResolvedValue({ id: kratosUserId, phone }) + const lookupFailed = new UnknownPhoneProviderServiceError("twilio lookups down") + getCarrier.mockResolvedValue(lookupFailed) + mockedCreateAccount.mockResolvedValue(account) + + const context = await sessionPublicContext({ tokenPayload, ip }) + + expect(context.domainAccount).toBe(account) + expect(context.user).toBe(user) + expect(mockedCreateAccount).toHaveBeenCalledWith({ + newAccountInfo: { kratosUserId, phone }, + config: expect.any(Object), + phoneMetadata: undefined, + }) + // Attributable, but not a failure: no error line, no span exception. + expect(childLogger.warn).toHaveBeenCalledWith( + expect.objectContaining({ err: lookupFailed, kratosUserId }), + "orphaned kratos identity: carrier lookup failed, repairing without phone metadata", + ) + expect(childLogger.warn).toHaveBeenCalledWith( + expect.objectContaining({ kratosUserId, accountId: account.id }), + "orphaned kratos identity repaired", + ) + expect(childLogger.error).not.toHaveBeenCalled() + expect(mockedRecordException).not.toHaveBeenCalled() + }) + + it("answers NOT_AUTHENTICATED when the identity has no phone trait", async () => { + getIdentity.mockResolvedValue({ id: kratosUserId, phone: undefined }) + + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + + expect(mockedCreateAccount).not.toHaveBeenCalled() + expect(mockedRecordException).toHaveBeenCalledWith( + expect.objectContaining({ + error: orphan, + level: ErrorLevel.Critical, + attributes: expect.objectContaining({ kratosUserId }), + }), + ) + }) + + it("answers NOT_AUTHENTICATED when the identity cannot be loaded", async () => { + const kratosDown = new UnknownKratosError("kratos down") + getIdentity.mockResolvedValue(kratosDown) + + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + + expect(mockedCreateAccount).not.toHaveBeenCalled() + expect(mockedRecordException).toHaveBeenCalledWith( + expect.objectContaining({ error: kratosDown, level: ErrorLevel.Critical }), + ) + }) + + it("answers NOT_AUTHENTICATED when the repair write fails, and records it as Critical", async () => { + getIdentity.mockResolvedValue({ id: kratosUserId, phone }) + // A collision on users.phone: the account is still missing afterwards, + // so this identity cannot be repaired by re-running registration. + const collision = new DuplicateKeyForPersistError( + "E11000 duplicate key error collection: galoy.users index: phone_1", + ) + mockedCreateAccount.mockResolvedValue(collision) + + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + + expect(mockedRecordException).toHaveBeenCalledWith( + expect.objectContaining({ + error: collision, + level: ErrorLevel.Critical, + attributes: expect.objectContaining({ kratosUserId }), + }), + ) + expect(childLogger.error).toHaveBeenCalledWith( + expect.objectContaining({ err: collision, kratosUserId }), + "orphaned kratos identity: repair failed", + ) + }) + + // An orphan opening the app fires several queries at once. accounts. + // kratosUserId is unique, so without single-flighting every request but + // one would lose the persistNew race and be answered NOT_AUTHENTICATED on + // the very launch that fixed the account. + it("single-flights concurrent requests: one repair, every request gets the account", async () => { + getIdentity.mockResolvedValue({ id: kratosUserId, phone }) + let releaseCreate: (value: Account) => void = () => undefined + mockedCreateAccount.mockReturnValue( + new Promise((resolve) => { + releaseCreate = resolve + }), + ) + + const first = sessionPublicContext({ tokenPayload, ip }) + const second = sessionPublicContext({ tokenPayload, ip }) + await flushMicrotasks() + + // Both requests are past the lookup and parked on the same repair. The + // carrier lookup is a billed Twilio call, so it is single-flighted too. + expect(mockedCreateAccount).toHaveBeenCalledTimes(1) + expect(getIdentity).toHaveBeenCalledTimes(1) + expect(getCarrier).toHaveBeenCalledTimes(1) + + releaseCreate(account) + const [one, two] = await Promise.all([first, second]) + + expect(one.domainAccount).toBe(account) + expect(two.domainAccount).toBe(account) + expect(mockedCreateAccount).toHaveBeenCalledTimes(1) + expect(mockedRecordException).not.toHaveBeenCalled() + }) + + it("adopts the account a request on another replica wrote after losing the persistNew race", async () => { + getIdentity.mockResolvedValue({ id: kratosUserId, phone }) + mockedCreateAccount.mockResolvedValue( + new DuplicateKeyForPersistError( + "E11000 duplicate key error collection: galoy.accounts index: kratosUserId_1", + ), + ) + mockedGetAccount.mockReset() + mockedGetAccount.mockResolvedValueOnce(orphan).mockResolvedValueOnce(account) + + const context = await sessionPublicContext({ tokenPayload, ip }) + + expect(context.domainAccount).toBe(account) + expect(context.user).toBe(user) + expect(mockedCreateAccount).toHaveBeenCalledTimes(1) + expect(mockedGetAccount).toHaveBeenCalledTimes(2) + expect(mockedRecordException).not.toHaveBeenCalled() + expect(childLogger.error).not.toHaveBeenCalled() + expect(childLogger.warn).toHaveBeenCalledWith( + expect.objectContaining({ kratosUserId, accountId: account.id }), + "orphaned kratos identity repaired by a concurrent request", + ) + }) + + // An identity that cannot be repaired keeps making requests (the mobile + // app's pollers do not stop on NOT_AUTHENTICATED). Each one must not cost + // a Kratos admin read plus Mongo writes, nor page anyone again. + it("does not retry a failed repair within the retry window, and warns rather than errors on the next failure", async () => { + const start = new Date("2026-09-01T12:00:00Z").getTime() + jest.useFakeTimers({ now: start }) + const kratosDown = new UnknownKratosError("kratos down") + getIdentity.mockResolvedValue(kratosDown) + + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + expect(getIdentity).toHaveBeenCalledTimes(1) + expect(childLogger.error).toHaveBeenCalledTimes(1) + expect(mockedRecordException).toHaveBeenCalledTimes(1) + + jest.setSystemTime(start + 30_000) + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + + // Still one Kratos round trip, one error line, one Critical span. + expect(getIdentity).toHaveBeenCalledTimes(1) + expect(childLogger.error).toHaveBeenCalledTimes(1) + expect(mockedRecordException).toHaveBeenCalledTimes(1) + expect(childLogger.debug).toHaveBeenCalledWith( + expect.objectContaining({ kratosUserId, repairFailures: 1 }), + "orphaned kratos identity: repair skipped, last attempt failed recently", + ) + + jest.setSystemTime(start + 61_000) + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + + // Window elapsed: one more attempt, reported at warn, not as Critical. + expect(getIdentity).toHaveBeenCalledTimes(2) + expect(childLogger.error).toHaveBeenCalledTimes(1) + expect(childLogger.warn).toHaveBeenCalledWith( + expect.objectContaining({ err: kratosDown, kratosUserId, repairFailures: 2 }), + "orphaned kratos identity: could not load identity", + ) + expect(mockedRecordException).toHaveBeenCalledTimes(2) + expect(mockedRecordException).toHaveBeenLastCalledWith( + expect.objectContaining({ error: kratosDown, level: ErrorLevel.Warn }), + ) + }) + + it("retries after the window and forgets the failure once the repair succeeds", async () => { + const start = new Date("2026-09-01T12:00:00Z").getTime() + jest.useFakeTimers({ now: start }) + getIdentity.mockResolvedValueOnce(new UnknownKratosError("kratos down")) + + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + + jest.setSystemTime(start + 61_000) + getIdentity.mockResolvedValue({ id: kratosUserId, phone }) + mockedCreateAccount.mockResolvedValue(account) + + const context = await sessionPublicContext({ tokenPayload, ip }) + + expect(context.domainAccount).toBe(account) + expect(mockedCreateAccount).toHaveBeenCalledTimes(1) + expect(childLogger.warn).toHaveBeenCalledWith( + expect.objectContaining({ kratosUserId, accountId: account.id }), + "orphaned kratos identity repaired", + ) + }) + + // An identity that will never repair (phone collides on users.phone, no + // phone trait) must not cost a Kratos read, two Mongo writes and a billed + // Twilio lookup every minute per replica for as long as its app polls. + it("doubles the retry window on every consecutive failure, capped at an hour", () => { + expect(repairRetryDelayMs(1)).toBe(REPAIR_RETRY_WINDOW_MS) + expect(repairRetryDelayMs(2)).toBe(REPAIR_RETRY_WINDOW_MS * 2) + expect(repairRetryDelayMs(3)).toBe(REPAIR_RETRY_WINDOW_MS * 4) + expect(repairRetryDelayMs(7)).toBe(REPAIR_RETRY_MAX_MS) // 64 min > cap + expect(repairRetryDelayMs(20)).toBe(REPAIR_RETRY_MAX_MS) + expect(repairRetryDelayMs(0)).toBe(REPAIR_RETRY_WINDOW_MS) + }) + + it("backs off: the second failure holds for two windows before the next attempt", async () => { + const start = new Date("2026-09-01T12:00:00Z").getTime() + jest.useFakeTimers({ now: start }) + getIdentity.mockResolvedValue(new UnknownKratosError("kratos down")) + + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + jest.setSystemTime(start + 61_000) + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + expect(getIdentity).toHaveBeenCalledTimes(2) + + // 90s after the second failure: inside its 120s window, no attempt. + jest.setSystemTime(start + 61_000 + 90_000) + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + expect(getIdentity).toHaveBeenCalledTimes(2) + expect(childLogger.debug).toHaveBeenLastCalledWith( + expect.objectContaining({ kratosUserId, repairFailures: 2 }), + "orphaned kratos identity: repair skipped, last attempt failed recently", + ) + + // 121s after the second failure: window elapsed, third attempt. + jest.setSystemTime(start + 61_000 + 121_000) + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + expect(getIdentity).toHaveBeenCalledTimes(3) + expect(childLogger.warn).toHaveBeenLastCalledWith( + expect.objectContaining({ + kratosUserId, + repairFailures: 3, + retryAfterMs: REPAIR_RETRY_WINDOW_MS * 4, + }), + "orphaned kratos identity: could not load identity", + ) + }) + + it("bills the carrier lookup once per identity across repair attempts", async () => { + const start = new Date("2026-09-01T12:00:00Z").getTime() + jest.useFakeTimers({ now: start }) + getIdentity.mockResolvedValue({ id: kratosUserId, phone }) + // users.phone collision: the account stays missing, so the repair fails. + mockedCreateAccount.mockResolvedValue( + new DuplicateKeyForPersistError("E11000 users.phone"), + ) + + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + expect(getCarrier).toHaveBeenCalledTimes(1) + + jest.setSystemTime(start + 61_000) + await expectNotAuthenticated(sessionPublicContext({ tokenPayload, ip })) + expect(mockedCreateAccount).toHaveBeenCalledTimes(2) + expect(getCarrier).toHaveBeenCalledTimes(1) + expect(mockedCreateAccount).toHaveBeenLastCalledWith( + expect.objectContaining({ phoneMetadata }), + ) + + // Once the repair succeeds the cached lookup is dropped with the failure record. + jest.setSystemTime(start + 61_000 + 121_000) + mockedCreateAccount.mockResolvedValue(account) + const context = await sessionPublicContext({ tokenPayload, ip }) + expect(context.domainAccount).toBe(account) + expect(getCarrier).toHaveBeenCalledTimes(1) + }) + }) +}) diff --git a/test/flash/unit/services/mongoose/parse-repository-error.spec.ts b/test/flash/unit/services/mongoose/parse-repository-error.spec.ts new file mode 100644 index 000000000..29e0b6029 --- /dev/null +++ b/test/flash/unit/services/mongoose/parse-repository-error.spec.ts @@ -0,0 +1,23 @@ +import { DuplicateKeyForPersistError, UnknownRepositoryError } from "@domain/errors" +import { parseRepositoryError } from "@services/mongoose/utils" + +describe("parseRepositoryError", () => { + // The driver message is the only thing that names the collection and index + // that collided. Dropping it left 26 failed registration writes on + // 2026-09-01 unattributable between accounts.kratosUserId and users.phone. + it("keeps the driver message on a duplicate key error", () => { + const driverMessage = + 'E11000 duplicate key error collection: galoy.users index: phone_1 dup key: { phone: "+2348012345678" }' + + const result = parseRepositoryError(new Error(driverMessage)) + + expect(result).toBeInstanceOf(DuplicateKeyForPersistError) + expect(result.message).toContain("galoy.users index: phone_1") + }) + + it("still falls through to the unknown repository error otherwise", () => { + const result = parseRepositoryError(new Error("something else entirely")) + + expect(result).toBeInstanceOf(UnknownRepositoryError) + }) +})