fix(auth): never crash the api on an orphaned kratos identity; self-heal missing accounts

src/graphql/error-map.ts

@@ -108,6 +108,16 @@ export const mapError = (error: ApplicationError): CustomApolloError => {
       message = `Account does not exist for id ${error.message}`
       return new NotFoundError({ message, logger: baseLogger })
 
+    // Same shape as the id/uuid/username siblings: a not-found for whichever
+    // user the caller looked up. Admin lookups (accountDetailsByUserPhone,
+    // accountDetailsByUserEmail) resolve OTHER users through findByUserId, so
+    // this must never read as "your session is unauthenticated". The one site
+    // that knows the missing account is the caller's own is the session
+    // middleware, which raises AuthenticationError itself.
+    case "CouldNotFindAccountFromKratosIdError":
+      message = `Account does not exist for user id ${error.message}`
+      return new NotFoundError({ message, logger: baseLogger })
+
     case "CouldNotFindAccountFromUuidError":
       message = `Account does not exist for uuid ${error.message}`
       return new NotFoundError({ message, logger: baseLogger })
@@ -855,7 +865,6 @@ export const mapError = (error: ApplicationError): CustomApolloError => {
     case "CouldNotFindTransactionMetadataError":
     case "CouldNotFindExpectedTransactionMetadataError":
     case "InvalidDocumentIdForDbError":
-    case "DuplicateKeyForPersistError":
     case "MismatchedResultForTransactionMetadataQuery":
     case "InvalidLedgerTransactionId":
     case "MultiplePendingPaymentsForHashError":
@@ -894,7 +903,6 @@ export const mapError = (error: ApplicationError): CustomApolloError => {
     case "InvalidCurrencyBaseAmountError":
     case "NoTransactionToUpdateError":
     case "BalanceLessThanZeroError":
-    case "CouldNotFindAccountFromKratosIdError":
     case "MissingPhoneError":
     case "InvalidUserId":
     case "InvalidLightningPaymentFlowBuilderStateError":
@@ -975,6 +983,13 @@ export const mapError = (error: ApplicationError): CustomApolloError => {
       }${error.message ? ": " + error.message : ""})`
       return new UnexpectedClientError({ message, logger: baseLogger })
 
+    // parseRepositoryError keeps the Mongo driver text on this error for logs
+    // and spans. It names internal collections, indexes and the colliding
+    // value, so unlike the catch-all above the message is not interpolated.
+    case "DuplicateKeyForPersistError":
+      message = `Unexpected error occurred, please try again or contact support if it persists (code: ${error.name})`
+      return new UnexpectedClientError({ message, logger: baseLogger })
+
     case "MissingSessionIdError":
     case "AuthTokenUserIdMismatchError":
       return new UnexpectedClientError({

src/servers/graphql-main-server.ts

@@ -1,6 +1,6 @@
 import { applyMiddleware } from "graphql-middleware"
 
-import { GALOY_API_PORT, UNSECURE_IP_FROM_REQUEST_OBJECT } from "@config"
+import { GALOY_API_PORT } from "@config"
 
 import { AuthorizationError } from "@graphql/error"
 import { gqlMainSchema, mutationFields, queryFields } from "@graphql/public"
@@ -10,17 +10,10 @@ import { baseLogger } from "@services/logger"
 import { setupMongoConnection } from "@services/mongodb"
 import { and, shield } from "graphql-shield"
 import { ShieldRule } from "graphql-shield/typings/types"
-import {
-  ACCOUNT_USERNAME,
-  SemanticAttributes,
-  addAttributesToCurrentSpanAndPropagate,
-} from "@services/tracing"
-
-import { NextFunction, Request, Response } from "express"
+import { recordExceptionInCurrentSpan } from "@services/tracing"
 
-import { parseIps } from "@domain/accounts-ips"
 import { apiKeyNestedFieldScopes } from "@domain/api-keys"
-import { parseCashWalletClientCapabilities } from "@app/cash-wallet-cutover/client-capability"
+import { ErrorLevel } from "@domain/shared"
 
 import { startApiKeyMetricsServer } from "./api-key-metrics"
 import { startApolloServerForAdminSchema } from "./graphql-admin-server"
@@ -30,53 +23,9 @@ import {
   scopedApiKeyTypeField,
   startApolloServer,
 } from "./graphql-server"
+import { setGqlContext } from "./middlewares/gql-context"
 import { walletIdMiddleware } from "./middlewares/wallet-id"
 
-import { sessionPublicContext } from "./middlewares/session"
-
-const setGqlContext = async (
-  req: Request,
-  res: Response,
-  next: NextFunction,
-): Promise<void> => {
-  const tokenPayload = req.token
-
-  const ipString = UNSECURE_IP_FROM_REQUEST_OBJECT
-    ? req.ip
-    : req.headers["x-real-ip"] || req.headers["x-forwarded-for"]
-
-  const ip = parseIps(ipString)
-
-  const gqlContext = await sessionPublicContext({
-    tokenPayload,
-    ip,
-  })
-  const cashWalletClientCapabilities = parseCashWalletClientCapabilities(req.headers)
-
-  req.gqlContext = {
-    ...gqlContext,
-    cashWalletClientCapabilities,
-  }
-
-  return addAttributesToCurrentSpanAndPropagate(
-    {
-      "token.iss": tokenPayload?.iss,
-      "token.session_id": tokenPayload?.session_id,
-      "token.expires_at": tokenPayload?.expires_at,
-      [SemanticAttributes.HTTP_CLIENT_IP]: ip,
-      [SemanticAttributes.HTTP_USER_AGENT]: req.headers["user-agent"],
-      [ACCOUNT_USERNAME]: gqlContext?.domainAccount?.username,
-      [SemanticAttributes.ENDUSER_ID]: tokenPayload?.sub,
-      "cash_wallet.client_presentation":
-        cashWalletClientCapabilities.cashWalletPresentation,
-      "cash_wallet.client_usdt_supported": String(
-        cashWalletClientCapabilities.hasUsdtCashWalletSupport,
-      ),
-    },
-    next,
-  )
-}
-
 export async function startApolloServerForCoreSchema() {
   const authedQueryFields: { [key: string]: ShieldRule } = {}
   for (const key of Object.keys({
@@ -135,6 +84,18 @@ export async function startApolloServerForCoreSchema() {
 }
 
 if (require.main === module) {
+  // A rejected promise nobody awaits must be logged, not fatal: Node's default
+  // `--unhandled-rejections=throw` exits the whole api replica on one stray
+  // rejection (see setGqlContext for the 2026-09-01 crash loop).
+  process.on("unhandledRejection", (reason) => {
+    baseLogger.error({ reason }, "unhandledRejection")
+    recordExceptionInCurrentSpan({
+      error: reason,
+      level: ErrorLevel.Critical,
+      fallbackMsg: "unhandledRejection",
+    })
+  })
+
   setupMongoConnection(true)
     .then(async () => {
       // activateLndHealthCheck()

src/servers/middlewares/gql-context.ts

@@ -0,0 +1,110 @@
+import { NextFunction, Request, Response } from "express"
+
+import { UNSECURE_IP_FROM_REQUEST_OBJECT } from "@config"
+
+import { parseCashWalletClientCapabilities } from "@app/cash-wallet-cutover/client-capability"
+import { parseIps } from "@domain/accounts-ips"
+import { ErrorLevel } from "@domain/shared"
+import { AuthenticationError } from "@graphql/error"
+import { baseLogger } from "@services/logger"
+import {
+  ACCOUNT_USERNAME,
+  SemanticAttributes,
+  addAttributesToCurrentSpanAndPropagate,
+  recordExceptionInCurrentSpan,
+} from "@services/tracing"
+
+import { sessionPublicContext } from "./session"
+
+// Express 4 does not catch a rejected async middleware. A throw while
+// resolving the session used to surface as an unhandled rejection, and Node
+// exits the process on those (`--unhandled-rejections=throw` is the default).
+// On 2026-09-01 a single Kratos identity whose account write had failed was
+// enough to crash every api replica on each request it made
+// (CouldNotFindAccountFromKratosIdError out of sessionPublicContext).
+//
+// Nothing that happens while building the context may take the process down.
+// A session that cannot be resolved is answered, not thrown.
+//
+// Response shape: same constraint as apiKeyRateLimitMiddleware — this /graphql
+// server is a federation subgraph behind the Apollo router, which swallows any
+// non-2xx subgraph response into an opaque SUBREQUEST_HTTP_ERROR. An
+// unauthenticated session is therefore answered as HTTP 200 with a GraphQL
+// error carrying a code the client can act on. Anything else is a genuine
+// server fault and stays a 500.
+export const setGqlContext = async (
+  req: Request,
+  res: Response,
+  next: NextFunction,
+): Promise<void> => {
+  const tokenPayload = req.token
+
+  const ipString = UNSECURE_IP_FROM_REQUEST_OBJECT
+    ? req.ip
+    : req.headers["x-real-ip"] || req.headers["x-forwarded-for"]
+
+  const ip = parseIps(ipString)
+
+  let gqlContext: Awaited<ReturnType<typeof sessionPublicContext>>
+  try {
+    gqlContext = await sessionPublicContext({
+      tokenPayload,
+      ip,
+    })
+  } catch (err) {
+    const kratosUserId = tokenPayload?.sub
+
+    if (err instanceof AuthenticationError) {
+      // The session resolved; it just has no usable account. Not a server
+      // fault — whatever made it unrepairable was recorded where the repair
+      // ran — and an orphan's pollers hit this on every request, so no
+      // Critical span here.
+      baseLogger.warn({ err, kratosUserId }, "unauthenticated session")
+      res.status(200).json({
+        data: null,
+        errors: [
+          {
+            message: err.message,
+            extensions: { code: err.extensions.code },
+          },
+        ],
+      })
+      return
+    }
+
+    baseLogger.error({ err, kratosUserId }, "failed to build graphql context")
+    recordExceptionInCurrentSpan({
+      error: err,
+      level: ErrorLevel.Critical,
+      attributes: { kratosUserId },
+      fallbackMsg: "failed to build graphql context",
+    })
+    res.status(500).json({ error: "failed to build graphql context" })
+    return
+  }
+
+  const cashWalletClientCapabilities = parseCashWalletClientCapabilities(req.headers)
+
+  req.gqlContext = {
+    ...gqlContext,
+    cashWalletClientCapabilities,
+  }
+
+  return addAttributesToCurrentSpanAndPropagate(
+    {
+      "token.iss": tokenPayload?.iss,
+      "token.session_id": tokenPayload?.session_id,
+      "token.expires_at": tokenPayload?.expires_at,
+      [SemanticAttributes.HTTP_CLIENT_IP]: ip,
+      [SemanticAttributes.HTTP_USER_AGENT]: req.headers["user-agent"],
+      [ACCOUNT_USERNAME]: gqlContext?.domainAccount?.username,
+      [SemanticAttributes.ENDUSER_ID]: tokenPayload?.sub,
+      "cash_wallet.client_presentation":
+        cashWalletClientCapabilities.cashWalletPresentation,
+      "cash_wallet.client_usdt_supported": String(
+        cashWalletClientCapabilities.hasUsdtCashWalletSupport,
+      ),
+    },
+    next,
+  )
+}