From 477c2bcbe3d60d8ed5d874d047e85175611c09de Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Sun, 6 Sep 2026 17:27:15 +0000 Subject: [PATCH 01/12] fix(app): expose phone audio diagnostics and meter --- .github/workflows/ios-testflight.yml | 2 + app/app.json | 2 +- .../duplexaudio/ChronicleDuplexAudioModule.kt | 1 + .../duplexaudio/DuplexAudioPolicy.kt | 15 ++ .../duplexaudio/DuplexAudioPolicyTest.kt | 10 ++ app/modules/chronicle-duplex-audio/index.ts | 1 + .../ios/ChronicleDuplexAudioModule.swift | 4 + .../ios/DuplexAudioState.swift | 12 ++ .../ios/Tests/DuplexAudioStateTests.swift | 15 ++ app/package.json | 1 + app/scripts/test-phone-audio-diagnostics.cjs | 106 +++++++++++ app/src/components/PhoneAudioButton.tsx | 2 +- app/src/hooks/useAudioStreamer.ts | 12 +- .../hooks/useAudioStreamingOrchestrator.ts | 10 +- app/src/hooks/usePhoneAudioRecorder.ts | 63 ++++++- app/src/services/phoneAudioDiagnostics.ts | 167 ++++++++++++++++++ 16 files changed, 416 insertions(+), 7 deletions(-) create mode 100644 app/scripts/test-phone-audio-diagnostics.cjs create mode 100644 app/src/services/phoneAudioDiagnostics.ts diff --git a/.github/workflows/ios-testflight.yml b/.github/workflows/ios-testflight.yml index 4ab77bccf..26d1e2310 100644 --- a/.github/workflows/ios-testflight.yml +++ b/.github/workflows/ios-testflight.yml @@ -52,9 +52,11 @@ jobs: npm run test:wearable-activation npm run typecheck npm run test:durable-audio-spool + npm run test:phone-audio-diagnostics npm run test:push-notifications npm run check:theme npx --no-install expo-modules-autolinking verify --platform ios --verbose + swift test --package-path modules/chronicle-duplex-audio/ios - name: Write and validate ASC API key env: diff --git a/app/app.json b/app/app.json index a95ff120d..e96794115 100644 --- a/app/app.json +++ b/app/app.json @@ -2,7 +2,7 @@ "expo": { "name": "chronicle", "slug": "friend-lite-app", - "version": "1.14.0", + "version": "1.15.0", "scheme": "chronicle", "orientation": "portrait", "icon": "./assets/icon.png", diff --git a/app/modules/chronicle-duplex-audio/android/src/main/java/com/chronicle/duplexaudio/ChronicleDuplexAudioModule.kt b/app/modules/chronicle-duplex-audio/android/src/main/java/com/chronicle/duplexaudio/ChronicleDuplexAudioModule.kt index a764e08c2..f3d0b99e5 100644 --- a/app/modules/chronicle-duplex-audio/android/src/main/java/com/chronicle/duplexaudio/ChronicleDuplexAudioModule.kt +++ b/app/modules/chronicle-duplex-audio/android/src/main/java/com/chronicle/duplexaudio/ChronicleDuplexAudioModule.kt @@ -219,6 +219,7 @@ class ChronicleDuplexAudioModule : Module() { "sampleRate" to 16_000, "channels" to 1, "frameDurationMs" to durationMs, + "audioLevel" to DuplexAudioPolicy.audioLevel(frame, count), "opusBase64" to Base64.encodeToString(packet, Base64.NO_WRAP), ), ) diff --git a/app/modules/chronicle-duplex-audio/android/src/main/java/com/chronicle/duplexaudio/DuplexAudioPolicy.kt b/app/modules/chronicle-duplex-audio/android/src/main/java/com/chronicle/duplexaudio/DuplexAudioPolicy.kt index 7593a98c7..de98eb498 100644 --- a/app/modules/chronicle-duplex-audio/android/src/main/java/com/chronicle/duplexaudio/DuplexAudioPolicy.kt +++ b/app/modules/chronicle-duplex-audio/android/src/main/java/com/chronicle/duplexaudio/DuplexAudioPolicy.kt @@ -20,6 +20,21 @@ internal object DuplexAudioPolicy { ): Boolean = current != null && (responseId == "*" || responseId == current.id) && cancellationGeneration >= current.generation + + fun audioLevel(pcm: ByteArray, byteCount: Int): Double { + val boundedBytes = minOf(byteCount, pcm.size) + val usableBytes = boundedBytes - (boundedBytes % 2) + if (usableBytes <= 0) return 0.0 + var sumOfSquares = 0.0 + var index = 0 + while (index < usableBytes) { + val sample = ((pcm[index].toInt() and 0xff) or (pcm[index + 1].toInt() shl 8)).toShort() + val normalized = sample.toDouble() / 32_768.0 + sumOfSquares += normalized * normalized + index += 2 + } + return kotlin.math.min(1.0, kotlin.math.sqrt(sumOfSquares / (usableBytes / 2).toDouble())) + } } internal data class EpochResponse( diff --git a/app/modules/chronicle-duplex-audio/android/src/test/java/com/chronicle/duplexaudio/DuplexAudioPolicyTest.kt b/app/modules/chronicle-duplex-audio/android/src/test/java/com/chronicle/duplexaudio/DuplexAudioPolicyTest.kt index c67657eb2..d1875a6fe 100644 --- a/app/modules/chronicle-duplex-audio/android/src/test/java/com/chronicle/duplexaudio/DuplexAudioPolicyTest.kt +++ b/app/modules/chronicle-duplex-audio/android/src/test/java/com/chronicle/duplexaudio/DuplexAudioPolicyTest.kt @@ -51,4 +51,14 @@ class DuplexAudioPolicyTest { gate.schedule(EpochResponse("two", 1, 4)) } } + + @Test fun audioMeterReportsSilenceAndNormalizedPeak() { + assertEquals(0.0, DuplexAudioPolicy.audioLevel(ByteArray(640), 640), 0.001) + val halfScale = ByteArray(640) + for (index in halfScale.indices step 2) { + halfScale[index] = 0 + halfScale[index + 1] = 64 + } + assertEquals(0.5, DuplexAudioPolicy.audioLevel(halfScale, 640), 0.001) + } } diff --git a/app/modules/chronicle-duplex-audio/index.ts b/app/modules/chronicle-duplex-audio/index.ts index 7419c0da0..73ed0c7cb 100644 --- a/app/modules/chronicle-duplex-audio/index.ts +++ b/app/modules/chronicle-duplex-audio/index.ts @@ -18,6 +18,7 @@ export interface NativeOpusFrame { sampleRate: 16000; channels: 1; frameDurationMs: number; + audioLevel: number; opusBase64: string; } diff --git a/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift b/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift index 221c0060d..eacd36b64 100644 --- a/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift +++ b/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift @@ -230,6 +230,9 @@ public final class ChronicleDuplexAudioModule: Module { compressed.byteLength > 0 else { return } let data = Data(bytes: compressed.data, count: Int(compressed.byteLength)) let durationMs = Double(output.frameLength) / 16_000 * 1_000 + let audioLevel = output.int16ChannelData.map { + ChronicleAudioMeter.level(samples: $0[0], count: Int(output.frameLength)) + } ?? 0 sendEvent("onOpusFrame", [ "captureEpoch": captureEpoch, "capturedAtMs": Date().timeIntervalSince1970 * 1_000 - durationMs, @@ -237,6 +240,7 @@ public final class ChronicleDuplexAudioModule: Module { "sampleRate": 16_000, "channels": 1, "frameDurationMs": durationMs, + "audioLevel": audioLevel, "opusBase64": data.base64EncodedString(), ]) } diff --git a/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift b/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift index 66ee2c519..4ba5a16bd 100644 --- a/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift +++ b/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift @@ -70,3 +70,15 @@ enum ChronicleDuplexResampler { AVAudioFrameCount(max(1, ceil(Double(inputFrames) * outputRate / inputRate))) } } + +enum ChronicleAudioMeter { + static func level(samples: UnsafePointer, count: Int) -> Double { + guard count > 0 else { return 0 } + var sumOfSquares = 0.0 + for index in 0.. mocks[request] ?? originalRequire(request); + loaded._compile(compiled.outputText, sourcePath); + return loaded.exports; +} + +const writes = []; +const sourcePath = path.join(__dirname, '../src/services/phoneAudioDiagnostics.ts'); +const { PhoneAudioDiagnostics } = loadTypeScript(sourcePath, { + '@/utils/logger': { + logInfo: (tag, message) => writes.push({ level: 'info', tag, message }), + logWarn: (tag, message) => writes.push({ level: 'warn', tag, message }), + logError: (tag, message) => writes.push({ level: 'error', tag, message }), + }, +}); + +const diagnostics = new PhoneAudioDiagnostics(() => 1_000); +diagnostics.beginAttempt(); +diagnostics.listenerInstalled(1); +diagnostics.engineStarted(1, { + mode: 'duplex_full', + input_route: 'built_in_mic', + output_route: 'speakerphone', + native_sample_rate: 48_000, +}); +diagnostics.nativeFrame({ captureEpoch: 1, opusBytes: 42, audioLevel: 0.25 }); +diagnostics.nativeFrame({ captureEpoch: 1, opusBytes: 43, audioLevel: 0.5 }); +diagnostics.audioLevelActive(0.5); +diagnostics.socketUnavailable(0); +diagnostics.socketUnavailable(0); +diagnostics.socketConnecting(); +diagnostics.socketOpen(); +diagnostics.captureStarted('capture-secret-id'); +diagnostics.frameEnqueued(44); +diagnostics.packetAccepted(0); +diagnostics.timeout('meter_stalled'); +diagnostics.failure('connect', 'wss://chronicle/ws/audio?token=secret-value'); + +assert.deepEqual( + writes.map(({ level, tag }) => [level, tag]), + [ + ['info', 'PhoneAudio'], + ['info', 'PhoneAudio'], + ['info', 'PhoneAudio'], + ['info', 'PhoneAudio'], + ['info', 'PhoneAudio'], + ['warn', 'PhoneAudio'], + ['info', 'PhoneAudio'], + ['info', 'PhoneAudio'], + ['info', 'PhoneAudio'], + ['info', 'PhoneAudio'], + ['info', 'PhoneAudio'], + ['warn', 'PhoneAudio'], + ['error', 'PhoneAudio'], + ], + 'each lifecycle boundary must be exported once while repeated frames/drops become counters', +); +const text = writes.map(({ message }) => message).join('\n'); +assert.match(text, /button_pressed attempt=1/); +assert.match(text, /native_first_frame.*opus_bytes=42.*audio_level=0\.250/); +assert.match(text, /audio_level_active.*audio_level=0\.500/); +assert.match(text, /frame_dropped_socket_not_open.*ready_state=0/); +assert.match(text, /first_frame_enqueued.*opus_bytes=44/); +assert.match(text, /first_packet_accepted.*sequence=0/); +assert.match( + text, + /meter_stalled.*native_frames=2.*socket_drops=2.*enqueued_frames=1.*acked_packets=1.*last_audio_level=0\.500/, +); +assert.doesNotMatch(text, /capture-secret-id/, 'server-issued identifiers must be abbreviated'); +assert.doesNotMatch(text, /secret-value/, 'credentials must be redacted from exported diagnostics'); + +const integrationSources = { + recorder: fs.readFileSync(path.join(__dirname, '../src/hooks/usePhoneAudioRecorder.ts'), 'utf8'), + streamer: fs.readFileSync(path.join(__dirname, '../src/hooks/useAudioStreamer.ts'), 'utf8'), + orchestrator: fs.readFileSync(path.join(__dirname, '../src/hooks/useAudioStreamingOrchestrator.ts'), 'utf8'), + ios: fs.readFileSync(path.join(__dirname, '../modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift'), 'utf8'), + android: fs.readFileSync(path.join(__dirname, '../modules/chronicle-duplex-audio/android/src/main/java/com/chronicle/duplexaudio/ChronicleDuplexAudioModule.kt'), 'utf8'), +}; +assert.match(integrationSources.recorder, /setAudioLevel\(/, 'native levels must drive the UI meter'); +assert.match(integrationSources.recorder, /native_frame_timeout/, 'a silent native engine must surface a diagnostic'); +assert.match(integrationSources.streamer, /packetAccepted\(sequence\)/, 'backend acknowledgements must be logged'); +assert.match(integrationSources.orchestrator, /beginAttempt\(\)/, 'the phone button must open a diagnostic attempt'); +assert.match(integrationSources.ios, /"audioLevel": audioLevel/, 'iOS must emit PCM audio levels'); +assert.match(integrationSources.android, /"audioLevel" to DuplexAudioPolicy\.audioLevel/, 'Android must emit PCM audio levels'); + +console.log('phone audio diagnostics tests passed'); diff --git a/app/src/components/PhoneAudioButton.tsx b/app/src/components/PhoneAudioButton.tsx index fb92565a8..da4127689 100644 --- a/app/src/components/PhoneAudioButton.tsx +++ b/app/src/components/PhoneAudioButton.tsx @@ -89,7 +89,7 @@ const PhoneAudioButton: React.FC = ({ )} - {error && !isRecording && ( + {error && ( {error} )} diff --git a/app/src/hooks/useAudioStreamer.ts b/app/src/hooks/useAudioStreamer.ts index 2299c0b1b..0aa465f11 100644 --- a/app/src/hooks/useAudioStreamer.ts +++ b/app/src/hooks/useAudioStreamer.ts @@ -29,6 +29,7 @@ import type { CapturedOpusFrame } from '../protocol/capturedOpusFrame'; import type { VoiceCapabilities } from '../protocol/audioCapabilities'; import { refreshToken } from '../services/auth'; import { durableAudioSpool, type SpoolPacket } from '../services/durableAudioSpool'; +import { phoneAudioDiagnostics } from '../services/phoneAudioDiagnostics'; interface UseAudioStreamerOptions { onTokenRefreshed?: (token: string) => void; @@ -157,6 +158,7 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr }, []); const packetAccepted = useCallback((sequence: number) => { + phoneAudioDiagnostics.packetAccepted(sequence); const packet = acceptedRef.current.get(sequence); if (packet) { acceptedRef.current.delete(sequence); @@ -315,6 +317,7 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr } }, onClosed: () => { + if (phoneVoice) phoneAudioDiagnostics.socketClosed(stoppedRef.current); setIsStreaming(false); if ( !stoppedRef.current && @@ -335,7 +338,9 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr }, }); socketRef.current = socket; + if (phoneVoice) phoneAudioDiagnostics.socketConnecting(); await socket.connect(); + if (phoneVoice) phoneAudioDiagnostics.socketOpen(); deliveryModeRef.current = 'recovering'; await drainRecovery(socket, phoneVoice?.captureEpoch ?? 0); liveStartedAtRef.current = Date.now(); @@ -343,7 +348,7 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr const capabilities = phoneVoice ? typedCapabilities(phoneVoice.capabilities) : undefined; - await socket.beginCapture({ + const binding = await socket.beginCapture({ captureEpoch: phoneVoice?.captureEpoch ?? 0, processingProfile: phoneVoice ? processingProfile(phoneVoice.capabilities) @@ -352,6 +357,9 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr deliveryClass: DeliveryClass.LIVE, capabilities, }); + if (phoneVoice) { + phoneAudioDiagnostics.captureStarted(binding.captureSessionId?.value ?? ''); + } deliveryModeRef.current = 'live'; if (capabilities) socket.voiceReady(capabilities); if (phoneVoice) { @@ -402,6 +410,7 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr try { await operation; } catch (cause) { + if (configRef.current?.phoneVoice) phoneAudioDiagnostics.failure('websocket_start', cause); setIsConnecting(false); setIsStreaming(false); const message = cause instanceof Error ? cause.message : 'Audio V2 connection failed'; @@ -432,6 +441,7 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr }, [enqueueLive]); const sendInteractiveFrame = useCallback((frame: CapturedOpusFrame) => { + phoneAudioDiagnostics.frameEnqueued(frame.opus.length); enqueueLive(frame.opus, frame.capturedAtMs); }, [enqueueLive]); diff --git a/app/src/hooks/useAudioStreamingOrchestrator.ts b/app/src/hooks/useAudioStreamingOrchestrator.ts index 8166cda7b..7a0e61faf 100644 --- a/app/src/hooks/useAudioStreamingOrchestrator.ts +++ b/app/src/hooks/useAudioStreamingOrchestrator.ts @@ -5,6 +5,7 @@ import { AppSettings } from './useAppSettings'; import type { StreamStartConfig } from './useAudioStreamer'; import type { PhoneCaptureSession } from './usePhoneAudioRecorder'; import type { CapturedOpusFrame } from '../protocol/capturedOpusFrame'; +import { phoneAudioDiagnostics } from '../services/phoneAudioDiagnostics'; interface OrchestratorParams { omiConnection: OmiConnection; @@ -82,13 +83,13 @@ export const useAudioStreamingOrchestrator = ({ if (isAdvanced) { const params = new URLSearchParams(); params.append('token', settings.jwtToken!); - const deviceName = settings.userId?.trim() || 'phone-mic'; + const deviceName = 'phone-mic'; params.append('device_name', deviceName); const separator = url.includes('?') ? '&' : '?'; url = `${url}${separator}${params.toString()}`; } return url; - }, [settings.jwtToken, settings.isAuthenticated, settings.userId]); + }, [settings.jwtToken, settings.isAuthenticated]); const handleStartAudioListeningAndStreaming = useCallback(async () => { if (!settings.webSocketUrl?.trim()) { @@ -135,11 +136,14 @@ export const useAudioStreamingOrchestrator = ({ const wsReady = audioStreamer.getWebSocketReadyState(); if (wsReady === WebSocket.OPEN && frame.opus.length > 0) { audioStreamer.sendInteractiveFrame(frame); + } else { + phoneAudioDiagnostics.socketUnavailable(wsReady); } }); await audioStreamer.startStreaming(finalUrl, { phoneVoice: capture }); setIsPhoneAudioMode(true); } catch (error) { + phoneAudioDiagnostics.failure('orchestrator_start', error); Alert.alert('Error', 'Could not start phone audio streaming.'); if (audioStreamer.isStreaming) audioStreamer.stopStreaming(); if (phoneAudioRecorder.isRecording) await phoneAudioRecorder.stopRecording(); @@ -151,12 +155,14 @@ export const useAudioStreamingOrchestrator = ({ await audioStreamer.stopStreaming(); await phoneAudioRecorder.stopRecording(); setIsPhoneAudioMode(false); + phoneAudioDiagnostics.stopped(); }, [phoneAudioRecorder, audioStreamer]); const handleTogglePhoneAudio = useCallback(async () => { if (isPhoneAudioMode || phoneAudioRecorder.isRecording) { await handleStopPhoneAudioStreaming(); } else { + phoneAudioDiagnostics.beginAttempt(); await handleStartPhoneAudioStreaming(); } }, [isPhoneAudioMode, phoneAudioRecorder.isRecording, handleStartPhoneAudioStreaming, handleStopPhoneAudioStreaming]); diff --git a/app/src/hooks/usePhoneAudioRecorder.ts b/app/src/hooks/usePhoneAudioRecorder.ts index 92e03038f..344d86615 100644 --- a/app/src/hooks/usePhoneAudioRecorder.ts +++ b/app/src/hooks/usePhoneAudioRecorder.ts @@ -13,6 +13,11 @@ import { capturedOpusFrameFromNative, type CapturedOpusFrame, } from '../protocol/capturedOpusFrame'; +import { phoneAudioDiagnostics } from '../services/phoneAudioDiagnostics'; + +const FIRST_FRAME_TIMEOUT_MS = 3_000; +const AUDIO_LEVEL_TIMEOUT_MS = 5_000; +const ACTIVE_AUDIO_LEVEL = 0.01; export interface PhoneCaptureSession { captureEpoch: number; @@ -50,8 +55,16 @@ export const usePhoneAudioRecorder = (): UsePhoneAudioRecorder => { const captureEpochRef = useRef(0); const frameSubscriptionRef = useRef<{ remove: () => void } | null>(null); const onAudioDataRef = useRef<((frame: CapturedOpusFrame) => void) | null>(null); + const firstFrameSeenRef = useRef(false); + const firstFrameTimeoutRef = useRef | null>(null); + const audioLevelActiveRef = useRef(false); + const audioLevelTimeoutRef = useRef | null>(null); const markCaptureStopped = useCallback(() => { + if (firstFrameTimeoutRef.current) clearTimeout(firstFrameTimeoutRef.current); + firstFrameTimeoutRef.current = null; + if (audioLevelTimeoutRef.current) clearTimeout(audioLevelTimeoutRef.current); + audioLevelTimeoutRef.current = null; frameSubscriptionRef.current?.remove(); frameSubscriptionRef.current = null; onAudioDataRef.current = null; @@ -74,6 +87,20 @@ export const usePhoneAudioRecorder = (): UsePhoneAudioRecorder => { const captureEpoch = captureEpochRef.current + 1; captureEpochRef.current = captureEpoch; const capabilities = await startVoiceSession({ captureEpoch }); + phoneAudioDiagnostics.engineStarted(captureEpoch, capabilities); + if (!firstFrameSeenRef.current) { + firstFrameTimeoutRef.current = setTimeout(() => { + phoneAudioDiagnostics.timeout('native_frame_timeout'); + if (mountedRef.current) { + setError('Microphone started, but Chronicle received no audio frames.'); + } + }, FIRST_FRAME_TIMEOUT_MS); + } + if (!audioLevelActiveRef.current) { + audioLevelTimeoutRef.current = setTimeout(() => { + phoneAudioDiagnostics.timeout('audio_level_stalled'); + }, AUDIO_LEVEL_TIMEOUT_MS); + } return { captureEpoch, capabilities, @@ -100,11 +127,40 @@ export const usePhoneAudioRecorder = (): UsePhoneAudioRecorder => { } try { + firstFrameSeenRef.current = false; + audioLevelActiveRef.current = false; onAudioDataRef.current = onAudioData; + phoneAudioDiagnostics.listenerInstalled(captureEpochRef.current + 1); frameSubscriptionRef.current = addOpusFrameListener((frame) => { if (!mountedRef.current || frame.captureEpoch !== captureEpochRef.current) return; - const captured = capturedOpusFrameFromNative(frame, decodeBase64); - if (!captured.opus.length) return; + let captured: CapturedOpusFrame; + try { + captured = capturedOpusFrameFromNative(frame, decodeBase64); + } catch (cause) { + phoneAudioDiagnostics.invalidNativeFrame( + cause instanceof Error ? cause.message : 'invalid_native_frame', + ); + return; + } + const level = Math.min(1, Math.max(0, frame.audioLevel || 0)); + phoneAudioDiagnostics.nativeFrame({ + captureEpoch: frame.captureEpoch, + opusBytes: captured.opus.length, + audioLevel: level, + }); + if (!firstFrameSeenRef.current) { + firstFrameSeenRef.current = true; + if (firstFrameTimeoutRef.current) clearTimeout(firstFrameTimeoutRef.current); + firstFrameTimeoutRef.current = null; + setError(null); + } + if (!audioLevelActiveRef.current && level >= ACTIVE_AUDIO_LEVEL) { + audioLevelActiveRef.current = true; + if (audioLevelTimeoutRef.current) clearTimeout(audioLevelTimeoutRef.current); + audioLevelTimeoutRef.current = null; + phoneAudioDiagnostics.audioLevelActive(level); + } + setAudioLevel(previous => previous === 0 ? level : (previous * 0.65) + (level * 0.35)); onAudioDataRef.current?.(captured); }); const capture = await startNativeCapture(); @@ -114,6 +170,7 @@ export const usePhoneAudioRecorder = (): UsePhoneAudioRecorder => { } return capture; } catch (cause) { + phoneAudioDiagnostics.failure('native_capture_start', cause); frameSubscriptionRef.current?.remove(); frameSubscriptionRef.current = null; const message = cause instanceof Error ? cause.message : 'Failed to start duplex audio'; @@ -130,6 +187,8 @@ export const usePhoneAudioRecorder = (): UsePhoneAudioRecorder => { mountedRef.current = false; frameSubscriptionRef.current?.remove(); frameSubscriptionRef.current = null; + if (firstFrameTimeoutRef.current) clearTimeout(firstFrameTimeoutRef.current); + if (audioLevelTimeoutRef.current) clearTimeout(audioLevelTimeoutRef.current); stopVoiceSession().catch(() => undefined); }, []); diff --git a/app/src/services/phoneAudioDiagnostics.ts b/app/src/services/phoneAudioDiagnostics.ts new file mode 100644 index 000000000..a86fcaeae --- /dev/null +++ b/app/src/services/phoneAudioDiagnostics.ts @@ -0,0 +1,167 @@ +import type { VoiceCapabilities } from '../protocol/audioCapabilities'; +import { logError, logInfo, logWarn } from '@/utils/logger'; + +type DiagnosticLevel = 'info' | 'warn' | 'error'; +type Clock = () => number; + +interface NativeFrameDiagnostic { + captureEpoch: number; + opusBytes: number; + audioLevel: number; +} + +type CapabilityDiagnostic = Pick< + VoiceCapabilities, + 'mode' | 'input_route' | 'output_route' | 'native_sample_rate' +>; + +const shortId = (value: string): string => value ? `${value.slice(0, 8)}…` : 'none'; +const safeLevel = (value: number): number => Math.min(1, Math.max(0, value || 0)); +const redactSecrets = (value: string): string => value + .replace(/([?&](?:token|access_token)=)[^&\s]+/gi, '$1') + .replace(/Bearer\s+[^\s]+/gi, 'Bearer '); + +export class PhoneAudioDiagnostics { + private attempt = 0; + private active = false; + private startedAtMs = 0; + private milestones = new Set(); + private nativeFrames = 0; + private socketDrops = 0; + private enqueuedFrames = 0; + private ackedPackets = 0; + private lastAudioLevel = 0; + + constructor(private readonly now: Clock = Date.now) {} + + private write(level: DiagnosticLevel, event: string, details = ''): void { + const message = `${event} attempt=${this.attempt}${details ? ` ${details}` : ''}`; + if (level === 'error') logError('PhoneAudio', message); + else if (level === 'warn') logWarn('PhoneAudio', message); + else logInfo('PhoneAudio', message); + } + + private once(level: DiagnosticLevel, event: string, details = ''): void { + if (this.milestones.has(event)) return; + this.milestones.add(event); + this.write(level, event, details); + } + + beginAttempt(): void { + this.attempt += 1; + this.active = true; + this.startedAtMs = this.now(); + this.milestones.clear(); + this.nativeFrames = 0; + this.socketDrops = 0; + this.enqueuedFrames = 0; + this.ackedPackets = 0; + this.lastAudioLevel = 0; + this.write('info', 'button_pressed'); + } + + listenerInstalled(captureEpoch: number): void { + this.once('info', 'native_listener_installed', `capture_epoch=${captureEpoch}`); + } + + engineStarted(captureEpoch: number, capabilities: CapabilityDiagnostic): void { + this.once( + 'info', + 'native_engine_started', + [ + `capture_epoch=${captureEpoch}`, + `mode=${capabilities.mode}`, + `input=${capabilities.input_route}`, + `output=${capabilities.output_route}`, + `sample_rate=${capabilities.native_sample_rate}`, + ].join(' '), + ); + } + + nativeFrame(frame: NativeFrameDiagnostic): void { + if (!this.active) return; + this.nativeFrames += 1; + this.lastAudioLevel = safeLevel(frame.audioLevel); + this.once( + 'info', + 'native_first_frame', + `capture_epoch=${frame.captureEpoch} opus_bytes=${frame.opusBytes} audio_level=${this.lastAudioLevel.toFixed(3)}`, + ); + } + + audioLevelActive(audioLevel: number): void { + this.once('info', 'audio_level_active', `audio_level=${safeLevel(audioLevel).toFixed(3)}`); + } + + invalidNativeFrame(reason: string): void { + this.once('warn', 'native_frame_rejected', `reason=${reason}`); + } + + socketUnavailable(readyState: number | undefined): void { + if (!this.active) return; + this.socketDrops += 1; + this.once( + 'warn', + 'frame_dropped_socket_not_open', + `ready_state=${readyState ?? 'undefined'}`, + ); + } + + socketConnecting(): void { + this.once('info', 'websocket_connecting'); + } + + socketOpen(): void { + this.once('info', 'websocket_open'); + } + + socketClosed(expected: boolean): void { + if (!this.active) return; + this.write(expected ? 'info' : 'warn', 'websocket_closed', `expected=${expected}`); + } + + captureStarted(captureSessionId: string): void { + this.once('info', 'backend_capture_started', `capture_id=${shortId(captureSessionId)}`); + } + + frameEnqueued(opusBytes: number): void { + if (!this.active) return; + this.enqueuedFrames += 1; + this.once('info', 'first_frame_enqueued', `opus_bytes=${opusBytes}`); + } + + packetAccepted(sequence: number): void { + if (!this.active || !this.milestones.has('backend_capture_started')) return; + this.ackedPackets += 1; + this.once('info', 'first_packet_accepted', `sequence=${sequence}`); + } + + timeout(reason: string): void { + if (!this.active) return; + this.write('warn', reason, this.snapshot()); + } + + failure(stage: string, cause: unknown): void { + const message = redactSecrets(cause instanceof Error ? cause.message : String(cause)); + this.write('error', 'failed', `stage=${stage} error=${message.slice(0, 300)} ${this.snapshot()}`); + } + + stopped(reason = 'user'): void { + if (!this.active) return; + this.write('info', 'stopped', `reason=${reason} ${this.snapshot()}`); + this.active = false; + } + + private snapshot(): string { + return [ + `elapsed_ms=${Math.max(0, this.now() - this.startedAtMs)}`, + `native_frames=${this.nativeFrames}`, + `socket_drops=${this.socketDrops}`, + `enqueued_frames=${this.enqueuedFrames}`, + `acked_packets=${this.ackedPackets}`, + `last_audio_level=${this.lastAudioLevel.toFixed(3)}`, + ].join(' '); + } +} + +export const phoneAudioDiagnostics = new PhoneAudioDiagnostics(); From 7881e616abf9523e310390209cf7d360a91aa110 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:04:49 +0000 Subject: [PATCH 02/12] fix(app): keep phone capture alive and expose native stages --- app/modules/chronicle-duplex-audio/index.ts | 26 +++++ .../ios/ChronicleDuplexAudioModule.swift | 105 +++++++++++------- .../ios/DuplexAudioState.swift | 80 +++++++++++++ .../ios/Tests/DuplexAudioStateTests.swift | 20 ++++ app/scripts/test-phone-audio-diagnostics.cjs | 20 ++++ app/src/hooks/useAudioStreamer.ts | 8 +- app/src/hooks/usePhoneAudioRecorder.ts | 12 ++ app/src/services/phoneAudioDiagnostics.ts | 16 +++ 8 files changed, 241 insertions(+), 46 deletions(-) diff --git a/app/modules/chronicle-duplex-audio/index.ts b/app/modules/chronicle-duplex-audio/index.ts index 73ed0c7cb..c93dd5af2 100644 --- a/app/modules/chronicle-duplex-audio/index.ts +++ b/app/modules/chronicle-duplex-audio/index.ts @@ -22,6 +22,22 @@ export interface NativeOpusFrame { opusBase64: string; } +export interface NativeCaptureDiagnostic { + captureEpoch: number; + stage: + | 'tap_received' + | 'pcm_converted' + | 'pcm_conversion_failed' + | 'pcm_empty' + | 'pcm_wrong_frame_count' + | 'opus_encoded' + | 'opus_encode_failed'; + monotonicTimestampMs: number; + frameCount?: number; + byteCount?: number; + detail?: string; +} + export interface NativeResponse { responseId: string; generation: number; @@ -58,6 +74,10 @@ type ChronicleDuplexAudioNative = NativeModule & { eventName: 'onOpusFrame', listener: (event: NativeOpusFrame) => void ): EventSubscription; + addListener( + eventName: 'onCaptureDiagnostic', + listener: (event: NativeCaptureDiagnostic) => void + ): EventSubscription; addListener( eventName: 'onPlaybackState', listener: (event: NativePlaybackState) => void @@ -95,6 +115,12 @@ export function addOpusFrameListener( return requireNative().addListener('onOpusFrame', listener); } +export function addCaptureDiagnosticListener( + listener: (event: NativeCaptureDiagnostic) => void +): EventSubscription { + return requireNative().addListener('onCaptureDiagnostic', listener); +} + export function addPlaybackStateListener( listener: (event: NativePlaybackState) => void ): EventSubscription { diff --git a/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift b/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift index eacd36b64..9ccc767b4 100644 --- a/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift +++ b/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift @@ -5,8 +5,10 @@ public final class ChronicleDuplexAudioModule: Module { private let engine = AVAudioEngine() private let player = AVAudioPlayerNode() private let controlQueue = DispatchQueue(label: "chronicle.duplex.audio") + private let captureDiagnosticLock = NSLock() private var converter: AVAudioConverter? - private var opusConverter: AVAudioConverter? + private var opusEncoder: ChronicleOpusPacketEncoder? + private var emittedCaptureDiagnosticStages = Set() private var captureEpoch = 0 private var voiceProcessingEnabled = false private var captureSuppressed = false @@ -21,7 +23,7 @@ public final class ChronicleDuplexAudioModule: Module { public func definition() -> ModuleDefinition { Name("ChronicleDuplexAudio") - Events("onOpusFrame", "onPlaybackState", "onRouteChange") + Events("onOpusFrame", "onCaptureDiagnostic", "onPlaybackState", "onRouteChange") OnCreate { [weak self] in self?.installObservers() @@ -124,6 +126,9 @@ public final class ChronicleDuplexAudioModule: Module { private func startEngine(captureEpoch: Int) throws { tearDownEngine(deactivateSession: false) self.captureEpoch = captureEpoch + captureDiagnosticLock.lock() + emittedCaptureDiagnosticStages.removeAll() + captureDiagnosticLock.unlock() let session = AVAudioSession.sharedInstance() if !sessionConfigured { @@ -153,27 +158,20 @@ public final class ChronicleDuplexAudioModule: Module { } let inputFormat = input.outputFormat(forBus: 0) - guard let targetFormat = AVAudioFormat( - commonFormat: .pcmFormatInt16, - sampleRate: 16_000, - channels: 1, - interleaved: true - ), let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else { - throw Exception(name: "engine_unavailable", description: "Cannot create the 16 kHz PCM converter") + let opusEncoder: ChronicleOpusPacketEncoder + do { + opusEncoder = try ChronicleOpusPacketEncoder() + } catch { + throw Exception(name: "engine_unavailable", description: "Cannot create the raw Opus encoder: \(error)") } - guard let opusFormat = AVAudioFormat(settings: [ - AVFormatIDKey: kAudioFormatOpus, - AVSampleRateKey: 16_000, - AVNumberOfChannelsKey: 1, - AVEncoderBitRateKey: 24_000, - ]), let opusConverter = AVAudioConverter(from: targetFormat, to: opusFormat) else { - throw Exception(name: "engine_unavailable", description: "Cannot create the raw Opus encoder") + guard let converter = AVAudioConverter(from: inputFormat, to: opusEncoder.inputFormat) else { + throw Exception(name: "engine_unavailable", description: "Cannot create the 16 kHz PCM converter") } - opusConverter.bitRate = 24_000 self.converter = converter - self.opusConverter = opusConverter + self.opusEncoder = opusEncoder let inputFrameCount = AVAudioFrameCount(round(inputFormat.sampleRate * 0.02)) input.installTap(onBus: 0, bufferSize: inputFrameCount, format: inputFormat) { [weak self] buffer, _ in + self?.emitCaptureDiagnostic(stage: "tap_received") self?.emitOpus(buffer) } tapInstalled = true @@ -186,7 +184,7 @@ public final class ChronicleDuplexAudioModule: Module { guard !captureSuppressed, engine.isRunning, let converter, - let opusConverter else { return } + let opusEncoder else { return } let capacity = ChronicleDuplexResampler.outputCapacity( inputFrames: input.frameLength, inputRate: input.format.sampleRate @@ -205,30 +203,30 @@ public final class ChronicleDuplexAudioModule: Module { state.pointee = .haveData return input } - guard status != .error, - conversionError == nil, - output.frameLength > 0 else { return } - let compressed = AVAudioCompressedBuffer( - format: opusConverter.outputFormat, - packetCapacity: 1, - maximumPacketSize: 1_275 - ) - var opusSupplied = false - var opusError: NSError? - let opusStatus = opusConverter.convert(to: compressed, error: &opusError) { _, state in - if opusSupplied { - state.pointee = .noDataNow - return nil - } - opusSupplied = true - state.pointee = .haveData - return output + guard status != .error, conversionError == nil else { + emitCaptureDiagnostic( + stage: "pcm_conversion_failed", + detail: conversionError?.localizedDescription ?? "converter_status_error" + ) + return + } + guard output.frameLength > 0 else { + emitCaptureDiagnostic(stage: "pcm_empty") + return + } + emitCaptureDiagnostic(stage: "pcm_converted", frameCount: Int(output.frameLength)) + guard output.frameLength == ChronicleOpusPacketEncoder.framesPerPacket else { + emitCaptureDiagnostic(stage: "pcm_wrong_frame_count", frameCount: Int(output.frameLength)) + return } - guard opusStatus != .error, - opusError == nil, - compressed.packetCount == 1, - compressed.byteLength > 0 else { return } - let data = Data(bytes: compressed.data, count: Int(compressed.byteLength)) + let data: Data + do { + data = try opusEncoder.encode(output) + } catch { + emitCaptureDiagnostic(stage: "opus_encode_failed", detail: String(describing: error)) + return + } + emitCaptureDiagnostic(stage: "opus_encoded", frameCount: Int(output.frameLength), byteCount: data.count) let durationMs = Double(output.frameLength) / 16_000 * 1_000 let audioLevel = output.int16ChannelData.map { ChronicleAudioMeter.level(samples: $0[0], count: Int(output.frameLength)) @@ -245,6 +243,27 @@ public final class ChronicleDuplexAudioModule: Module { ]) } + private func emitCaptureDiagnostic( + stage: String, + frameCount: Int? = nil, + byteCount: Int? = nil, + detail: String? = nil + ) { + captureDiagnosticLock.lock() + let inserted = emittedCaptureDiagnosticStages.insert(stage).inserted + captureDiagnosticLock.unlock() + guard inserted else { return } + var payload: [String: Any] = [ + "captureEpoch": captureEpoch, + "stage": stage, + "monotonicTimestampMs": ProcessInfo.processInfo.systemUptime * 1_000, + ] + if let frameCount { payload["frameCount"] = frameCount } + if let byteCount { payload["byteCount"] = byteCount } + if let detail { payload["detail"] = String(detail.prefix(240)) } + sendEvent("onCaptureDiagnostic", payload) + } + private func schedule(response: [String: Any]) throws { guard let responseId = response["responseId"] as? String, let generation = response["generation"] as? Int, @@ -427,7 +446,7 @@ public final class ChronicleDuplexAudioModule: Module { engine.stop() if player.engine != nil { engine.detach(player) } converter = nil - opusConverter = nil + opusEncoder = nil voiceProcessingEnabled = false captureSuppressed = false if deactivateSession { diff --git a/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift b/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift index 4ba5a16bd..efa4fd429 100644 --- a/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift +++ b/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift @@ -5,6 +5,13 @@ enum DuplexAudioStateError: Error { case responseAlreadyScheduled } +enum ChronicleOpusEncoderError: Error { + case formatUnavailable + case converterUnavailable + case conversionFailed(String) + case packetUnavailable +} + struct DuplexResponseBinding: Equatable { let id: String let generation: Int @@ -82,3 +89,76 @@ enum ChronicleAudioMeter { return min(1, sqrt(sumOfSquares / Double(count))) } } + +final class ChronicleOpusPacketEncoder { + static let sampleRate = 16_000.0 + static let framesPerPacket: AVAudioFrameCount = 320 + + let inputFormat: AVAudioFormat + let outputFormat: AVAudioFormat + private let converter: AVAudioConverter + + init(bitRate: Int = 24_000) throws { + guard let inputFormat = AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: Self.sampleRate, + channels: 1, + interleaved: true + ) else { + throw ChronicleOpusEncoderError.formatUnavailable + } + var description = AudioStreamBasicDescription( + mSampleRate: Self.sampleRate, + mFormatID: kAudioFormatOpus, + mFormatFlags: 0, + mBytesPerPacket: 0, + mFramesPerPacket: UInt32(Self.framesPerPacket), + mBytesPerFrame: 0, + mChannelsPerFrame: 1, + mBitsPerChannel: 0, + mReserved: 0 + ) + guard let outputFormat = AVAudioFormat(streamDescription: &description), + let converter = AVAudioConverter(from: inputFormat, to: outputFormat) else { + throw ChronicleOpusEncoderError.converterUnavailable + } + converter.bitRate = bitRate + converter.primeMethod = .none + self.inputFormat = inputFormat + self.outputFormat = outputFormat + self.converter = converter + } + + func encode(_ input: AVAudioPCMBuffer) throws -> Data { + guard input.format == inputFormat, + input.frameLength == Self.framesPerPacket else { + throw ChronicleOpusEncoderError.formatUnavailable + } + let maximumPacketSize = max(1, converter.maximumOutputPacketSize) + let compressed = AVAudioCompressedBuffer( + format: outputFormat, + packetCapacity: 1, + maximumPacketSize: maximumPacketSize + ) + var supplied = false + var conversionError: NSError? + let status = converter.convert(to: compressed, error: &conversionError) { _, state in + if supplied { + state.pointee = .noDataNow + return nil + } + supplied = true + state.pointee = .haveData + return input + } + if status == .error || conversionError != nil { + throw ChronicleOpusEncoderError.conversionFailed( + conversionError?.localizedDescription ?? "converter_status_error" + ) + } + guard compressed.packetCount == 1, compressed.byteLength > 0 else { + throw ChronicleOpusEncoderError.packetUnavailable + } + return Data(bytes: compressed.data, count: Int(compressed.byteLength)) + } +} diff --git a/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift b/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift index a205e4091..8e7889560 100644 --- a/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift +++ b/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift @@ -87,6 +87,26 @@ final class DuplexAudioStateTests: XCTestCase { } } + func testOpusEncoderProducesOnePacketForTwentyMillisecondsOfPcm() throws { + let encoder = try ChronicleOpusPacketEncoder() + let buffer = try XCTUnwrap( + AVAudioPCMBuffer( + pcmFormat: encoder.inputFormat, + frameCapacity: ChronicleOpusPacketEncoder.framesPerPacket + ) + ) + buffer.frameLength = ChronicleOpusPacketEncoder.framesPerPacket + let samples = try XCTUnwrap(buffer.int16ChannelData)[0] + for index in 0.. message).join('\n'); assert.match(text, /button_pressed attempt=1/); assert.match(text, /native_first_frame.*opus_bytes=42.*audio_level=0\.250/); +assert.match(text, /native_tap_received.*capture_epoch=1/); +assert.match(text, /native_pcm_converted.*frames=320/); +assert.match(text, /native_opus_encoded.*bytes=42/); assert.match(text, /audio_level_active.*audio_level=0\.500/); assert.match(text, /frame_dropped_socket_not_open.*ready_state=0/); assert.match(text, /first_frame_enqueued.*opus_bytes=44/); @@ -99,8 +108,19 @@ const integrationSources = { assert.match(integrationSources.recorder, /setAudioLevel\(/, 'native levels must drive the UI meter'); assert.match(integrationSources.recorder, /native_frame_timeout/, 'a silent native engine must surface a diagnostic'); assert.match(integrationSources.streamer, /packetAccepted\(sequence\)/, 'backend acknowledgements must be logged'); +assert.match( + integrationSources.streamer, + /const optionsRef = useRef\(options\)/, + 'rerenders must not replace the WebSocket lifecycle callback through a fresh options object', +); +assert.doesNotMatch( + integrationSources.streamer, + /\}, \[drainRecovery, encodeBase64, options, packetAccepted\]\);/, + 'startStreaming must not close an active socket merely because its options object was recreated', +); assert.match(integrationSources.orchestrator, /beginAttempt\(\)/, 'the phone button must open a diagnostic attempt'); assert.match(integrationSources.ios, /"audioLevel": audioLevel/, 'iOS must emit PCM audio levels'); +assert.match(integrationSources.ios, /"onCaptureDiagnostic"/, 'iOS must expose the native capture stages'); assert.match(integrationSources.android, /"audioLevel" to DuplexAudioPolicy\.audioLevel/, 'Android must emit PCM audio levels'); console.log('phone audio diagnostics tests passed'); diff --git a/app/src/hooks/useAudioStreamer.ts b/app/src/hooks/useAudioStreamer.ts index 0aa465f11..8f35c24f2 100644 --- a/app/src/hooks/useAudioStreamer.ts +++ b/app/src/hooks/useAudioStreamer.ts @@ -128,6 +128,8 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr const [error, setError] = useState(null); const [phonePlaybackState, setPhonePlaybackState] = useState(null); const socketRef = useRef(null); + const optionsRef = useRef(options); + optionsRef.current = options; const configRef = useRef(undefined); const urlRef = useRef(''); const stoppedRef = useRef(false); @@ -259,7 +261,7 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr if (!parsed.bearerToken) { const token = await refreshToken(); if (!token) throw new Error('Audio authentication expired'); - options?.onTokenRefreshed?.(token); + optionsRef.current?.onTokenRefreshed?.(token); const refreshed = new URL(url); refreshed.searchParams.set('token', token); urlRef.current = refreshed.toString(); @@ -321,7 +323,7 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr setIsStreaming(false); if ( !stoppedRef.current && - (options?.autoReconnectEnabled ?? true) && + (optionsRef.current?.autoReconnectEnabled ?? true) && !reconnectRef.current ) { const delay = Math.min( @@ -426,7 +428,7 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr } finally { connectingRef.current = null; } - }, [drainRecovery, encodeBase64, options, packetAccepted]); + }, [drainRecovery, encodeBase64, packetAccepted]); const enqueueLive = useCallback((opus: Uint8Array, capturedAtMs: number) => { if (!opus.length) return; diff --git a/app/src/hooks/usePhoneAudioRecorder.ts b/app/src/hooks/usePhoneAudioRecorder.ts index 344d86615..b30654322 100644 --- a/app/src/hooks/usePhoneAudioRecorder.ts +++ b/app/src/hooks/usePhoneAudioRecorder.ts @@ -4,6 +4,7 @@ import { PermissionsAndroid, Platform } from 'react-native'; import base64 from 'react-native-base64'; import { + addCaptureDiagnosticListener, addOpusFrameListener, startVoiceSession, stopVoiceSession, @@ -54,6 +55,7 @@ export const usePhoneAudioRecorder = (): UsePhoneAudioRecorder => { const mountedRef = useRef(true); const captureEpochRef = useRef(0); const frameSubscriptionRef = useRef<{ remove: () => void } | null>(null); + const diagnosticSubscriptionRef = useRef<{ remove: () => void } | null>(null); const onAudioDataRef = useRef<((frame: CapturedOpusFrame) => void) | null>(null); const firstFrameSeenRef = useRef(false); const firstFrameTimeoutRef = useRef | null>(null); @@ -67,6 +69,8 @@ export const usePhoneAudioRecorder = (): UsePhoneAudioRecorder => { audioLevelTimeoutRef.current = null; frameSubscriptionRef.current?.remove(); frameSubscriptionRef.current = null; + diagnosticSubscriptionRef.current?.remove(); + diagnosticSubscriptionRef.current = null; onAudioDataRef.current = null; if (mountedRef.current) { setIsRecording(false); @@ -131,6 +135,10 @@ export const usePhoneAudioRecorder = (): UsePhoneAudioRecorder => { audioLevelActiveRef.current = false; onAudioDataRef.current = onAudioData; phoneAudioDiagnostics.listenerInstalled(captureEpochRef.current + 1); + diagnosticSubscriptionRef.current = addCaptureDiagnosticListener((event) => { + if (!mountedRef.current || event.captureEpoch !== captureEpochRef.current) return; + phoneAudioDiagnostics.nativeStage(event); + }); frameSubscriptionRef.current = addOpusFrameListener((frame) => { if (!mountedRef.current || frame.captureEpoch !== captureEpochRef.current) return; let captured: CapturedOpusFrame; @@ -173,6 +181,8 @@ export const usePhoneAudioRecorder = (): UsePhoneAudioRecorder => { phoneAudioDiagnostics.failure('native_capture_start', cause); frameSubscriptionRef.current?.remove(); frameSubscriptionRef.current = null; + diagnosticSubscriptionRef.current?.remove(); + diagnosticSubscriptionRef.current = null; const message = cause instanceof Error ? cause.message : 'Failed to start duplex audio'; if (mountedRef.current) { setError(message); @@ -187,6 +197,8 @@ export const usePhoneAudioRecorder = (): UsePhoneAudioRecorder => { mountedRef.current = false; frameSubscriptionRef.current?.remove(); frameSubscriptionRef.current = null; + diagnosticSubscriptionRef.current?.remove(); + diagnosticSubscriptionRef.current = null; if (firstFrameTimeoutRef.current) clearTimeout(firstFrameTimeoutRef.current); if (audioLevelTimeoutRef.current) clearTimeout(audioLevelTimeoutRef.current); stopVoiceSession().catch(() => undefined); diff --git a/app/src/services/phoneAudioDiagnostics.ts b/app/src/services/phoneAudioDiagnostics.ts index a86fcaeae..87dffd21e 100644 --- a/app/src/services/phoneAudioDiagnostics.ts +++ b/app/src/services/phoneAudioDiagnostics.ts @@ -1,4 +1,5 @@ import type { VoiceCapabilities } from '../protocol/audioCapabilities'; +import type { NativeCaptureDiagnostic } from '../../modules/chronicle-duplex-audio'; import { logError, logInfo, logWarn } from '@/utils/logger'; type DiagnosticLevel = 'info' | 'warn' | 'error'; @@ -89,6 +90,21 @@ export class PhoneAudioDiagnostics { ); } + nativeStage(event: NativeCaptureDiagnostic): void { + if (!this.active) return; + const detail = redactSecrets(event.detail ?? '').slice(0, 240); + this.once( + event.stage.endsWith('_failed') ? 'error' : 'info', + `native_${event.stage}`, + [ + `capture_epoch=${event.captureEpoch}`, + event.frameCount === undefined ? '' : `frames=${event.frameCount}`, + event.byteCount === undefined ? '' : `bytes=${event.byteCount}`, + detail ? `detail=${detail}` : '', + ].filter(Boolean).join(' '), + ); + } + audioLevelActive(audioLevel: number): void { this.once('info', 'audio_level_active', `audio_level=${safeLevel(audioLevel).toFixed(3)}`); } From 32f4b4338e2bbb5b1d950bd2647c97a64786d578 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Sun, 6 Sep 2026 20:04:48 +0000 Subject: [PATCH 03/12] fix(app): identify installed binary in diagnostics --- app/package-lock.json | 1 + app/package.json | 3 +- .../test-client-diagnostic-metadata.cjs | 112 ++++++++++++++++++ app/src/services/clientDiagnostics.ts | 5 +- app/src/utils/logger.ts | 11 +- 5 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 app/scripts/test-client-diagnostic-metadata.cjs diff --git a/app/package-lock.json b/app/package-lock.json index 09a3a7db8..e28eb6fba 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -17,6 +17,7 @@ "@react-native/virtualized-lists": "^0.80.2", "deprecated-react-native-prop-types": "^5.0.0", "expo": "~55.0.15", + "expo-application": "~55.0.19", "expo-asset": "~55.0.17", "expo-build-properties": "~55.0.13", "expo-camera": "~55.0.15", diff --git a/app/package.json b/app/package.json index 4fe827ba7..d681cd2fb 100644 --- a/app/package.json +++ b/app/package.json @@ -12,7 +12,7 @@ "typecheck": "tsc --noEmit", "test:wearable-activation": "node scripts/test-wearable-activation.cjs", "test:durable-audio-spool": "node scripts/test-durable-audio-spool.cjs", - "test:phone-audio-diagnostics": "node scripts/test-phone-audio-diagnostics.cjs", + "test:phone-audio-diagnostics": "node scripts/test-phone-audio-diagnostics.cjs && node scripts/test-client-diagnostic-metadata.cjs", "test:push-notifications": "node scripts/test-push-notifications.cjs", "check:theme": "node scripts/check-theme-tokens.mjs" }, @@ -26,6 +26,7 @@ "@react-native/virtualized-lists": "^0.80.2", "deprecated-react-native-prop-types": "^5.0.0", "expo": "~55.0.15", + "expo-application": "~55.0.19", "expo-asset": "~55.0.17", "expo-build-properties": "~55.0.13", "expo-camera": "~55.0.15", diff --git a/app/scripts/test-client-diagnostic-metadata.cjs b/app/scripts/test-client-diagnostic-metadata.cjs new file mode 100644 index 000000000..349213d2d --- /dev/null +++ b/app/scripts/test-client-diagnostic-metadata.cjs @@ -0,0 +1,112 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const Module = require('node:module'); +const path = require('node:path'); +const ts = require('typescript'); + +function loadTypeScript(sourcePath, mocks) { + const source = fs.readFileSync(sourcePath, 'utf8'); + const compiled = ts.transpileModule(source, { + compilerOptions: { + esModuleInterop: true, + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2020, + strict: true, + }, + fileName: sourcePath, + }); + const loaded = new Module(sourcePath, module); + loaded.filename = sourcePath; + loaded.paths = Module._nodeModulePaths(path.dirname(sourcePath)); + const originalRequire = loaded.require.bind(loaded); + loaded.require = (request) => mocks[request] ?? originalRequire(request); + loaded._compile(compiled.outputText, sourcePath); + return loaded.exports; +} + +const application = { + applicationId: 'com.chronicle.app', + applicationName: 'Chronicle', + nativeApplicationVersion: '1.15.0', + nativeBuildVersion: '79', +}; +const constants = { + expoConfig: { version: 'wrong-config-version' }, + executionEnvironment: 'standalone', +}; +const updates = { + isEmbeddedLaunch: true, + updateId: 'update-123', + channel: 'testflight', + runtimeVersion: '1.15.0', + createdAt: new Date('2026-09-06T18:25:35Z'), +}; + +async function testLogHeader() { + let stored = ''; + const loggerPath = path.join(__dirname, '../src/utils/logger.ts'); + const logger = loadTypeScript(loggerPath, { + 'expo-application': application, + 'expo-constants': { __esModule: true, default: constants }, + 'expo-updates': updates, + 'react-native': { Platform: { OS: 'ios', Version: '26.6.1' } }, + 'expo-file-system/legacy': { + documentDirectory: 'memory://', + getInfoAsync: async () => ({ exists: stored.length > 0, isDirectory: false, size: stored.length }), + makeDirectoryAsync: async () => undefined, + readAsStringAsync: async () => stored, + writeAsStringAsync: async (_path, value) => { stored = value; }, + deleteAsync: async () => { stored = ''; }, + moveAsync: async () => undefined, + }, + }); + + await logger.initLogger(); + const text = await logger.readLog(); + assert.match(text, /appVersion=1\.15\.0/); + assert.match(text, /nativeBuildVersion=79/); + assert.match(text, /applicationId=com\.chronicle\.app/); + assert.match(text, /applicationName=Chronicle/); + assert.match(text, /executionEnvironment=standalone/); + assert.match(text, /updateId=update-123/); + assert.match(text, /channel=testflight/); + assert.match(text, /runtimeVersion=1\.15\.0/); + assert.match(text, /sessionId=[a-z0-9]{8}/); + assert.doesNotMatch(text, /Version=unknown/); +} + +async function testUploadHeaders() { + let request; + const clientPath = path.join(__dirname, '../src/services/clientDiagnostics.ts'); + const client = loadTypeScript(clientPath, { + 'expo-application': application, + 'expo-constants': { __esModule: true, default: constants }, + 'react-native': { Platform: { OS: 'ios', Version: '26.6.1' } }, + './auth': { + deriveBaseUrl: () => 'https://chronicle.example', + fetchAuthed: async (url, init) => { + request = { url, init }; + return { + ok: true, + json: async () => ({ app_version: '1.15.0', build_version: '79' }), + }; + }, + }, + '../utils/storage': { + getLastConnectedDeviceId: async () => null, + getWebSocketUrl: async () => 'wss://chronicle.example/ws/audio', + }, + }); + + await client.uploadClientDiagnostic('diagnostic body'); + assert.equal(request.init.headers['X-Chronicle-App-Version'], '1.15.0'); + assert.equal(request.init.headers['X-Chronicle-Build-Version'], '79'); + assert.notEqual(request.init.headers['X-Chronicle-Build-Version'], 'unknown'); +} + +Promise.all([testLogHeader(), testUploadHeaders()]) + .then(() => console.log('client diagnostic metadata tests passed')) + .catch((error) => { + console.error(error); + process.exitCode = 1; + }); diff --git a/app/src/services/clientDiagnostics.ts b/app/src/services/clientDiagnostics.ts index dcba1d91d..3dd72354c 100644 --- a/app/src/services/clientDiagnostics.ts +++ b/app/src/services/clientDiagnostics.ts @@ -1,3 +1,4 @@ +import * as Application from 'expo-application'; import Constants from 'expo-constants'; import { Platform } from 'react-native'; @@ -22,8 +23,8 @@ export async function uploadClientDiagnostic(contents: string): Promise { `sessionId=${sessionId}`, `time=${ts()}`, `platform=${Platform.OS} ${Platform.Version}`, - `appVersion=${Constants.expoConfig?.version ?? 'unknown'}`, - `nativeAppVersion=${(Constants as any).nativeAppVersion ?? 'unknown'}`, - `nativeBuildVersion=${(Constants as any).nativeBuildVersion ?? 'unknown'}`, + `appVersion=${Application.nativeApplicationVersion ?? 'unknown'}`, + `nativeAppVersion=${Application.nativeApplicationVersion ?? 'unknown'}`, + `nativeBuildVersion=${Application.nativeBuildVersion ?? 'unknown'}`, + `applicationId=${Application.applicationId ?? 'unknown'}`, + `applicationName=${Application.applicationName ?? 'unknown'}`, + `configuredAppVersion=${Constants.expoConfig?.version ?? 'unknown'}`, + `executionEnvironment=${Constants.executionEnvironment ?? 'unknown'}`, `updates: ${describeUpdatesState()}`, '=====================================================', '', From 5c6fe1059de777d759f9ead78afc88b2a8c3f1fd Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Sun, 6 Sep 2026 20:41:00 +0000 Subject: [PATCH 04/12] fix(ios): recover silent phone capture --- app/modules/chronicle-duplex-audio/index.ts | 5 +- .../ios/ChronicleDuplexAudioModule.swift | 151 ++++++++++++++---- .../ios/DuplexAudioState.swift | 52 ++++++ .../ios/Tests/DuplexAudioStateTests.swift | 75 +++++++++ app/scripts/test-phone-audio-diagnostics.cjs | 20 +++ app/src/hooks/usePhoneAudioRecorder.ts | 4 + 6 files changed, 274 insertions(+), 33 deletions(-) diff --git a/app/modules/chronicle-duplex-audio/index.ts b/app/modules/chronicle-duplex-audio/index.ts index c93dd5af2..a865830e6 100644 --- a/app/modules/chronicle-duplex-audio/index.ts +++ b/app/modules/chronicle-duplex-audio/index.ts @@ -29,9 +29,10 @@ export interface NativeCaptureDiagnostic { | 'pcm_converted' | 'pcm_conversion_failed' | 'pcm_empty' - | 'pcm_wrong_frame_count' | 'opus_encoded' - | 'opus_encode_failed'; + | 'opus_encode_failed' + | 'voice_processing_fallback' + | 'capture_failed'; monotonicTimestampMs: number; frameCount?: number; byteCount?: number; diff --git a/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift b/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift index 9ccc767b4..06eb177cd 100644 --- a/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift +++ b/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift @@ -6,10 +6,15 @@ public final class ChronicleDuplexAudioModule: Module { private let player = AVAudioPlayerNode() private let controlQueue = DispatchQueue(label: "chronicle.duplex.audio") private let captureDiagnosticLock = NSLock() + private let captureMetricsLock = NSLock() private var converter: AVAudioConverter? private var opusEncoder: ChronicleOpusPacketEncoder? + private var pcmPacketizer: ChroniclePcm16Packetizer? private var emittedCaptureDiagnosticStages = Set() private var captureEpoch = 0 + private var tapFrameCount = 0 + private var captureWatchdogGeneration = 0 + private var voiceProcessingFallbackForced = false private var voiceProcessingEnabled = false private var captureSuppressed = false private var currentResponse: (id: String, generation: Int)? @@ -75,7 +80,9 @@ public final class ChronicleDuplexAudioModule: Module { AsyncFunction("stopVoiceSession") { () async -> [String: Any?] in let restored = await self.onControlQueueValue { - self.tearDownEngine(deactivateSession: true) + let restored = self.tearDownEngine(deactivateSession: true) + self.voiceProcessingFallbackForced = false + return restored } return [ "restorationSucceeded": restored, @@ -150,14 +157,22 @@ public final class ChronicleDuplexAudioModule: Module { engine.connect(player, to: engine.mainMixerNode, format: outputFormat) let input = engine.inputNode - do { - try input.setVoiceProcessingEnabled(true) - voiceProcessingEnabled = input.isVoiceProcessingEnabled - } catch { + if voiceProcessingFallbackForced { + try? input.setVoiceProcessingEnabled(false) voiceProcessingEnabled = false + } else { + do { + try input.setVoiceProcessingEnabled(true) + voiceProcessingEnabled = input.isVoiceProcessingEnabled + } catch { + voiceProcessingEnabled = false + } } - let inputFormat = input.outputFormat(forBus: 0) + // iOS input taps must be installed with the hardware input format. After + // VoiceProcessingIO is enabled, outputFormat can produce a running engine + // whose input tap never receives a buffer. + let inputFormat = input.inputFormat(forBus: 0) let opusEncoder: ChronicleOpusPacketEncoder do { opusEncoder = try ChronicleOpusPacketEncoder() @@ -169,8 +184,11 @@ public final class ChronicleDuplexAudioModule: Module { } self.converter = converter self.opusEncoder = opusEncoder + self.pcmPacketizer = ChroniclePcm16Packetizer() + resetCaptureMetrics() let inputFrameCount = AVAudioFrameCount(round(inputFormat.sampleRate * 0.02)) input.installTap(onBus: 0, bufferSize: inputFrameCount, format: inputFormat) { [weak self] buffer, _ in + self?.observeTapFrame() self?.emitCaptureDiagnostic(stage: "tap_received") self?.emitOpus(buffer) } @@ -178,13 +196,15 @@ public final class ChronicleDuplexAudioModule: Module { engine.prepare() try engine.start() sessionRunning = true + scheduleCaptureWatchdog() } private func emitOpus(_ input: AVAudioPCMBuffer) { guard !captureSuppressed, engine.isRunning, let converter, - let opusEncoder else { return } + let opusEncoder, + let pcmPacketizer else { return } let capacity = ChronicleDuplexResampler.outputCapacity( inputFrames: input.frameLength, inputRate: input.format.sampleRate @@ -215,32 +235,97 @@ public final class ChronicleDuplexAudioModule: Module { return } emitCaptureDiagnostic(stage: "pcm_converted", frameCount: Int(output.frameLength)) - guard output.frameLength == ChronicleOpusPacketEncoder.framesPerPacket else { - emitCaptureDiagnostic(stage: "pcm_wrong_frame_count", frameCount: Int(output.frameLength)) + guard let samples = output.int16ChannelData?[0] else { + emitCaptureDiagnostic(stage: "pcm_conversion_failed", detail: "16 kHz PCM samples unavailable") return } - let data: Data - do { - data = try opusEncoder.encode(output) - } catch { - emitCaptureDiagnostic(stage: "opus_encode_failed", detail: String(describing: error)) - return + let packets = pcmPacketizer.append(samples: samples, count: Int(output.frameLength)) + let durationMs = 20.0 + let batchEndWallMs = Date().timeIntervalSince1970 * 1_000 + let batchEndMonotonicMs = ProcessInfo.processInfo.systemUptime * 1_000 + for (index, packet) in packets.enumerated() { + let data: Data + do { + data = try opusEncoder.encode(samples: packet) + } catch { + emitCaptureDiagnostic(stage: "opus_encode_failed", detail: String(describing: error)) + return + } + emitCaptureDiagnostic(stage: "opus_encoded", frameCount: packet.count, byteCount: data.count) + let packetOffsetMs = Double(packets.count - index) * durationMs + let audioLevel = packet.withUnsafeBufferPointer { + ChronicleAudioMeter.level(samples: $0.baseAddress!, count: $0.count) + } + sendEvent("onOpusFrame", [ + "captureEpoch": captureEpoch, + "capturedAtMs": batchEndWallMs - packetOffsetMs, + "monotonicTimestampMs": batchEndMonotonicMs - packetOffsetMs, + "sampleRate": 16_000, + "channels": 1, + "frameDurationMs": durationMs, + "audioLevel": audioLevel, + "opusBase64": data.base64EncodedString(), + ]) + } + } + + private func resetCaptureMetrics() { + captureMetricsLock.lock() + tapFrameCount = 0 + captureMetricsLock.unlock() + } + + private func observeTapFrame() { + captureMetricsLock.lock() + tapFrameCount += 1 + captureMetricsLock.unlock() + } + + private func capturedTapFrameCount() -> Int { + captureMetricsLock.lock() + let count = tapFrameCount + captureMetricsLock.unlock() + return count + } + + private func scheduleCaptureWatchdog() { + captureWatchdogGeneration += 1 + let generation = captureWatchdogGeneration + let epoch = captureEpoch + controlQueue.asyncAfter(deadline: .now() + 1.5) { [weak self] in + guard let self, + self.sessionRunning, + self.captureEpoch == epoch, + self.captureWatchdogGeneration == generation else { return } + let tapCount = self.capturedTapFrameCount() + switch DuplexCaptureWatchdog.recoveryAction( + tapFrameCount: tapCount, + voiceProcessingEnabled: self.voiceProcessingEnabled + ) { + case .none: + return + case .disableVoiceProcessing: + self.voiceProcessingFallbackForced = true + self.emitCaptureDiagnostic( + stage: "voice_processing_fallback", + detail: "no input tap after 1500ms taps=\(tapCount)" + ) + self.tearDownEngine(deactivateSession: false) + let payload: [String: Any] = [ + "captureEpoch": epoch, + "reason": "effect_failed", + "capabilities": self.capabilities(), + ] + DispatchQueue.main.async { [weak self] in + self?.sendEvent("onRouteChange", payload) + } + case .reportFailure: + self.emitCaptureDiagnostic( + stage: "capture_failed", + detail: "no input tap after voice-processing fallback taps=\(tapCount)" + ) + } } - emitCaptureDiagnostic(stage: "opus_encoded", frameCount: Int(output.frameLength), byteCount: data.count) - let durationMs = Double(output.frameLength) / 16_000 * 1_000 - let audioLevel = output.int16ChannelData.map { - ChronicleAudioMeter.level(samples: $0[0], count: Int(output.frameLength)) - } ?? 0 - sendEvent("onOpusFrame", [ - "captureEpoch": captureEpoch, - "capturedAtMs": Date().timeIntervalSince1970 * 1_000 - durationMs, - "monotonicTimestampMs": ProcessInfo.processInfo.systemUptime * 1_000 - durationMs, - "sampleRate": 16_000, - "channels": 1, - "frameDurationMs": durationMs, - "audioLevel": audioLevel, - "opusBase64": data.base64EncodedString(), - ]) } private func emitCaptureDiagnostic( @@ -261,7 +346,9 @@ public final class ChronicleDuplexAudioModule: Module { if let frameCount { payload["frameCount"] = frameCount } if let byteCount { payload["byteCount"] = byteCount } if let detail { payload["detail"] = String(detail.prefix(240)) } - sendEvent("onCaptureDiagnostic", payload) + DispatchQueue.main.async { [weak self] in + self?.sendEvent("onCaptureDiagnostic", payload) + } } private func schedule(response: [String: Any]) throws { @@ -436,6 +523,7 @@ public final class ChronicleDuplexAudioModule: Module { @discardableResult private func tearDownEngine(deactivateSession: Bool) -> Bool { var restored = true + captureWatchdogGeneration += 1 cancelCurrent(errorCode: nil) sessionRunning = false if tapInstalled { @@ -447,6 +535,7 @@ public final class ChronicleDuplexAudioModule: Module { if player.engine != nil { engine.detach(player) } converter = nil opusEncoder = nil + pcmPacketizer = nil voiceProcessingEnabled = false captureSuppressed = false if deactivateSession { diff --git a/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift b/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift index efa4fd429..3ae115bb6 100644 --- a/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift +++ b/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift @@ -90,6 +90,42 @@ enum ChronicleAudioMeter { } } +enum DuplexCaptureRecoveryAction: Equatable { + case none + case disableVoiceProcessing + case reportFailure +} + +enum DuplexCaptureWatchdog { + static func recoveryAction( + tapFrameCount: Int, + voiceProcessingEnabled: Bool + ) -> DuplexCaptureRecoveryAction { + guard tapFrameCount == 0 else { return .none } + return voiceProcessingEnabled ? .disableVoiceProcessing : .reportFailure + } +} + +final class ChroniclePcm16Packetizer { + private(set) var pendingSampleCount = 0 + private var pending: [Int16] = [] + + func append(samples: UnsafePointer, count: Int) -> [[Int16]] { + guard count > 0 else { return [] } + pending.append(contentsOf: UnsafeBufferPointer(start: samples, count: count)) + var packets: [[Int16]] = [] + let packetSize = Int(ChronicleOpusPacketEncoder.framesPerPacket) + let packetCount = pending.count / packetSize + for packetIndex in 0.. Data { + guard samples.count == Int(Self.framesPerPacket), + let input = AVAudioPCMBuffer( + pcmFormat: inputFormat, + frameCapacity: Self.framesPerPacket + ), + let target = input.int16ChannelData?[0] else { + throw ChronicleOpusEncoderError.formatUnavailable + } + input.frameLength = Self.framesPerPacket + samples.withUnsafeBufferPointer { source in + target.update(from: source.baseAddress!, count: source.count) + } + return try encode(input) + } } diff --git a/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift b/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift index 8e7889560..90f2e150c 100644 --- a/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift +++ b/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift @@ -87,6 +87,69 @@ final class DuplexAudioStateTests: XCTestCase { } } + func testCaptureWatchdogFallsBackWhenVoiceProcessingTapIsSilent() { + XCTAssertEqual( + DuplexCaptureWatchdog.recoveryAction( + tapFrameCount: 0, + voiceProcessingEnabled: true + ), + .disableVoiceProcessing + ) + } + + func testCaptureWatchdogLeavesDeliveringTapAlone() { + XCTAssertEqual( + DuplexCaptureWatchdog.recoveryAction( + tapFrameCount: 1, + voiceProcessingEnabled: true + ), + .none + ) + } + + func testCaptureWatchdogReportsFailureAfterFallbackTapIsSilent() { + XCTAssertEqual( + DuplexCaptureWatchdog.recoveryAction( + tapFrameCount: 0, + voiceProcessingEnabled: false + ), + .reportFailure + ) + } + + func testPacketizerSplitsLargePcmBuffersIntoTwentyMillisecondPackets() { + let packetizer = ChroniclePcm16Packetizer() + let samples = Array(0..<1_600).map(Int16.init) + let packets = samples.withUnsafeBufferPointer { + packetizer.append(samples: $0.baseAddress!, count: $0.count) + } + + XCTAssertEqual(packets.count, 5) + XCTAssertTrue(packets.allSatisfy { $0.count == 320 }) + XCTAssertEqual(packets[0].first, 0) + XCTAssertEqual(packets[4].last, 1_599) + XCTAssertEqual(packetizer.pendingSampleCount, 0) + } + + func testPacketizerCarriesPartialPcmAcrossTapCallbacks() { + let packetizer = ChroniclePcm16Packetizer() + let first = [Int16](repeating: 1, count: 100) + let second = [Int16](repeating: 2, count: 220) + + let initialPackets = first.withUnsafeBufferPointer { + packetizer.append(samples: $0.baseAddress!, count: $0.count) + } + let completedPackets = second.withUnsafeBufferPointer { + packetizer.append(samples: $0.baseAddress!, count: $0.count) + } + + XCTAssertTrue(initialPackets.isEmpty) + XCTAssertEqual(completedPackets.count, 1) + XCTAssertEqual(Array(completedPackets[0].prefix(100)), [Int16](repeating: 1, count: 100)) + XCTAssertEqual(Array(completedPackets[0].suffix(220)), [Int16](repeating: 2, count: 220)) + XCTAssertEqual(packetizer.pendingSampleCount, 0) + } + func testOpusEncoderProducesOnePacketForTwentyMillisecondsOfPcm() throws { let encoder = try ChronicleOpusPacketEncoder() let buffer = try XCTUnwrap( @@ -107,6 +170,18 @@ final class DuplexAudioStateTests: XCTestCase { XCTAssertLessThanOrEqual(packet.count, 1_275) } + func testOpusEncoderAcceptsOnePacketOfRawSamples() throws { + let encoder = try ChronicleOpusPacketEncoder() + let samples = (0.. { }, [markCaptureStopped]); const startNativeCapture = useCallback(async (): Promise => { + if (firstFrameTimeoutRef.current) clearTimeout(firstFrameTimeoutRef.current); + firstFrameTimeoutRef.current = null; + if (audioLevelTimeoutRef.current) clearTimeout(audioLevelTimeoutRef.current); + audioLevelTimeoutRef.current = null; const captureEpoch = captureEpochRef.current + 1; captureEpochRef.current = captureEpoch; const capabilities = await startVoiceSession({ captureEpoch }); From f2c217773cf3ba204af86c83d5607da082a50eab Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Sun, 6 Sep 2026 21:27:34 +0000 Subject: [PATCH 05/12] feat(ios): add one-tap audio diagnostic matrix --- app/app/settings.tsx | 7 + app/modules/chronicle-duplex-audio/index.ts | 37 +- .../ios/ChronicleDuplexAudioModule.swift | 165 +++++- .../ios/DuplexAudioState.swift | 23 + .../ios/Tests/DuplexAudioStateTests.swift | 17 + app/package.json | 2 +- app/scripts/test-phone-audio-diagnostics.cjs | 14 + app/scripts/test-phone-audio-self-test.cjs | 221 ++++++++ .../PhoneAudioDiagnosticsSection.tsx | 135 +++++ app/src/hooks/useAudioStreamer.ts | 3 + app/src/protocol/audioV2Socket.ts | 99 +++- app/src/services/phoneAudioDiagnostics.ts | 10 + app/src/services/phoneAudioSelfTest.ts | 497 ++++++++++++++++++ 13 files changed, 1203 insertions(+), 27 deletions(-) create mode 100644 app/scripts/test-phone-audio-self-test.cjs create mode 100644 app/src/components/PhoneAudioDiagnosticsSection.tsx create mode 100644 app/src/services/phoneAudioSelfTest.ts diff --git a/app/app/settings.tsx b/app/app/settings.tsx index 1e07ab9ab..fe123ea6c 100644 --- a/app/app/settings.tsx +++ b/app/app/settings.tsx @@ -4,6 +4,7 @@ import AuthSection from '@/components/AuthSection'; import BackendStatus from '@/components/BackendStatus'; import NetworkOverview from '@/components/NetworkOverview'; import NotificationsSection from '@/components/NotificationsSection'; +import PhoneAudioDiagnosticsSection from '@/components/PhoneAudioDiagnosticsSection'; import SystemAdminControls from '@/components/SystemAdminControls'; import { Screen, SectionLabel } from '@/components/ui'; import { useSharedAppSettings } from '@/contexts/AppSettingsContext'; @@ -30,6 +31,12 @@ export default function SettingsScreen() { authenticated={settings.isAuthenticated} /> + Diagnostics + + Administration diff --git a/app/modules/chronicle-duplex-audio/index.ts b/app/modules/chronicle-duplex-audio/index.ts index a865830e6..2bb290922 100644 --- a/app/modules/chronicle-duplex-audio/index.ts +++ b/app/modules/chronicle-duplex-audio/index.ts @@ -9,6 +9,11 @@ import type { VoiceCapabilities } from '../../src/protocol/audioCapabilities'; export interface StartVoiceSessionOptions { captureEpoch: number; + diagnosticProfile?: + | 'production' + | 'voice_processing_hold' + | 'plain_capture_hold' + | 'system_tap_format_hold'; } export interface NativeOpusFrame { @@ -32,7 +37,9 @@ export interface NativeCaptureDiagnostic { | 'opus_encoded' | 'opus_encode_failed' | 'voice_processing_fallback' - | 'capture_failed'; + | 'capture_failed' + | 'system_change' + | 'watchdog_evaluated'; monotonicTimestampMs: number; frameCount?: number; byteCount?: number; @@ -66,8 +73,32 @@ export interface NativeStopResult { failureCode: 'far_field_restore_failed' | 'permission_denied' | 'engine_unavailable' | null; } +export interface NativeVoiceSessionDiagnostics { + diagnosticProfile: NonNullable; + captureEpoch: number; + engineRunning: boolean; + sessionRunning: boolean; + tapInstalled: boolean; + tapFrameCount: number; + convertedFrameCount: number; + opusPacketCount: number; + opusByteCount: number; + peakAudioLevel: number; + systemChangeCount: number; + lastSystemChangeReason: string; + watchdogEvaluationCount: number; + voiceProcessingEnabled: boolean; + audioSessionCategory: string; + audioSessionMode: string; + audioSessionSampleRate: number; + audioSessionIOBufferDurationMs: number; + inputFormat: string; + outputFormat: string; +} + type ChronicleDuplexAudioNative = NativeModule & { startVoiceSession(options: StartVoiceSessionOptions): Promise; + getVoiceSessionDiagnostics(): Promise; scheduleResponse(response: NativeResponse): Promise; cancelResponse(responseId: string, generation: number): Promise; stopVoiceSession(): Promise; @@ -116,6 +147,10 @@ export function addOpusFrameListener( return requireNative().addListener('onOpusFrame', listener); } +export function getVoiceSessionDiagnostics(): Promise { + return requireNative().getVoiceSessionDiagnostics(); +} + export function addCaptureDiagnosticListener( listener: (event: NativeCaptureDiagnostic) => void ): EventSubscription { diff --git a/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift b/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift index 06eb177cd..c96870c5c 100644 --- a/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift +++ b/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift @@ -13,7 +13,15 @@ public final class ChronicleDuplexAudioModule: Module { private var emittedCaptureDiagnosticStages = Set() private var captureEpoch = 0 private var tapFrameCount = 0 + private var convertedFrameCount = 0 + private var opusPacketCount = 0 + private var opusByteCount = 0 + private var peakAudioLevel = 0.0 + private var systemChangeCount = 0 + private var lastSystemChangeReason = "none" + private var watchdogEvaluationCount = 0 private var captureWatchdogGeneration = 0 + private var diagnosticProfile = DuplexDiagnosticProfile.production private var voiceProcessingFallbackForced = false private var voiceProcessingEnabled = false private var captureSuppressed = false @@ -45,15 +53,25 @@ public final class ChronicleDuplexAudioModule: Module { guard let epoch = options["captureEpoch"] as? Int, epoch >= 0 else { throw Exception(name: "invalid_capture_epoch", description: "captureEpoch must be non-negative") } + let profileName = options["diagnosticProfile"] as? String ?? DuplexDiagnosticProfile.production.rawValue + guard let profile = DuplexDiagnosticProfile(rawValue: profileName) else { + throw Exception(name: "invalid_diagnostic_profile", description: "Unknown diagnosticProfile") + } guard await self.requestRecordPermission() else { throw Exception(name: "permission_denied", description: "Microphone permission denied") } return try await self.onControlQueue { - try self.startEngine(captureEpoch: epoch) + try self.startEngine(captureEpoch: epoch, diagnosticProfile: profile) return self.capabilities() } } + AsyncFunction("getVoiceSessionDiagnostics") { () async -> [String: Any] in + await self.onControlQueueValue { + self.voiceSessionDiagnostics() + } + } + AsyncFunction("scheduleResponse") { (response: [String: Any]) async throws in try await self.onControlQueue { try self.schedule(response: response) @@ -130,9 +148,13 @@ public final class ChronicleDuplexAudioModule: Module { } } - private func startEngine(captureEpoch: Int) throws { + private func startEngine( + captureEpoch: Int, + diagnosticProfile: DuplexDiagnosticProfile + ) throws { tearDownEngine(deactivateSession: false) self.captureEpoch = captureEpoch + self.diagnosticProfile = diagnosticProfile captureDiagnosticLock.lock() emittedCaptureDiagnosticStages.removeAll() captureDiagnosticLock.unlock() @@ -157,7 +179,23 @@ public final class ChronicleDuplexAudioModule: Module { engine.connect(player, to: engine.mainMixerNode, format: outputFormat) let input = engine.inputNode - if voiceProcessingFallbackForced { + if let forcedVoiceProcessing = diagnosticProfile.forcedVoiceProcessing { + if forcedVoiceProcessing { + do { + try input.setVoiceProcessingEnabled(true) + voiceProcessingEnabled = input.isVoiceProcessingEnabled + } catch { + voiceProcessingEnabled = false + emitCaptureDiagnostic( + stage: "capture_failed", + detail: "voice processing could not be enabled: \(error)" + ) + } + } else { + try? input.setVoiceProcessingEnabled(false) + voiceProcessingEnabled = false + } + } else if voiceProcessingFallbackForced { try? input.setVoiceProcessingEnabled(false) voiceProcessingEnabled = false } else { @@ -187,7 +225,8 @@ public final class ChronicleDuplexAudioModule: Module { self.pcmPacketizer = ChroniclePcm16Packetizer() resetCaptureMetrics() let inputFrameCount = AVAudioFrameCount(round(inputFormat.sampleRate * 0.02)) - input.installTap(onBus: 0, bufferSize: inputFrameCount, format: inputFormat) { [weak self] buffer, _ in + let tapFormat: AVAudioFormat? = diagnosticProfile.usesSystemTapFormat ? nil : inputFormat + input.installTap(onBus: 0, bufferSize: inputFrameCount, format: tapFormat) { [weak self] buffer, _ in self?.observeTapFrame() self?.emitCaptureDiagnostic(stage: "tap_received") self?.emitOpus(buffer) @@ -235,6 +274,7 @@ public final class ChronicleDuplexAudioModule: Module { return } emitCaptureDiagnostic(stage: "pcm_converted", frameCount: Int(output.frameLength)) + observeConvertedFrames(Int(output.frameLength)) guard let samples = output.int16ChannelData?[0] else { emitCaptureDiagnostic(stage: "pcm_conversion_failed", detail: "16 kHz PCM samples unavailable") return @@ -256,6 +296,7 @@ public final class ChronicleDuplexAudioModule: Module { let audioLevel = packet.withUnsafeBufferPointer { ChronicleAudioMeter.level(samples: $0.baseAddress!, count: $0.count) } + observeEncodedPacket(byteCount: data.count, audioLevel: audioLevel) sendEvent("onOpusFrame", [ "captureEpoch": captureEpoch, "capturedAtMs": batchEndWallMs - packetOffsetMs, @@ -272,6 +313,13 @@ public final class ChronicleDuplexAudioModule: Module { private func resetCaptureMetrics() { captureMetricsLock.lock() tapFrameCount = 0 + convertedFrameCount = 0 + opusPacketCount = 0 + opusByteCount = 0 + peakAudioLevel = 0 + systemChangeCount = 0 + lastSystemChangeReason = "none" + watchdogEvaluationCount = 0 captureMetricsLock.unlock() } @@ -288,6 +336,33 @@ public final class ChronicleDuplexAudioModule: Module { return count } + private func observeConvertedFrames(_ count: Int) { + captureMetricsLock.lock() + convertedFrameCount += count + captureMetricsLock.unlock() + } + + private func observeEncodedPacket(byteCount: Int, audioLevel: Double) { + captureMetricsLock.lock() + opusPacketCount += 1 + opusByteCount += byteCount + peakAudioLevel = max(peakAudioLevel, audioLevel) + captureMetricsLock.unlock() + } + + private func observeSystemChange(_ reason: String) { + captureMetricsLock.lock() + systemChangeCount += 1 + lastSystemChangeReason = reason + captureMetricsLock.unlock() + } + + private func observeWatchdogEvaluation() { + captureMetricsLock.lock() + watchdogEvaluationCount += 1 + captureMetricsLock.unlock() + } + private func scheduleCaptureWatchdog() { captureWatchdogGeneration += 1 let generation = captureWatchdogGeneration @@ -298,10 +373,25 @@ public final class ChronicleDuplexAudioModule: Module { self.captureEpoch == epoch, self.captureWatchdogGeneration == generation else { return } let tapCount = self.capturedTapFrameCount() - switch DuplexCaptureWatchdog.recoveryAction( + self.observeWatchdogEvaluation() + let action = DuplexCaptureWatchdog.recoveryAction( tapFrameCount: tapCount, voiceProcessingEnabled: self.voiceProcessingEnabled - ) { + ) + self.emitCaptureDiagnostic( + stage: "watchdog_evaluated", + detail: "profile=\(self.diagnosticProfile.rawValue) taps=\(tapCount) action=\(String(describing: action))" + ) + if self.diagnosticProfile != .production { + if tapCount == 0 { + self.emitCaptureDiagnostic( + stage: "capture_failed", + detail: "diagnostic profile produced no input tap after 1500ms" + ) + } + return + } + switch action { case .none: return case .disableVoiceProcessing: @@ -495,6 +585,52 @@ public final class ChronicleDuplexAudioModule: Module { ] } + private func voiceSessionDiagnostics() -> [String: Any] { + captureMetricsLock.lock() + let metrics: [String: Any] = [ + "tapFrameCount": tapFrameCount, + "convertedFrameCount": convertedFrameCount, + "opusPacketCount": opusPacketCount, + "opusByteCount": opusByteCount, + "peakAudioLevel": peakAudioLevel, + "systemChangeCount": systemChangeCount, + "lastSystemChangeReason": lastSystemChangeReason, + "watchdogEvaluationCount": watchdogEvaluationCount, + ] + captureMetricsLock.unlock() + + let session = AVAudioSession.sharedInstance() + let inputFormat = engine.inputNode.inputFormat(forBus: 0) + let outputFormat = engine.outputNode.outputFormat(forBus: 0) + return metrics.merging([ + "diagnosticProfile": diagnosticProfile.rawValue, + "captureEpoch": captureEpoch, + "engineRunning": engine.isRunning, + "sessionRunning": sessionRunning, + "tapInstalled": tapInstalled, + "voiceProcessingEnabled": voiceProcessingEnabled, + "audioSessionCategory": session.category.rawValue, + "audioSessionMode": session.mode.rawValue, + "audioSessionSampleRate": session.sampleRate, + "audioSessionIOBufferDurationMs": session.ioBufferDuration * 1_000, + "inputFormat": formatSummary(inputFormat), + "outputFormat": formatSummary(outputFormat), + ]) { _, newest in newest } + } + + private func formatSummary(_ format: AVAudioFormat) -> String { + let commonFormat: String + switch format.commonFormat { + case .pcmFormatFloat32: commonFormat = "float32" + case .pcmFormatFloat64: commonFormat = "float64" + case .pcmFormatInt16: commonFormat = "int16" + case .pcmFormatInt32: commonFormat = "int32" + case .otherFormat: commonFormat = "other" + @unknown default: commonFormat = "unknown" + } + return "\(Int(format.sampleRate))Hz/\(format.channelCount)ch/\(commonFormat)/\(format.isInterleaved ? "interleaved" : "noninterleaved")" + } + private func effect(requested: Bool, available: Bool, enabled: Bool) -> [String: Bool] { ["requested": requested, "available": available, "enabled": enabled] } @@ -585,8 +721,23 @@ public final class ChronicleDuplexAudioModule: Module { private func handleSystemChange(reason: String, errorCode: String) { controlQueue.async { [weak self] in - guard let self, self.sessionRunning else { return } + guard let self else { return } + self.observeSystemChange(reason) + let held = self.diagnosticProfile.holdsEngineOnSystemChange && self.sessionRunning + self.emitCaptureDiagnostic( + stage: "system_change", + detail: "reason=\(reason) session_running=\(self.sessionRunning) held=\(held) engine_running=\(self.engine.isRunning)" + ) + guard self.sessionRunning else { return } let changedCapabilities = self.capabilities() + if held { + self.sendEvent("onRouteChange", [ + "captureEpoch": self.captureEpoch, + "reason": reason, + "capabilities": changedCapabilities, + ]) + return + } self.cancelCurrent(errorCode: errorCode) self.tearDownEngine(deactivateSession: false) self.sendEvent("onRouteChange", [ diff --git a/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift b/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift index 3ae115bb6..5a8ddfd58 100644 --- a/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift +++ b/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift @@ -106,6 +106,29 @@ enum DuplexCaptureWatchdog { } } +enum DuplexDiagnosticProfile: String, CaseIterable { + case production + case voiceProcessingHold = "voice_processing_hold" + case plainCaptureHold = "plain_capture_hold" + case systemTapFormatHold = "system_tap_format_hold" + + var forcedVoiceProcessing: Bool? { + switch self { + case .production: return nil + case .voiceProcessingHold, .systemTapFormatHold: return true + case .plainCaptureHold: return false + } + } + + var usesSystemTapFormat: Bool { + self == .systemTapFormatHold + } + + var holdsEngineOnSystemChange: Bool { + self != .production + } +} + final class ChroniclePcm16Packetizer { private(set) var pendingSampleCount = 0 private var pending: [Int16] = [] diff --git a/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift b/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift index 90f2e150c..c0e922024 100644 --- a/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift +++ b/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift @@ -117,6 +117,23 @@ final class DuplexAudioStateTests: XCTestCase { ) } + func testDiagnosticMatrixChangesOneCaptureVariableAtATime() { + XCTAssertNil(DuplexDiagnosticProfile.production.forcedVoiceProcessing) + XCTAssertFalse(DuplexDiagnosticProfile.production.holdsEngineOnSystemChange) + + XCTAssertEqual(DuplexDiagnosticProfile.voiceProcessingHold.forcedVoiceProcessing, true) + XCTAssertTrue(DuplexDiagnosticProfile.voiceProcessingHold.holdsEngineOnSystemChange) + XCTAssertFalse(DuplexDiagnosticProfile.voiceProcessingHold.usesSystemTapFormat) + + XCTAssertEqual(DuplexDiagnosticProfile.plainCaptureHold.forcedVoiceProcessing, false) + XCTAssertTrue(DuplexDiagnosticProfile.plainCaptureHold.holdsEngineOnSystemChange) + XCTAssertFalse(DuplexDiagnosticProfile.plainCaptureHold.usesSystemTapFormat) + + XCTAssertEqual(DuplexDiagnosticProfile.systemTapFormatHold.forcedVoiceProcessing, true) + XCTAssertTrue(DuplexDiagnosticProfile.systemTapFormatHold.holdsEngineOnSystemChange) + XCTAssertTrue(DuplexDiagnosticProfile.systemTapFormatHold.usesSystemTapFormat) + } + func testPacketizerSplitsLargePcmBuffersIntoTwentyMillisecondPackets() { let packetizer = ChroniclePcm16Packetizer() let samples = Array(0..<1_600).map(Int16.init) diff --git a/app/package.json b/app/package.json index d681cd2fb..9a8a45a74 100644 --- a/app/package.json +++ b/app/package.json @@ -12,7 +12,7 @@ "typecheck": "tsc --noEmit", "test:wearable-activation": "node scripts/test-wearable-activation.cjs", "test:durable-audio-spool": "node scripts/test-durable-audio-spool.cjs", - "test:phone-audio-diagnostics": "node scripts/test-phone-audio-diagnostics.cjs && node scripts/test-client-diagnostic-metadata.cjs", + "test:phone-audio-diagnostics": "node scripts/test-phone-audio-diagnostics.cjs && node scripts/test-phone-audio-self-test.cjs && node scripts/test-client-diagnostic-metadata.cjs", "test:push-notifications": "node scripts/test-push-notifications.cjs", "check:theme": "node scripts/check-theme-tokens.mjs" }, diff --git a/app/scripts/test-phone-audio-diagnostics.cjs b/app/scripts/test-phone-audio-diagnostics.cjs index 53d33a175..e4596081e 100644 --- a/app/scripts/test-phone-audio-diagnostics.cjs +++ b/app/scripts/test-phone-audio-diagnostics.cjs @@ -52,6 +52,9 @@ diagnostics.audioLevelActive(0.5); diagnostics.socketUnavailable(0); diagnostics.socketUnavailable(0); diagnostics.socketConnecting(); +diagnostics.socketStage('transport_open'); +diagnostics.socketStage('client_hello_sent'); +diagnostics.socketStage('transport_error', 'wss://chronicle/ws/audio?token=secret-value'); diagnostics.socketOpen(); diagnostics.captureStarted('capture-secret-id'); diagnostics.frameEnqueued(44); @@ -74,6 +77,9 @@ assert.deepEqual( ['info', 'PhoneAudio'], ['info', 'PhoneAudio'], ['info', 'PhoneAudio'], + ['warn', 'PhoneAudio'], + ['info', 'PhoneAudio'], + ['info', 'PhoneAudio'], ['info', 'PhoneAudio'], ['info', 'PhoneAudio'], ['warn', 'PhoneAudio'], @@ -89,6 +95,9 @@ assert.match(text, /native_pcm_converted.*frames=320/); assert.match(text, /native_opus_encoded.*bytes=42/); assert.match(text, /audio_level_active.*audio_level=0\.500/); assert.match(text, /frame_dropped_socket_not_open.*ready_state=0/); +assert.match(text, /websocket_transport_open/); +assert.match(text, /websocket_client_hello_sent/); +assert.match(text, /websocket_transport_error.*token=/); assert.match(text, /first_frame_enqueued.*opus_bytes=44/); assert.match(text, /first_packet_accepted.*sequence=0/); assert.match( @@ -113,6 +122,11 @@ assert.match( /const optionsRef = useRef\(options\)/, 'rerenders must not replace the WebSocket lifecycle callback through a fresh options object', ); +assert.match( + integrationSources.streamer, + /onDiagnostic: event =>/, + 'production phone streaming must log each WebSocket handshake phase', +); assert.doesNotMatch( integrationSources.streamer, /\}, \[drainRecovery, encodeBase64, options, packetAccepted\]\);/, diff --git a/app/scripts/test-phone-audio-self-test.cjs b/app/scripts/test-phone-audio-self-test.cjs new file mode 100644 index 000000000..1332851ab --- /dev/null +++ b/app/scripts/test-phone-audio-self-test.cjs @@ -0,0 +1,221 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const Module = require('node:module'); +const path = require('node:path'); +const ts = require('typescript'); + +function loadTypeScript(sourcePath, mocks) { + const source = fs.readFileSync(sourcePath, 'utf8'); + const compiled = ts.transpileModule(source, { + compilerOptions: { + esModuleInterop: true, + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2020, + strict: true, + }, + fileName: sourcePath, + }); + const loaded = new Module(sourcePath, module); + loaded.filename = sourcePath; + loaded.paths = Module._nodeModulePaths(path.dirname(sourcePath)); + const originalRequire = loaded.require.bind(loaded); + loaded.require = request => mocks[request] ?? originalRequire(request); + loaded._compile(compiled.outputText, sourcePath); + return loaded.exports; +} + +const writes = []; +const sourcePath = path.join(__dirname, '../src/services/phoneAudioSelfTest.ts'); +const { runPhoneAudioDiagnosticSuite } = loadTypeScript(sourcePath, { + '../../modules/chronicle-duplex-audio': {}, + '../protocol/audioV2': { + DataPurpose: { ANNOTATION: 2 }, + DeliveryClass: { RECOVERED: 2 }, + DeviceKind: { IOS_PHONE: 1 }, + ProcessingProfile: { SOURCE_NATIVE: 2 }, + }, + '../protocol/audioV2Socket': { AudioV2Socket: class {} }, + '@/utils/logger': { + logInfo: (tag, message) => writes.push({ level: 'info', tag, message }), + logWarn: (tag, message) => writes.push({ level: 'warn', tag, message }), + logError: (tag, message) => writes.push({ level: 'error', tag, message }), + }, + 'react-native-base64': { + decode: value => Buffer.from(value, 'base64').toString('binary'), + }, + 'react-native': { Platform: { OS: 'ios' } }, +}); + +const profiles = []; +let frameListener = () => undefined; +let nativeListener = () => undefined; +let routeListener = () => undefined; +let stopCalls = 0; +let sentPackets = 0; +let closed = false; +const progress = []; + +const dependencies = { + now: (() => { + let value = 1_000_000; + return () => value += 25; + })(), + sleep: async () => undefined, + addOpusFrameListener(listener) { + frameListener = listener; + return { remove: () => { frameListener = () => undefined; } }; + }, + addCaptureDiagnosticListener(listener) { + nativeListener = listener; + return { remove: () => { nativeListener = () => undefined; } }; + }, + addRouteChangeListener(listener) { + routeListener = listener; + return { remove: () => { routeListener = () => undefined; } }; + }, + async startVoiceSession(options) { + profiles.push(options.diagnosticProfile); + nativeListener({ + captureEpoch: options.captureEpoch, + stage: 'system_change', + monotonicTimestampMs: 123, + detail: 'engine_reset ignored=true', + }); + routeListener({ + captureEpoch: options.captureEpoch, + reason: 'engine_reset', + capabilities: { + mode: 'duplex_full', + input_route: 'built_in_mic', + output_route: 'speakerphone', + native_sample_rate: 48_000, + aec: { requested: true, available: true, enabled: true }, + noise_suppression: { requested: true, available: true, enabled: true }, + fallback_reason: null, + }, + }); + if (options.diagnosticProfile === 'voice_processing_hold') { + for (let index = 0; index < 25; index += 1) { + frameListener({ + captureEpoch: options.captureEpoch, + capturedAtMs: 1_700_000_000_000 + (index * 20), + monotonicTimestampMs: 1000 + (index * 20), + sampleRate: 16_000, + channels: 1, + frameDurationMs: 20, + audioLevel: 0.25, + opusBase64: Buffer.from([0xf8, 0xff, 0xfe]).toString('base64'), + }); + } + } + return { + mode: 'duplex_full', + input_route: 'built_in_mic', + output_route: 'speakerphone', + native_sample_rate: 48_000, + aec: { requested: true, available: true, enabled: true }, + noise_suppression: { requested: true, available: true, enabled: true }, + fallback_reason: null, + }; + }, + async getVoiceSessionDiagnostics() { + const profile = profiles.at(-1); + const frames = profile === 'voice_processing_hold' ? 25 : 0; + return { + diagnosticProfile: profile, + captureEpoch: profiles.length, + engineRunning: frames > 0, + sessionRunning: frames > 0, + tapInstalled: true, + tapFrameCount: frames, + convertedFrameCount: frames * 320, + opusPacketCount: frames, + opusByteCount: frames * 3, + peakAudioLevel: frames ? 0.25 : 0, + systemChangeCount: 1, + lastSystemChangeReason: 'engine_reset', + watchdogEvaluationCount: 1, + voiceProcessingEnabled: profile !== 'plain_capture_hold', + audioSessionCategory: 'AVAudioSessionCategoryPlayAndRecord', + audioSessionMode: 'AVAudioSessionModeVoiceChat', + audioSessionSampleRate: 48_000, + audioSessionIOBufferDurationMs: 20, + inputFormat: '48000Hz/1ch/float32/noninterleaved', + outputFormat: '48000Hz/2ch/float32/noninterleaved', + }; + }, + async stopVoiceSession() { + stopCalls += 1; + return { restorationSucceeded: true, failureCode: null }; + }, + createSocket(options) { + return { + async connect() { + options.onDiagnostic?.({ stage: 'transport_open' }); + options.onDiagnostic?.({ stage: 'client_hello_sent' }); + options.onDiagnostic?.({ stage: 'server_hello_received' }); + }, + async beginCapture() { + return { + captureSessionId: { value: 'capture-diagnostic-full-id' }, + captureEpoch: 0n, + }; + }, + sendPacket(packet) { + sentPackets += 1; + options.onPacketAccepted?.(packet.sequence); + }, + async stopCapture() {}, + close() { closed = true; }, + }; + }, +}; + +(async () => { + const result = await runPhoneAudioDiagnosticSuite({ + backendUrl: 'wss://chronicle.example/ws/audio?token=must-not-log', + jwtToken: 'jwt-must-not-log', + onProgress: value => progress.push(value), + }, dependencies); + + assert.deepEqual(profiles, [ + 'production', + 'voice_processing_hold', + 'plain_capture_hold', + 'system_tap_format_hold', + ], 'the bounded matrix must distinguish reset handling, VoiceProcessingIO, and tap format'); + assert.equal(stopCalls, 4, 'every native probe must restore the audio session'); + assert.equal(sentPackets, 25, 'the backend probe must send a bounded half-second payload'); + assert.equal(closed, true, 'the backend diagnostic socket must always close'); + assert.equal(result.nativeProbes.filter(probe => probe.status === 'pass').length, 1); + assert.equal(result.networkProbe.status, 'pass'); + assert.equal(result.networkProbe.captureSessionId, 'capture-diagnostic-full-id'); + assert.equal(result.status, 'pass'); + assert.ok(progress.some(value => value.phase === 'native')); + assert.ok(progress.some(value => value.phase === 'network')); + assert.equal(progress.at(-1).phase, 'complete'); + + const logText = writes.map(({ tag, message }) => `${tag} ${message}`).join('\n'); + assert.match(logText, /profile=production status=fail/); + assert.match(logText, /profile=voice_processing_hold status=pass/); + assert.match(logText, /capture_session_id=capture-diagnostic-full-id/); + assert.match(logText, /payload_source=native_mic packets_sent=25 packets_acked=25/); + assert.match(logText, /event=suite_complete status=pass/); + assert.doesNotMatch(logText, /jwt-must-not-log|must-not-log/, 'diagnostics must never log credentials'); + assert.doesNotMatch(logText, /opusBase64/, 'diagnostics must never log captured audio payloads'); + + const settingsSource = fs.readFileSync(path.join(__dirname, '../app/settings.tsx'), 'utf8'); + const sectionSource = fs.readFileSync( + path.join(__dirname, '../src/components/PhoneAudioDiagnosticsSection.tsx'), + 'utf8', + ); + assert.match(settingsSource, / { + console.error(error); + process.exitCode = 1; +}); diff --git a/app/src/components/PhoneAudioDiagnosticsSection.tsx b/app/src/components/PhoneAudioDiagnosticsSection.tsx new file mode 100644 index 000000000..b8d2cfbe3 --- /dev/null +++ b/app/src/components/PhoneAudioDiagnosticsSection.tsx @@ -0,0 +1,135 @@ +import { useRouter } from 'expo-router'; +import React, { useState } from 'react'; +import { Alert, Platform, StyleSheet, View } from 'react-native'; + +import { Body, Button, ButtonRow, Caption, Card, CardWell, Mono } from '@/components/ui'; +import { + runPhoneAudioDiagnosticSuite, + type PhoneAudioDiagnosticProgress, + type PhoneAudioDiagnosticRunResult, +} from '@/services/phoneAudioSelfTest'; +import { useTheme, type Theme } from '@/theme'; + +interface PhoneAudioDiagnosticsSectionProps { + backendUrl: string; + jwtToken: string | null; +} + +function progressLabel(progress: PhoneAudioDiagnosticProgress | null): string { + if (!progress) return 'Ready'; + if (progress.phase === 'native') { + return `${progress.label} (${progress.current}/${progress.total})`; + } + return progress.label; +} + +export default function PhoneAudioDiagnosticsSection({ + backendUrl, + jwtToken, +}: PhoneAudioDiagnosticsSectionProps) { + const router = useRouter(); + const t = useTheme(); + const s = createStyles(t); + const [running, setRunning] = useState(false); + const [progress, setProgress] = useState(null); + const [result, setResult] = useState(null); + + const run = async () => { + setRunning(true); + setResult(null); + try { + const next = await runPhoneAudioDiagnosticSuite({ + backendUrl, + jwtToken, + onProgress: setProgress, + }); + setResult(next); + const nativePassed = next.nativeProbes.filter(probe => probe.status === 'pass').length; + Alert.alert( + next.status === 'pass' ? 'Audio checks complete' : 'Audio issue captured', + `Native profiles: ${nativePassed}/${next.nativeProbes.length}. Backend: ${next.networkProbe.status}. The full trace was appended to Device Log.`, + ); + } catch (cause) { + Alert.alert('Audio check failed', String(cause)); + } finally { + setRunning(false); + } + }; + + if (Platform.OS !== 'ios') { + return ( + + The exhaustive native audio matrix is currently available in the iOS TestFlight app. + + ); + } + + const nativePassed = result?.nativeProbes.filter(probe => probe.status === 'pass').length ?? 0; + return ( + + + Runs four bounded microphone configurations, records engine/tap/PCM/Opus counters, + then performs an authenticated Audio V2 capture with 25 packet acknowledgements. + + + Takes about 12 seconds. Stop any active phone stream first. If a microphone probe + succeeds, up to 0.5 seconds of its audio is sent as a diagnostic annotation; otherwise + the backend probe uses synthetic silence. Every result is appended to Device Log. + + + + {running ? progressLabel(progress) : result ? `Run ${result.runId}` : 'Ready to run'} + {result && ( + + Overall: {result.status} + Native: {nativePassed}/{result.nativeProbes.length} profiles + Backend: {result.networkProbe.status} + Packets: {result.networkProbe.packetsAcked}/{result.networkProbe.packetsSent} acked + {result.networkProbe.captureSessionId && ( + Capture: {result.networkProbe.captureSessionId} + )} + + )} + + + + + + + + ); +} + +const createStyles = (t: Theme) => StyleSheet.create({ + note: { + marginTop: t.space[2], + marginBottom: t.space[3], + }, + statusWell: { + marginBottom: t.space[3], + }, + resultRows: { + gap: t.space[1], + marginTop: t.space[2], + }, + logActions: { + marginTop: t.space[2], + }, +}); diff --git a/app/src/hooks/useAudioStreamer.ts b/app/src/hooks/useAudioStreamer.ts index 8f35c24f2..4d174ebd7 100644 --- a/app/src/hooks/useAudioStreamer.ts +++ b/app/src/hooks/useAudioStreamer.ts @@ -338,6 +338,9 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr }, delay); } }, + onDiagnostic: event => { + if (phoneVoice) phoneAudioDiagnostics.socketStage(event.stage, event.detail); + }, }); socketRef.current = socket; if (phoneVoice) phoneAudioDiagnostics.socketConnecting(); diff --git a/app/src/protocol/audioV2Socket.ts b/app/src/protocol/audioV2Socket.ts index 67a842db6..31a418b9f 100644 --- a/app/src/protocol/audioV2Socket.ts +++ b/app/src/protocol/audioV2Socket.ts @@ -53,7 +53,24 @@ export interface StartCaptureOptions { capabilities?: CaptureCapabilities; } -interface AudioV2SocketOptions { +export interface AudioV2SocketDiagnostic { + stage: + | 'socket_created' + | 'transport_open' + | 'client_hello_sent' + | 'server_hello_received' + | 'capture_start_sent' + | 'capture_started_received' + | 'capture_stop_sent' + | 'capture_stopped_received' + | 'server_error' + | 'transport_error' + | 'transport_closed' + | 'control_decode_failed'; + detail?: string; +} + +export interface AudioV2SocketOptions { url: string; bearerToken: string; sourceId: string; @@ -63,6 +80,7 @@ interface AudioV2SocketOptions { onPlaybackPacket?: (packet: PlaybackMediaPacket) => void; onControl?: (control: ServerControl) => void; onClosed?: () => void; + onDiagnostic?: (event: AudioV2SocketDiagnostic) => void; webSocketFactory?: (url: string, protocols: string | string[]) => WebSocket; } @@ -99,6 +117,7 @@ export class AudioV2Socket { private waiters = new Map void; reject: (error: Error) => void; + timeout: ReturnType; }>(); constructor(options: AudioV2SocketOptions) { @@ -117,26 +136,40 @@ export class AudioV2Socket { if (this.socket) throw new Error('audio-v2 socket already exists'); const factory = this.options.webSocketFactory ?? ((url, protocols) => new WebSocket(url, protocols)); const socket = factory(this.options.url, AUDIO_V2_SUBPROTOCOL); + this.diagnostic('socket_created'); socket.binaryType = 'arraybuffer'; this.socket = socket; const hello = this.waitFor('hello'); await new Promise((resolve, reject) => { socket.onopen = () => { - this.sendControl({ - case: 'hello', - value: create(ClientHelloSchema, { - bearerToken: this.options.bearerToken, - sourceId: create(CaptureSourceIdSchema, { value: this.options.sourceId }), - deviceKind: this.options.deviceKind, - displayName: this.options.displayName, - supportedUplink: [uplinkSpec()], - supportedDownlink: [downlinkSpec()], - }), - }); - resolve(); + this.diagnostic('transport_open'); + try { + this.sendControl({ + case: 'hello', + value: create(ClientHelloSchema, { + bearerToken: this.options.bearerToken, + sourceId: create(CaptureSourceIdSchema, { value: this.options.sourceId }), + deviceKind: this.options.deviceKind, + displayName: this.options.displayName, + supportedUplink: [uplinkSpec()], + supportedDownlink: [downlinkSpec()], + }), + }); + this.diagnostic('client_hello_sent'); + resolve(); + } catch (error) { + reject(error instanceof Error ? error : new Error(String(error))); + } + }; + socket.onerror = () => { + this.diagnostic('transport_error'); + reject(new Error('audio-v2 WebSocket failed')); }; - socket.onerror = () => reject(new Error('audio-v2 WebSocket failed')); - socket.onclose = () => { + socket.onclose = event => { + this.diagnostic( + 'transport_closed', + `code=${event.code} clean=${event.wasClean} reason=${event.reason.slice(0, 160)}`, + ); this.rejectWaiters(new Error('audio-v2 WebSocket closed')); this.options.onClosed?.(); }; @@ -160,6 +193,7 @@ export class AudioV2Socket { recoveryBatchId: options.recoveryBatchId ?? '', }), }); + this.diagnostic('capture_start_sent'); const control = await started; if (control.event.case !== 'captureStarted') { throw new Error('expected capture_started'); @@ -167,6 +201,7 @@ export class AudioV2Socket { const binding = control.event.value.binding; if (!binding?.captureSessionId?.value) throw new Error('capture start has no binding'); this.binding = binding; + this.diagnostic('capture_started_received'); return binding; } @@ -208,7 +243,9 @@ export class AudioV2Socket { case: 'stopCapture', value: create(StopCaptureSchema, { binding, reason }), }); + this.diagnostic('capture_stop_sent'); await stopped; + this.diagnostic('capture_stopped_received'); this.binding = null; this.currentDeliveryClass = DeliveryClass.UNSPECIFIED; } @@ -272,12 +309,24 @@ export class AudioV2Socket { private receive(payload: string | ArrayBuffer | Blob): void { if (typeof payload === 'string') { - const control = decodeServerControl(payload); + let control: ServerControl; + try { + control = decodeServerControl(payload); + } catch (error) { + this.diagnostic( + 'control_decode_failed', + error instanceof Error ? error.message.slice(0, 160) : String(error).slice(0, 160), + ); + this.rejectWaiters(new Error('audio-v2 server control could not be decoded')); + return; + } const event = control.event.case; if (event === 'error') { + this.diagnostic('server_error', control.event.value.detail.slice(0, 160)); this.rejectWaiters(new Error(control.event.value.detail)); return; } + if (event === 'hello') this.diagnostic('server_hello_received'); if (event === 'captureStarted') this.binding = control.event.value.binding ?? null; if (event === 'capturePacketAccepted') { this.options.onPacketAccepted?.(Number(control.event.value.sequence)); @@ -285,6 +334,7 @@ export class AudioV2Socket { const waiter = event ? this.waiters.get(event as AwaitedEvent) : undefined; if (waiter) { this.waiters.delete(event as AwaitedEvent); + clearTimeout(waiter.timeout); waiter.resolve(control); } this.options.onControl?.(control); @@ -302,14 +352,27 @@ export class AudioV2Socket { private waitFor(event: AwaitedEvent): Promise { if (this.waiters.has(event)) throw new Error(`already waiting for ${event}`); - return new Promise((resolve, reject) => this.waiters.set(event, { resolve, reject })); + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.waiters.delete(event); + reject(new Error(`audio-v2 timed out waiting for ${event}`)); + }, 10_000); + this.waiters.set(event, { resolve, reject, timeout }); + }); } private rejectWaiters(error: Error): void { - this.waiters.forEach(waiter => waiter.reject(error)); + this.waiters.forEach(waiter => { + clearTimeout(waiter.timeout); + waiter.reject(error); + }); this.waiters.clear(); } + private diagnostic(stage: AudioV2SocketDiagnostic['stage'], detail?: string): void { + this.options.onDiagnostic?.({ stage, detail }); + } + private requireOpen(): WebSocket { if (!this.socket || this.socket.readyState !== WebSocket.OPEN) { throw new Error('audio-v2 WebSocket is not open'); diff --git a/app/src/services/phoneAudioDiagnostics.ts b/app/src/services/phoneAudioDiagnostics.ts index 87dffd21e..dff0f2769 100644 --- a/app/src/services/phoneAudioDiagnostics.ts +++ b/app/src/services/phoneAudioDiagnostics.ts @@ -127,6 +127,16 @@ export class PhoneAudioDiagnostics { this.once('info', 'websocket_connecting'); } + socketStage(stage: string, detail?: string): void { + if (!this.active) return; + const safeDetail = redactSecrets(detail ?? '').replace(/[\r\n]+/g, ' ').slice(0, 180); + this.once( + stage.endsWith('error') || stage.endsWith('failed') ? 'warn' : 'info', + `websocket_${stage}`, + safeDetail ? `detail=${safeDetail}` : '', + ); + } + socketOpen(): void { this.once('info', 'websocket_open'); } diff --git a/app/src/services/phoneAudioSelfTest.ts b/app/src/services/phoneAudioSelfTest.ts new file mode 100644 index 000000000..71207f712 --- /dev/null +++ b/app/src/services/phoneAudioSelfTest.ts @@ -0,0 +1,497 @@ +import { Platform } from 'react-native'; +// @ts-ignore - no type declarations available +import base64 from 'react-native-base64'; + +import { + addCaptureDiagnosticListener, + addOpusFrameListener, + addRouteChangeListener, + getVoiceSessionDiagnostics, + startVoiceSession, + stopVoiceSession, + type NativeCaptureDiagnostic, + type NativeOpusFrame, + type NativeRouteChange, + type NativeStopResult, + type NativeVoiceSessionDiagnostics, + type StartVoiceSessionOptions, +} from '../../modules/chronicle-duplex-audio'; +import { + DataPurpose, + DeliveryClass, + DeviceKind, + ProcessingProfile, +} from '../protocol/audioV2'; +import { + AudioV2Socket, + type AudioV2SocketDiagnostic, + type AudioV2SocketOptions, + type CapturePacket, + type StartCaptureOptions, +} from '../protocol/audioV2Socket'; +import { logError, logInfo, logWarn } from '@/utils/logger'; + +type DiagnosticProfile = NonNullable; +type DiagnosticStatus = 'pass' | 'fail' | 'skipped'; + +const PROFILES: DiagnosticProfile[] = [ + 'production', + 'voice_processing_hold', + 'plain_capture_hold', + 'system_tap_format_hold', +]; +const PROBE_DURATION_MS = 2_250; +const PROBE_SETTLE_MS = 250; +const NETWORK_PACKET_COUNT = 25; +const NETWORK_TIMEOUT_MS = 10_000; +const SYNTHETIC_OPUS_SILENCE = new Uint8Array([0xf8, 0xff, 0xfe]); + +interface Subscription { + remove(): void; +} + +interface DiagnosticSocket { + connect(): Promise; + beginCapture(options: StartCaptureOptions): Promise<{ + captureSessionId?: { value?: string }; + }>; + sendPacket(packet: CapturePacket): void; + stopCapture(): Promise; + close(): void; +} + +export interface PhoneAudioDiagnosticDependencies { + now(): number; + sleep(milliseconds: number): Promise; + addOpusFrameListener(listener: (event: NativeOpusFrame) => void): Subscription; + addCaptureDiagnosticListener(listener: (event: NativeCaptureDiagnostic) => void): Subscription; + addRouteChangeListener(listener: (event: NativeRouteChange) => void): Subscription; + startVoiceSession(options: StartVoiceSessionOptions): Promise; + getVoiceSessionDiagnostics(): Promise; + stopVoiceSession(): Promise; + createSocket(options: AudioV2SocketOptions): DiagnosticSocket; +} + +type VoiceCapabilitiesResult = Awaited>; + +export interface PhoneAudioDiagnosticProgress { + phase: 'native' | 'network' | 'complete'; + label: string; + current: number; + total: number; +} + +export interface NativeProbeResult { + profile: DiagnosticProfile; + status: Exclude; + elapsedMs: number; + frameCount: number; + failure: string | null; + snapshot: NativeVoiceSessionDiagnostics | null; +} + +export interface NetworkProbeResult { + status: DiagnosticStatus; + elapsedMs: number; + phase: string; + payloadSource: 'native_mic' | 'synthetic_silence' | 'none'; + packetsSent: number; + packetsAcked: number; + captureSessionId: string | null; + failure: string | null; +} + +export interface PhoneAudioDiagnosticRunResult { + runId: string; + status: 'pass' | 'partial' | 'fail'; + elapsedMs: number; + nativeProbes: NativeProbeResult[]; + networkProbe: NetworkProbeResult; +} + +export interface PhoneAudioDiagnosticRunOptions { + backendUrl: string; + jwtToken: string | null; + onProgress?: (progress: PhoneAudioDiagnosticProgress) => void; +} + +interface CapturedPacket { + capturedAtMs: number; + monotonicTimestampMs: number; + opus: Uint8Array; +} + +const defaultDependencies: PhoneAudioDiagnosticDependencies = { + now: Date.now, + sleep: milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)), + addOpusFrameListener, + addCaptureDiagnosticListener, + addRouteChangeListener, + startVoiceSession, + getVoiceSessionDiagnostics, + stopVoiceSession, + createSocket: options => new AudioV2Socket(options), +}; + +function safeText(value: unknown, maximum = 300): string { + return String(value) + .replace(/([?&](?:token|access_token)=)[^&\s]+/gi, '$1') + .replace(/Bearer\s+[^\s]+/gi, 'Bearer ') + .replace(/[\r\n]+/g, ' ') + .slice(0, maximum); +} + +function decodeBase64(value: string): Uint8Array { + const binary = base64.decode(value); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) { + bytes[index] = binary.charCodeAt(index); + } + return bytes; +} + +function write( + runId: string, + level: 'info' | 'warn' | 'error', + event: string, + details = '', +): void { + const message = `run_id=${runId} event=${event}${details ? ` ${details}` : ''}`; + if (level === 'error') logError('PhoneAudioSelfTest', message); + else if (level === 'warn') logWarn('PhoneAudioSelfTest', message); + else logInfo('PhoneAudioSelfTest', message); +} + +async function withTimeout( + operation: Promise, + milliseconds: number, + label: string, +): Promise { + let timeout: ReturnType | null = null; + const deadline = new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error(`${label} timed out after ${milliseconds}ms`)), milliseconds); + }); + try { + return await Promise.race([operation, deadline]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +function backendSocketUrl(value: string): string { + const url = new URL(value.trim()); + if (url.protocol === 'http:') url.protocol = 'ws:'; + if (url.protocol === 'https:') url.protocol = 'wss:'; + if (url.protocol !== 'ws:' && url.protocol !== 'wss:') { + throw new Error('Chronicle backend URL must use HTTP(S) or WS(S)'); + } + url.pathname = '/ws/audio'; + url.search = ''; + url.hash = ''; + return url.toString(); +} + +function packetsForNetwork(nativePackets: CapturedPacket[], nowMs: number): { + source: NetworkProbeResult['payloadSource']; + packets: CapturePacket[]; +} { + const source = nativePackets.length ? 'native_mic' : 'synthetic_silence'; + const firstMonotonic = nativePackets[0]?.monotonicTimestampMs ?? 0; + const packets = Array.from({ length: NETWORK_PACKET_COUNT }, (_, sequence) => { + const native = nativePackets[sequence % Math.max(1, nativePackets.length)]; + return { + sequence, + capturedAtMs: nowMs + (sequence * 20), + monotonicOffsetUs: native + ? Math.max(0, Math.round((native.monotonicTimestampMs - firstMonotonic) * 1_000)) + : sequence * 20_000, + opus: native?.opus ?? SYNTHETIC_OPUS_SILENCE, + }; + }); + return { source, packets }; +} + +async function runNetworkProbe( + runId: string, + options: PhoneAudioDiagnosticRunOptions, + nativePackets: CapturedPacket[], + dependencies: PhoneAudioDiagnosticDependencies, +): Promise { + const startedAt = dependencies.now(); + if (!options.backendUrl.trim() || !options.jwtToken) { + write(runId, 'warn', 'network_skipped', 'reason=backend_or_auth_not_configured'); + return { + status: 'skipped', + elapsedMs: 0, + phase: 'configuration', + payloadSource: 'none', + packetsSent: 0, + packetsAcked: 0, + captureSessionId: null, + failure: 'Backend URL or authentication is not configured', + }; + } + + let phase = 'construct'; + let packetsSent = 0; + let captureSessionId: string | null = null; + const acked = new Set(); + let finishAcknowledgements: () => void = () => {}; + const acknowledgements = new Promise(resolve => { + finishAcknowledgements = resolve; + }); + let socket: DiagnosticSocket | null = null; + try { + const url = backendSocketUrl(options.backendUrl); + const payload = packetsForNetwork(nativePackets, dependencies.now()); + write( + runId, + 'info', + 'network_started', + `endpoint=${new URL(url).host} payload_source=${payload.source} packet_target=${payload.packets.length}`, + ); + socket = dependencies.createSocket({ + url, + bearerToken: options.jwtToken, + sourceId: 'phone-audio-diagnostics', + displayName: 'phone-audio-diagnostics', + deviceKind: Platform.OS === 'ios' ? DeviceKind.IOS_PHONE : DeviceKind.ANDROID_PHONE, + onPacketAccepted: sequence => { + acked.add(sequence); + if (acked.size >= payload.packets.length) finishAcknowledgements(); + }, + onDiagnostic: (event: AudioV2SocketDiagnostic) => { + phase = event.stage; + write( + runId, + event.stage.endsWith('error') || event.stage.endsWith('failed') ? 'warn' : 'info', + 'network_phase', + `phase=${event.stage}${event.detail ? ` detail=${safeText(event.detail, 180)}` : ''}`, + ); + }, + }); + + phase = 'connect'; + await withTimeout(socket.connect(), NETWORK_TIMEOUT_MS, 'WebSocket hello'); + phase = 'begin_capture'; + const binding = await withTimeout(socket.beginCapture({ + captureEpoch: 0, + processingProfile: ProcessingProfile.SOURCE_NATIVE, + dataPurpose: DataPurpose.ANNOTATION, + deliveryClass: DeliveryClass.RECOVERED, + recoveryBatchId: `phone-diag-${runId}`, + }), NETWORK_TIMEOUT_MS, 'backend capture start'); + captureSessionId = binding.captureSessionId?.value ?? null; + if (!captureSessionId) throw new Error('backend returned no capture session ID'); + write(runId, 'info', 'network_capture_bound', `capture_session_id=${captureSessionId}`); + + phase = 'send_packets'; + for (const packet of payload.packets) { + socket.sendPacket(packet); + packetsSent += 1; + } + await withTimeout(acknowledgements, NETWORK_TIMEOUT_MS, 'packet acknowledgements'); + phase = 'stop_capture'; + await withTimeout(socket.stopCapture(), NETWORK_TIMEOUT_MS, 'backend capture stop'); + + const result: NetworkProbeResult = { + status: acked.size === packetsSent ? 'pass' : 'fail', + elapsedMs: Math.max(0, dependencies.now() - startedAt), + phase: 'complete', + payloadSource: payload.source, + packetsSent, + packetsAcked: acked.size, + captureSessionId, + failure: null, + }; + write( + runId, + result.status === 'pass' ? 'info' : 'error', + 'network_result', + `status=${result.status} capture_session_id=${captureSessionId} payload_source=${payload.source} packets_sent=${packetsSent} packets_acked=${acked.size} elapsed_ms=${result.elapsedMs}`, + ); + return result; + } catch (cause) { + const failure = safeText(cause instanceof Error ? cause.message : cause); + const result: NetworkProbeResult = { + status: 'fail', + elapsedMs: Math.max(0, dependencies.now() - startedAt), + phase, + payloadSource: nativePackets.length ? 'native_mic' : 'synthetic_silence', + packetsSent, + packetsAcked: acked.size, + captureSessionId, + failure, + }; + write( + runId, + 'error', + 'network_result', + `status=fail phase=${phase} capture_session_id=${captureSessionId ?? 'none'} packets_sent=${packetsSent} packets_acked=${acked.size} error=${failure}`, + ); + return result; + } finally { + socket?.close(); + } +} + +export async function runPhoneAudioDiagnosticSuite( + options: PhoneAudioDiagnosticRunOptions, + dependencies: PhoneAudioDiagnosticDependencies = defaultDependencies, +): Promise { + const suiteStartedAt = dependencies.now(); + const runId = `${Math.floor(suiteStartedAt).toString(36)}-${Math.random().toString(36).slice(2, 8)}`; + const baseEpoch = Math.max(1, Math.floor(suiteStartedAt % 2_000_000_000)); + const nativeProbes: NativeProbeResult[] = []; + let activeEpoch = -1; + let activePackets: CapturedPacket[] = []; + let bestPackets: CapturedPacket[] = []; + + write( + runId, + 'info', + 'suite_started', + `platform=${Platform.OS} native_profiles=${PROFILES.join(',')} probe_duration_ms=${PROBE_DURATION_MS} network_packets=${NETWORK_PACKET_COUNT}`, + ); + + const frameSubscription = dependencies.addOpusFrameListener(frame => { + if (frame.captureEpoch !== activeEpoch || activePackets.length >= NETWORK_PACKET_COUNT) return; + try { + const opus = decodeBase64(frame.opusBase64); + if (!opus.length) return; + activePackets.push({ + capturedAtMs: frame.capturedAtMs, + monotonicTimestampMs: frame.monotonicTimestampMs, + opus, + }); + } catch (cause) { + write(runId, 'warn', 'native_frame_decode_failed', `error=${safeText(cause)}`); + } + }); + const nativeSubscription = dependencies.addCaptureDiagnosticListener(event => { + if (event.captureEpoch !== activeEpoch) return; + write( + runId, + event.stage.endsWith('_failed') ? 'warn' : 'info', + 'native_stage', + [ + `profile=${PROFILES[nativeProbes.length] ?? 'unknown'}`, + `stage=${event.stage}`, + event.frameCount === undefined ? '' : `frames=${event.frameCount}`, + event.byteCount === undefined ? '' : `bytes=${event.byteCount}`, + event.detail ? `detail=${safeText(event.detail, 220)}` : '', + ].filter(Boolean).join(' '), + ); + }); + const routeSubscription = dependencies.addRouteChangeListener(event => { + if (event.captureEpoch !== activeEpoch) return; + write( + runId, + 'info', + 'native_route_event', + `profile=${PROFILES[nativeProbes.length] ?? 'unknown'} reason=${event.reason} input=${event.capabilities.input_route} output=${event.capabilities.output_route} mode=${event.capabilities.mode}`, + ); + }); + + try { + for (const [index, profile] of PROFILES.entries()) { + options.onProgress?.({ + phase: 'native', + label: `Testing ${profile.replace(/_/g, ' ')}`, + current: index + 1, + total: PROFILES.length, + }); + activeEpoch = baseEpoch + index; + activePackets = []; + const probeStartedAt = dependencies.now(); + let snapshot: NativeVoiceSessionDiagnostics | null = null; + let failure: string | null = null; + let capabilities: VoiceCapabilitiesResult | null = null; + write(runId, 'info', 'native_probe_started', `profile=${profile} capture_epoch=${activeEpoch}`); + try { + capabilities = await dependencies.startVoiceSession({ + captureEpoch: activeEpoch, + diagnosticProfile: profile, + }); + write( + runId, + 'info', + 'native_probe_capabilities', + `profile=${profile} mode=${capabilities.mode} input=${capabilities.input_route} output=${capabilities.output_route} sample_rate=${capabilities.native_sample_rate} aec=${capabilities.aec.enabled} noise_suppression=${capabilities.noise_suppression.enabled} fallback=${capabilities.fallback_reason ?? 'none'}`, + ); + await dependencies.sleep(PROBE_DURATION_MS); + snapshot = await dependencies.getVoiceSessionDiagnostics(); + } catch (cause) { + failure = safeText(cause instanceof Error ? cause.message : cause); + try { + snapshot = await dependencies.getVoiceSessionDiagnostics(); + } catch { + // The start failure remains the useful signal. + } + } finally { + try { + const restoration = await dependencies.stopVoiceSession(); + write( + runId, + restoration.restorationSucceeded ? 'info' : 'warn', + 'native_probe_cleanup', + `profile=${profile} restoration_succeeded=${restoration.restorationSucceeded} failure_code=${restoration.failureCode ?? 'none'}`, + ); + } catch (cause) { + write(runId, 'warn', 'native_probe_cleanup', `profile=${profile} error=${safeText(cause)}`); + } + } + + const frameCount = Math.max(activePackets.length, snapshot?.opusPacketCount ?? 0); + if (!failure && frameCount === 0) failure = 'no_opus_frames'; + const result: NativeProbeResult = { + profile, + status: failure ? 'fail' : 'pass', + elapsedMs: Math.max(0, dependencies.now() - probeStartedAt), + frameCount, + failure, + snapshot, + }; + nativeProbes.push(result); + if (result.status === 'pass' && activePackets.length > bestPackets.length) { + bestPackets = [...activePackets]; + } + write( + runId, + result.status === 'pass' ? 'info' : 'warn', + 'native_probe_result', + `profile=${profile} status=${result.status} elapsed_ms=${result.elapsedMs} js_frames=${activePackets.length} failure=${failure ?? 'none'} snapshot=${JSON.stringify(snapshot)}`, + ); + await dependencies.sleep(PROBE_SETTLE_MS); + } + } finally { + activeEpoch = -1; + frameSubscription.remove(); + nativeSubscription.remove(); + routeSubscription.remove(); + } + + options.onProgress?.({ phase: 'network', label: 'Testing Chronicle backend', current: 1, total: 1 }); + const networkProbe = await runNetworkProbe(runId, options, bestPackets, dependencies); + const nativePassCount = nativeProbes.filter(probe => probe.status === 'pass').length; + const status: PhoneAudioDiagnosticRunResult['status'] = nativePassCount > 0 && networkProbe.status === 'pass' + ? 'pass' + : nativePassCount === 0 && networkProbe.status === 'fail' + ? 'fail' + : 'partial'; + const result: PhoneAudioDiagnosticRunResult = { + runId, + status, + elapsedMs: Math.max(0, dependencies.now() - suiteStartedAt), + nativeProbes, + networkProbe, + }; + write( + runId, + status === 'pass' ? 'info' : status === 'partial' ? 'warn' : 'error', + 'suite_complete', + `status=${status} native_passed=${nativePassCount}/${nativeProbes.length} network_status=${networkProbe.status} elapsed_ms=${result.elapsedMs}`, + ); + options.onProgress?.({ phase: 'complete', label: 'Diagnostic run complete', current: 1, total: 1 }); + return result; +} From a4b2491b740ba4a5e1722fd983e5c3d26b414e2e Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:09:02 +0000 Subject: [PATCH 06/12] fix(ios): preserve capture during route settlement --- .../ios/ChronicleDuplexAudioModule.swift | 6 ++- .../ios/DuplexAudioState.swift | 18 +++++++++ .../ios/Tests/DuplexAudioStateTests.swift | 39 +++++++++++++++++++ app/scripts/test-phone-audio-diagnostics.cjs | 21 ++++++++++ app/src/protocol/audioV2Socket.ts | 26 ++++++++++++- 5 files changed, 108 insertions(+), 2 deletions(-) diff --git a/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift b/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift index c96870c5c..976afbdc3 100644 --- a/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift +++ b/app/modules/chronicle-duplex-audio/ios/ChronicleDuplexAudioModule.swift @@ -723,7 +723,11 @@ public final class ChronicleDuplexAudioModule: Module { controlQueue.async { [weak self] in guard let self else { return } self.observeSystemChange(reason) - let held = self.diagnosticProfile.holdsEngineOnSystemChange && self.sessionRunning + let held = DuplexSystemChangePolicy.shouldHoldEngine( + reason: reason, + sessionRunning: self.sessionRunning, + diagnosticProfile: self.diagnosticProfile + ) self.emitCaptureDiagnostic( stage: "system_change", detail: "reason=\(reason) session_running=\(self.sessionRunning) held=\(held) engine_running=\(self.engine.isRunning)" diff --git a/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift b/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift index 5a8ddfd58..467640f31 100644 --- a/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift +++ b/app/modules/chronicle-duplex-audio/ios/DuplexAudioState.swift @@ -106,6 +106,24 @@ enum DuplexCaptureWatchdog { } } +enum DuplexSystemChangePolicy { + static func shouldHoldEngine( + reason: String, + sessionRunning: Bool, + diagnosticProfile: DuplexDiagnosticProfile + ) -> Bool { + guard sessionRunning else { return false } + if diagnosticProfile.holdsEngineOnSystemChange { return true } + + // AVAudioSession emits route-change notifications while the initial + // play-and-record route is settling. Tearing down here leaves the engine + // looking successfully started but prevents its input tap from ever + // delivering a frame. A later, real route change is still forwarded to JS, + // whose bound capture lifecycle performs the restart. + return reason == "route_changed" + } +} + enum DuplexDiagnosticProfile: String, CaseIterable { case production case voiceProcessingHold = "voice_processing_hold" diff --git a/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift b/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift index c0e922024..29472beea 100644 --- a/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift +++ b/app/modules/chronicle-duplex-audio/ios/Tests/DuplexAudioStateTests.swift @@ -134,6 +134,45 @@ final class DuplexAudioStateTests: XCTestCase { XCTAssertTrue(DuplexDiagnosticProfile.systemTapFormatHold.usesSystemTapFormat) } + func testProductionKeepsCapturingThroughInitialRouteSettlement() { + XCTAssertTrue( + DuplexSystemChangePolicy.shouldHoldEngine( + reason: "route_changed", + sessionRunning: true, + diagnosticProfile: .production + ) + ) + } + + func testProductionStillRestartsForEngineResetAndInterruption() { + for reason in ["engine_reset", "interruption"] { + XCTAssertFalse( + DuplexSystemChangePolicy.shouldHoldEngine( + reason: reason, + sessionRunning: true, + diagnosticProfile: .production + ) + ) + } + } + + func testDiagnosticHoldProfilesCanObserveEverySystemChange() { + XCTAssertTrue( + DuplexSystemChangePolicy.shouldHoldEngine( + reason: "engine_reset", + sessionRunning: true, + diagnosticProfile: .voiceProcessingHold + ) + ) + XCTAssertFalse( + DuplexSystemChangePolicy.shouldHoldEngine( + reason: "route_changed", + sessionRunning: false, + diagnosticProfile: .voiceProcessingHold + ) + ) + } + func testPacketizerSplitsLargePcmBuffersIntoTwentyMillisecondPackets() { let packetizer = ChroniclePcm16Packetizer() let samples = Array(0..<1_600).map(Int16.init) diff --git a/app/scripts/test-phone-audio-diagnostics.cjs b/app/scripts/test-phone-audio-diagnostics.cjs index e4596081e..e788665ee 100644 --- a/app/scripts/test-phone-audio-diagnostics.cjs +++ b/app/scripts/test-phone-audio-diagnostics.cjs @@ -25,6 +25,22 @@ function loadTypeScript(sourcePath, mocks) { } const writes = []; +const socketSourcePath = path.join(__dirname, '../src/protocol/audioV2Socket.ts'); +const { createClientEventIdValue } = loadTypeScript(socketSourcePath, { + '@bufbuild/protobuf': { create: (_schema, value) => value }, + '@bufbuild/protobuf/wkt': {}, + './audioV2': { EventIdSchema: {}, ProcessingProfile: { SOURCE_NATIVE: 2 } }, +}); +const fallbackEventId = createClientEventIdValue(null); +const nextFallbackEventId = createClientEventIdValue(null); +assert.match(fallbackEventId, /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/); +assert.notEqual(fallbackEventId, nextFallbackEventId, 'fallback event IDs must remain unique'); +assert.equal( + createClientEventIdValue({ randomUUID: () => 'native-random-uuid' }), + 'native-random-uuid', + 'native randomUUID should be used when the runtime provides it', +); + const sourcePath = path.join(__dirname, '../src/services/phoneAudioDiagnostics.ts'); const { PhoneAudioDiagnostics } = loadTypeScript(sourcePath, { '@/utils/logger': { @@ -150,6 +166,11 @@ assert.match( /scheduleCaptureWatchdog\(\)/, 'a running iOS engine must recover if its input tap never delivers a frame', ); +assert.match( + integrationSources.ios, + /DuplexSystemChangePolicy\.shouldHoldEngine/, + 'iOS must survive the initial route-settlement notification that precedes mic frames', +); assert.match( integrationSources.ios, /ChroniclePcm16Packetizer/, diff --git a/app/src/protocol/audioV2Socket.ts b/app/src/protocol/audioV2Socket.ts index 31a418b9f..b274f42ef 100644 --- a/app/src/protocol/audioV2Socket.ts +++ b/app/src/protocol/audioV2Socket.ts @@ -86,8 +86,32 @@ export interface AudioV2SocketOptions { type AwaitedEvent = 'hello' | 'captureStarted' | 'captureStopped'; +interface RuntimeCrypto { + randomUUID?: () => string; +} + +let fallbackEventIdSequence = 0; + +function fallbackEventId(): string { + fallbackEventIdSequence = (fallbackEventIdSequence + 1) % 0x1_0000; + const random = Array.from( + { length: 19 }, + () => Math.floor(Math.random() * 16).toString(16), + ).join(''); + const sequence = fallbackEventIdSequence.toString(16).padStart(4, '0'); + const timestamp = Date.now().toString(16).padStart(12, '0').slice(-12); + const variant = (8 + Math.floor(Math.random() * 4)).toString(16); + return `${random.slice(0, 8)}-${sequence}-4${random.slice(8, 11)}-${variant}${random.slice(11, 14)}-${timestamp}`; +} + +export function createClientEventIdValue( + runtimeCrypto: RuntimeCrypto | null | undefined = (globalThis as { crypto?: RuntimeCrypto }).crypto, +): string { + return runtimeCrypto?.randomUUID ? runtimeCrypto.randomUUID() : fallbackEventId(); +} + function eventId() { - return create(EventIdSchema, { value: crypto.randomUUID() }); + return create(EventIdSchema, { value: createClientEventIdValue() }); } function uplinkSpec() { From 8ffe60e0ed01fb5cd592839775fcbcdf0ac21232 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:09:31 +0000 Subject: [PATCH 07/12] fix(app): recover queued phone audio before live capture --- app/package.json | 2 +- app/scripts/test-phone-audio-recovery.cjs | 180 ++++++++++++++++++++++ app/src/hooks/useAudioStreamer.ts | 11 +- 3 files changed, 186 insertions(+), 7 deletions(-) create mode 100644 app/scripts/test-phone-audio-recovery.cjs diff --git a/app/package.json b/app/package.json index 9a8a45a74..60e425424 100644 --- a/app/package.json +++ b/app/package.json @@ -12,7 +12,7 @@ "typecheck": "tsc --noEmit", "test:wearable-activation": "node scripts/test-wearable-activation.cjs", "test:durable-audio-spool": "node scripts/test-durable-audio-spool.cjs", - "test:phone-audio-diagnostics": "node scripts/test-phone-audio-diagnostics.cjs && node scripts/test-phone-audio-self-test.cjs && node scripts/test-client-diagnostic-metadata.cjs", + "test:phone-audio-diagnostics": "node scripts/test-phone-audio-diagnostics.cjs && node scripts/test-phone-audio-recovery.cjs && node scripts/test-phone-audio-self-test.cjs && node scripts/test-client-diagnostic-metadata.cjs", "test:push-notifications": "node scripts/test-push-notifications.cjs", "check:theme": "node scripts/check-theme-tokens.mjs" }, diff --git a/app/scripts/test-phone-audio-recovery.cjs b/app/scripts/test-phone-audio-recovery.cjs new file mode 100644 index 000000000..96be7389c --- /dev/null +++ b/app/scripts/test-phone-audio-recovery.cjs @@ -0,0 +1,180 @@ +const assert = require('node:assert/strict'); +const Module = require('node:module'); +const path = require('node:path'); +const ts = require('typescript'); +const fs = require('node:fs'); + +function loadTypeScript(sourcePath, mocks) { + const source = fs.readFileSync(sourcePath, 'utf8'); + const compiled = ts.transpileModule(source, { + compilerOptions: { + esModuleInterop: true, + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2020, + strict: true, + }, + fileName: sourcePath, + }); + const loaded = new Module(sourcePath, module); + loaded.filename = sourcePath; + loaded.paths = Module._nodeModulePaths(path.dirname(sourcePath)); + const originalRequire = loaded.require.bind(loaded); + loaded.require = (request) => mocks[request] ?? originalRequire(request); + loaded._compile(compiled.outputText, sourcePath); + return loaded.exports; +} + +const ProcessingProfile = { + SOURCE_NATIVE: 1, + DUPLEX_AEC: 2, + DUPLEX_ISOLATED: 3, + HALF_DUPLEX: 4, +}; +const DeliveryClass = { LIVE: 1, RECOVERED: 2 }; +const beginCaptureCalls = []; +let pendingReads = 0; + +class MockAudioV2Socket { + constructor(options) { + this.options = options; + this.activeBinding = null; + } + + async connect() {} + + async beginCapture(options) { + beginCaptureCalls.push(options); + this.activeBinding = { + captureSessionId: { value: `capture-${beginCaptureCalls.length}` }, + voiceSessionId: { value: beginCaptureCalls.length === 2 ? 'voice-live' : '' }, + captureEpoch: options.captureEpoch, + }; + return this.activeBinding; + } + + sendPacket(packet) { + this.options.onPacketAccepted(packet.sequence); + } + + async stopCapture() { + this.activeBinding = null; + } + + voiceReady() {} + heartbeat() {} + close() {} + acknowledgePlayback() {} +} + +const noDiagnostics = new Proxy({}, { get: () => () => {} }); +const sourcePath = path.join(__dirname, '../src/hooks/useAudioStreamer.ts'); +const { useAudioStreamer } = loadTypeScript(sourcePath, { + '@bufbuild/protobuf': { create: (_schema, value) => value }, + '@react-native-community/netinfo': { addEventListener: () => () => {} }, + react: { + useCallback: (callback) => callback, + useEffect: () => {}, + useRef: (value) => ({ current: value }), + useState: (value) => [value, () => {}], + }, + 'react-native': { Platform: { OS: 'ios' } }, + 'react-native-base64': { encode: (value) => value }, + '../../modules/chronicle-duplex-audio': { + addPlaybackStateListener: () => ({ remove() {} }), + addRouteChangeListener: () => ({ remove() {} }), + cancelResponse: async () => {}, + scheduleResponse: async () => {}, + }, + '../protocol/audioV2': { + CaptureCapabilitiesSchema: {}, + DataPurpose: { NORMAL_CAPTURE: 1 }, + DeliveryClass, + DeviceKind: { IOS_PHONE: 1, ANDROID_PHONE: 2, OMI: 3 }, + DuplexMode: { FULL: 1, ISOLATED: 2, HALF: 3 }, + EffectStatusSchema: {}, + InputRoute: { BUILT_IN_MIC: 1, BLUETOOTH_HFP: 2, WIRED_MIC: 3, USB: 4, REMOTE: 5 }, + OutputRoute: { SPEAKERPHONE: 1, EARPIECE: 2, HEADPHONES: 3, BLUETOOTH_HFP: 4, USB: 5, REMOTE: 6 }, + PlaybackState: { STARTED: 1, DONE: 2, CANCELLED: 3, FAILED: 4 }, + ProcessingProfile, + }, + '../protocol/audioV2Socket': { AudioV2Socket: MockAudioV2Socket }, + '../services/auth': { refreshToken: async () => 'refreshed-token' }, + '../services/durableAudioSpool': { + durableAudioSpool: { + async pendingPackets() { + pendingReads += 1; + return pendingReads === 1 + ? [{ + fileName: 'old.spool', + segmentId: 'old', + sequence: 7, + capturedAtMs: 1_780_000_000_000, + payload: new Uint8Array([1, 2, 3]), + }] + : []; + }, + async acknowledge() {}, + close() {}, + append() { + throw new Error('not used by this test'); + }, + }, + }, + '../services/phoneAudioDiagnostics': { phoneAudioDiagnostics: noDiagnostics }, +}); + +(async () => { + const streamer = useAudioStreamer({ autoReconnectEnabled: false }); + const phoneVoice = { + captureEpoch: 1, + capabilities: { + mode: 'duplex_full', + input_route: 'built_in_mic', + output_route: 'speakerphone', + native_sample_rate: 48_000, + aec: { requested: true, available: true, enabled: true }, + noise_suppression: { requested: true, available: true, enabled: true }, + }, + restartCapture: async () => phoneVoice, + stopCapture: async () => {}, + }; + + await streamer.startStreaming( + 'https://chronicle.invalid/ws/audio?token=test-token&device_name=phone-mic', + { phoneVoice }, + ); + + assert.equal(beginCaptureCalls.length, 2, 'queued audio must recover before live capture starts'); + assert.deepEqual( + { + captureEpoch: beginCaptureCalls[0].captureEpoch, + processingProfile: beginCaptureCalls[0].processingProfile, + deliveryClass: beginCaptureCalls[0].deliveryClass, + }, + { + captureEpoch: 0, + processingProfile: ProcessingProfile.SOURCE_NATIVE, + deliveryClass: DeliveryClass.RECOVERED, + }, + 'recovered source-native audio must use epoch zero', + ); + assert.deepEqual( + { + captureEpoch: beginCaptureCalls[1].captureEpoch, + processingProfile: beginCaptureCalls[1].processingProfile, + deliveryClass: beginCaptureCalls[1].deliveryClass, + }, + { + captureEpoch: 1, + processingProfile: ProcessingProfile.DUPLEX_AEC, + deliveryClass: DeliveryClass.LIVE, + }, + 'the following live duplex capture must retain the native phone epoch', + ); + + await streamer.stopStreaming(); + console.log('phone audio recovery tests passed'); +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/app/src/hooks/useAudioStreamer.ts b/app/src/hooks/useAudioStreamer.ts index 4d174ebd7..12bc2e23e 100644 --- a/app/src/hooks/useAudioStreamer.ts +++ b/app/src/hooks/useAudioStreamer.ts @@ -188,16 +188,15 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr return true; }, []); - const drainRecovery = useCallback(async ( - socket: AudioV2Socket, - captureEpoch: number, - ) => { + const drainRecovery = useCallback(async (socket: AudioV2Socket) => { let recoverySequence = 0; while (true) { const packets = await durableAudioSpool.pendingPackets(); if (!packets.length) return; await socket.beginCapture({ - captureEpoch, + // Recovery is a source-native capture, whose protocol epoch is always zero. + // The native phone epoch belongs only to the subsequent live voice session. + captureEpoch: 0, processingProfile: ProcessingProfile.SOURCE_NATIVE, dataPurpose: DataPurpose.NORMAL_CAPTURE, deliveryClass: DeliveryClass.RECOVERED, @@ -347,7 +346,7 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr await socket.connect(); if (phoneVoice) phoneAudioDiagnostics.socketOpen(); deliveryModeRef.current = 'recovering'; - await drainRecovery(socket, phoneVoice?.captureEpoch ?? 0); + await drainRecovery(socket); liveStartedAtRef.current = Date.now(); liveSequenceRef.current = 0; const capabilities = phoneVoice From 75fe684740a714e5ad4b3ab188e26121fcdeb587 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:28:14 +0000 Subject: [PATCH 08/12] fix(app): refresh phone audio authentication --- app/scripts/test-phone-audio-diagnostics.cjs | 68 ++++++++++++++++++- app/scripts/test-phone-audio-recovery.cjs | 17 ++++- app/src/hooks/useAudioStreamer.ts | 13 ++-- .../hooks/useAudioStreamingOrchestrator.ts | 8 ++- app/src/services/phoneAudioDiagnostics.ts | 10 +-- 5 files changed, 98 insertions(+), 18 deletions(-) diff --git a/app/scripts/test-phone-audio-diagnostics.cjs b/app/scripts/test-phone-audio-diagnostics.cjs index e788665ee..63b51d3f1 100644 --- a/app/scripts/test-phone-audio-diagnostics.cjs +++ b/app/scripts/test-phone-audio-diagnostics.cjs @@ -110,7 +110,7 @@ assert.match(text, /native_tap_received.*capture_epoch=1/); assert.match(text, /native_pcm_converted.*frames=320/); assert.match(text, /native_opus_encoded.*bytes=42/); assert.match(text, /audio_level_active.*audio_level=0\.500/); -assert.match(text, /frame_dropped_socket_not_open.*ready_state=0/); +assert.match(text, /frame_buffered_socket_not_open.*ready_state=0/); assert.match(text, /websocket_transport_open/); assert.match(text, /websocket_client_hello_sent/); assert.match(text, /websocket_transport_error.*token=/); @@ -118,7 +118,7 @@ assert.match(text, /first_frame_enqueued.*opus_bytes=44/); assert.match(text, /first_packet_accepted.*sequence=0/); assert.match( text, - /meter_stalled.*native_frames=2.*socket_drops=2.*enqueued_frames=1.*acked_packets=1.*last_audio_level=0\.500/, + /meter_stalled.*native_frames=2.*buffered_while_disconnected=2.*enqueued_frames=1.*acked_packets=1.*last_audio_level=0\.500/, ); assert.doesNotMatch(text, /capture-secret-id/, 'server-issued identifiers must be abbreviated'); assert.doesNotMatch(text, /secret-value/, 'credentials must be redacted from exported diagnostics'); @@ -178,4 +178,66 @@ assert.match( ); assert.match(integrationSources.android, /"audioLevel" to DuplexAudioPolicy\.audioLevel/, 'Android must emit PCM audio levels'); -console.log('phone audio diagnostics tests passed'); +const orchestratorPath = path.join(__dirname, '../src/hooks/useAudioStreamingOrchestrator.ts'); +const noDiagnostics = new Proxy({}, { get: () => () => {} }); +global.WebSocket = { OPEN: 1 }; +const { useAudioStreamingOrchestrator } = loadTypeScript(orchestratorPath, { + react: { + useCallback: (callback) => callback, + useState: (value) => [value, () => {}], + }, + 'react-native': { Alert: { alert: () => {} } }, + 'friend-lite-react-native': {}, + '../services/phoneAudioDiagnostics': { phoneAudioDiagnostics: noDiagnostics }, +}); + +(async () => { + const queuedFrames = []; + const frame = { captureEpoch: 1, capturedAtMs: 1_780_000_000_000, opus: new Uint8Array([1, 2, 3]) }; + const orchestrator = useAudioStreamingOrchestrator({ + omiConnection: { isConnected: () => false }, + deviceConnection: { connectedDeviceId: null }, + audioStreamer: { + isStreaming: false, + startStreaming: async () => {}, + stopStreaming: async () => {}, + sendDurableAudio: () => {}, + sendInteractiveFrame: (value) => queuedFrames.push(value), + getWebSocketReadyState: () => 0, + }, + phoneAudioRecorder: { + isRecording: false, + startRecording: async (onData) => { + await onData(frame); + return { + captureEpoch: 1, + capabilities: { + mode: 'duplex_full', + input_route: 'built_in_mic', + output_route: 'speakerphone', + native_sample_rate: 48_000, + aec: { requested: true, available: true, enabled: true }, + noise_suppression: { requested: true, available: true, enabled: true }, + }, + restartCapture: async () => {}, + stopCapture: async () => {}, + }; + }, + stopRecording: async () => {}, + }, + originalStartAudioListener: async () => {}, + originalStopAudioListener: async () => {}, + settings: { + webSocketUrl: 'https://chronicle.invalid', + jwtToken: 'token', + isAuthenticated: true, + }, + }); + + await orchestrator.handleTogglePhoneAudio(); + assert.deepEqual(queuedFrames, [frame], 'phone frames must enter the durable spool while the socket reconnects'); + console.log('phone audio diagnostics tests passed'); +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/app/scripts/test-phone-audio-recovery.cjs b/app/scripts/test-phone-audio-recovery.cjs index 96be7389c..0a8e7a46b 100644 --- a/app/scripts/test-phone-audio-recovery.cjs +++ b/app/scripts/test-phone-audio-recovery.cjs @@ -32,12 +32,14 @@ const ProcessingProfile = { }; const DeliveryClass = { LIVE: 1, RECOVERED: 2 }; const beginCaptureCalls = []; +const socketBearerTokens = []; let pendingReads = 0; class MockAudioV2Socket { constructor(options) { this.options = options; this.activeBinding = null; + socketBearerTokens.push(options.bearerToken); } async connect() {} @@ -98,7 +100,10 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { ProcessingProfile, }, '../protocol/audioV2Socket': { AudioV2Socket: MockAudioV2Socket }, - '../services/auth': { refreshToken: async () => 'refreshed-token' }, + '../services/auth': { + getValidToken: async () => 'fresh-token', + isTokenExpired: (token) => token === 'expired-token', + }, '../services/durableAudioSpool': { durableAudioSpool: { async pendingPackets() { @@ -124,7 +129,11 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { }); (async () => { - const streamer = useAudioStreamer({ autoReconnectEnabled: false }); + const refreshedTokens = []; + const streamer = useAudioStreamer({ + autoReconnectEnabled: false, + onTokenRefreshed: (token) => refreshedTokens.push(token), + }); const phoneVoice = { captureEpoch: 1, capabilities: { @@ -140,10 +149,12 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { }; await streamer.startStreaming( - 'https://chronicle.invalid/ws/audio?token=test-token&device_name=phone-mic', + 'https://chronicle.invalid/ws/audio?token=expired-token&device_name=phone-mic', { phoneVoice }, ); + assert.deepEqual(socketBearerTokens, ['fresh-token'], 'audio must replace an expired URL token before opening the socket'); + assert.deepEqual(refreshedTokens, ['fresh-token'], 'the refreshed token must propagate to app settings'); assert.equal(beginCaptureCalls.length, 2, 'queued audio must recover before live capture starts'); assert.deepEqual( { diff --git a/app/src/hooks/useAudioStreamer.ts b/app/src/hooks/useAudioStreamer.ts index 12bc2e23e..30318c180 100644 --- a/app/src/hooks/useAudioStreamer.ts +++ b/app/src/hooks/useAudioStreamer.ts @@ -27,7 +27,7 @@ import { import { AudioV2Socket } from '../protocol/audioV2Socket'; import type { CapturedOpusFrame } from '../protocol/capturedOpusFrame'; import type { VoiceCapabilities } from '../protocol/audioCapabilities'; -import { refreshToken } from '../services/auth'; +import { getValidToken, isTokenExpired } from '../services/auth'; import { durableAudioSpool, type SpoolPacket } from '../services/durableAudioSpool'; import { phoneAudioDiagnostics } from '../services/phoneAudioDiagnostics'; @@ -257,9 +257,14 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr setError(null); const phoneVoice = configRef.current?.phoneVoice; let parsed = socketOptions(url, Boolean(phoneVoice)); - if (!parsed.bearerToken) { - const token = await refreshToken(); - if (!token) throw new Error('Audio authentication expired'); + const managedToken = await getValidToken(); + const token = managedToken ?? ( + parsed.bearerToken && !isTokenExpired(parsed.bearerToken) + ? parsed.bearerToken + : null + ); + if (!token) throw new Error('Audio authentication expired'); + if (token !== parsed.bearerToken) { optionsRef.current?.onTokenRefreshed?.(token); const refreshed = new URL(url); refreshed.searchParams.set('token', token); diff --git a/app/src/hooks/useAudioStreamingOrchestrator.ts b/app/src/hooks/useAudioStreamingOrchestrator.ts index 7a0e61faf..695d9f69c 100644 --- a/app/src/hooks/useAudioStreamingOrchestrator.ts +++ b/app/src/hooks/useAudioStreamingOrchestrator.ts @@ -133,12 +133,14 @@ export const useAudioStreamingOrchestrator = ({ try { const finalUrl = buildPhoneWebSocketUrl(settings.webSocketUrl); const capture = await phoneAudioRecorder.startRecording(async (frame) => { + if (frame.opus.length === 0) return; const wsReady = audioStreamer.getWebSocketReadyState(); - if (wsReady === WebSocket.OPEN && frame.opus.length > 0) { - audioStreamer.sendInteractiveFrame(frame); - } else { + if (wsReady !== WebSocket.OPEN) { phoneAudioDiagnostics.socketUnavailable(wsReady); } + // Native capture owns durability. Queue every frame first; the streamer + // sends immediately while live and replays the spool after reconnecting. + audioStreamer.sendInteractiveFrame(frame); }); await audioStreamer.startStreaming(finalUrl, { phoneVoice: capture }); setIsPhoneAudioMode(true); diff --git a/app/src/services/phoneAudioDiagnostics.ts b/app/src/services/phoneAudioDiagnostics.ts index dff0f2769..9ae0b727e 100644 --- a/app/src/services/phoneAudioDiagnostics.ts +++ b/app/src/services/phoneAudioDiagnostics.ts @@ -28,7 +28,7 @@ export class PhoneAudioDiagnostics { private startedAtMs = 0; private milestones = new Set(); private nativeFrames = 0; - private socketDrops = 0; + private bufferedWhileDisconnected = 0; private enqueuedFrames = 0; private ackedPackets = 0; private lastAudioLevel = 0; @@ -54,7 +54,7 @@ export class PhoneAudioDiagnostics { this.startedAtMs = this.now(); this.milestones.clear(); this.nativeFrames = 0; - this.socketDrops = 0; + this.bufferedWhileDisconnected = 0; this.enqueuedFrames = 0; this.ackedPackets = 0; this.lastAudioLevel = 0; @@ -115,10 +115,10 @@ export class PhoneAudioDiagnostics { socketUnavailable(readyState: number | undefined): void { if (!this.active) return; - this.socketDrops += 1; + this.bufferedWhileDisconnected += 1; this.once( 'warn', - 'frame_dropped_socket_not_open', + 'frame_buffered_socket_not_open', `ready_state=${readyState ?? 'undefined'}`, ); } @@ -182,7 +182,7 @@ export class PhoneAudioDiagnostics { return [ `elapsed_ms=${Math.max(0, this.now() - this.startedAtMs)}`, `native_frames=${this.nativeFrames}`, - `socket_drops=${this.socketDrops}`, + `buffered_while_disconnected=${this.bufferedWhileDisconnected}`, `enqueued_frames=${this.enqueuedFrames}`, `acked_packets=${this.ackedPackets}`, `last_audio_level=${this.lastAudioLevel.toFixed(3)}`, From 55e04f0d826eaf1148538c8e0156354afb4088e2 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:37:05 +0000 Subject: [PATCH 09/12] fix(audio): simplify capture source contract --- app/app/index.tsx | 33 +-- app/scripts/test-phone-audio-diagnostics.cjs | 181 ++++++++++++++-- app/scripts/test-phone-audio-recovery.cjs | 30 ++- app/src/hooks/useAppSettings.ts | 3 + app/src/hooks/useAudioStreamer.ts | 200 ++++++------------ .../hooks/useAudioStreamingOrchestrator.ts | 114 ++++------ app/src/protocol/audioV2Socket.ts | 9 +- app/src/services/auth.ts | 13 +- app/src/services/durableAudioSpool.ts | 35 ++- app/src/services/phoneAudioSelfTest.ts | 1 + .../audio_contract/v2/codec.py | 48 ++++- .../controllers/audio_v2_controller.py | 112 ++++++---- .../controllers/capture_lifecycle.py | 31 ++- .../services/audio_stream/session_store.py | 1 + .../workers/audio_jobs.py | 10 +- .../advanced/tests/test_audio_durability.py | 1 + .../tests/test_audio_persistence_lifecycle.py | 52 +++++ .../advanced/tests/test_audio_protocol_v2.py | 26 ++- .../advanced/tests/test_audio_v2_ingress.py | 120 +++++++++-- docs/backend/audio-interface-map.md | 23 +- .../chronicle_client/audio_v2.py | 30 +-- .../chronicle-client/tests/test_audio_v2.py | 21 +- extras/havpe-relay/relay_core.py | 1 + .../chronicle_wearable/backend.py | 1 + tests/libs/audio_stream_library.py | 2 + 25 files changed, 711 insertions(+), 387 deletions(-) diff --git a/app/app/index.tsx b/app/app/index.tsx index 6c57a7699..d5c0d6f69 100644 --- a/app/app/index.tsx +++ b/app/app/index.tsx @@ -38,7 +38,7 @@ export default function App() { // Bluetooth const { bleManager, bluetoothState, permissionGranted, requestBluetoothPermission, isPermissionsLoading } = useBluetoothManager(); - // Settings (must be before audioStreamer so the token refresh callback can reference it) + // Settings const settings = useSharedAppSettings(); // Live backend reachability (Connection Doctor), re-probed on pull-to-refresh. @@ -46,26 +46,16 @@ export default function App() { const [refreshing, setRefreshing] = useState(false); // Audio - const audioStreamer = useAudioStreamer({ - autoReconnectEnabled: settings.autoReconnectEnabled, - onTokenRefreshed: (newToken) => { - // Update app-level auth state when auto-re-login refreshes the token - if (settings.currentUserEmail) { - settings.handleAuthStatusChange(true, settings.currentUserEmail, newToken); - } - }, - }); + const audioStreamer = useAudioStreamer(); const phoneAudioRecorder = usePhoneAudioRecorder(); const { isListeningAudio: isOmiAudioListenerActive, audioPacketsReceived, startAudioListener: originalStartAudioListener, stopAudioListener: originalStopAudioListener, isRetrying: isAudioListenerRetrying, retryAttempts: audioListenerRetryAttempts } = useAudioListener(omiConnection, () => !!deviceConnection.connectedDeviceId); // Refs for disconnect cleanup const isOmiAudioListenerActiveRef = useRef(isOmiAudioListenerActive); - const isAudioStreamingRef = useRef(audioStreamer.isStreaming); // Track if audio pipeline was active before BLE disconnect (for auto-restart on reconnect) const wasStreamingBeforeDisconnectRef = useRef(false); useEffect(() => { isOmiAudioListenerActiveRef.current = isOmiAudioListenerActive; }, [isOmiAudioListenerActive]); - useEffect(() => { isAudioStreamingRef.current = audioStreamer.isStreaming; }, [audioStreamer.isStreaming]); // Refs to break the declaration-order cycle: // onDeviceConnect/onDeviceDisconnect need orchestrator + autoReconnect, @@ -99,22 +89,13 @@ export default function App() { }, [omiConnection]); const onDeviceDisconnect = useCallback(async () => { - // Remember if audio was active so we can auto-restart on reconnect - if (isOmiAudioListenerActiveRef.current || isAudioStreamingRef.current) { + // BLE disconnect only owns the wearable pipeline. Phone capture is independent. + if (isOmiAudioListenerActiveRef.current) { wasStreamingBeforeDisconnectRef.current = true; + await originalStopAudioListener(); + await audioStreamer.stopStreaming(); } - - // Stop audio listener (BLE is gone, can't read audio) - if (isOmiAudioListenerActiveRef.current) await originalStopAudioListener(); - - // Keep WebSocket alive — it will reconnect or idle until BLE comes back. - // Only stop WebSocket for phone audio mode (no BLE needed there). - if (phoneAudioRecorder.isRecording) { - audioStreamer.stopStreaming(); - await phoneAudioRecorder.stopRecording(); - orchestratorRef.current?.setIsPhoneAudioMode(false); - } - }, [originalStopAudioListener, audioStreamer.stopStreaming, phoneAudioRecorder.stopRecording, phoneAudioRecorder.isRecording]); + }, [originalStopAudioListener, audioStreamer.stopStreaming]); const deviceConnection = useDeviceConnection(omiConnection, bleManager, onDeviceDisconnect, onDeviceConnect); diff --git a/app/scripts/test-phone-audio-diagnostics.cjs b/app/scripts/test-phone-audio-diagnostics.cjs index 63b51d3f1..18d2d281f 100644 --- a/app/scripts/test-phone-audio-diagnostics.cjs +++ b/app/scripts/test-phone-audio-diagnostics.cjs @@ -26,10 +26,50 @@ function loadTypeScript(sourcePath, mocks) { const writes = []; const socketSourcePath = path.join(__dirname, '../src/protocol/audioV2Socket.ts'); -const { createClientEventIdValue } = loadTypeScript(socketSourcePath, { +const serverControls = { + hello: { event: { case: 'hello', value: {} } }, + captureStarted: { + event: { + case: 'captureStarted', + value: { + binding: { + captureSessionId: { value: 'capture-1' }, + voiceSessionId: { value: '' }, + captureEpoch: 0, + }, + }, + }, + }, +}; +const { AudioV2Socket, createClientEventIdValue } = loadTypeScript(socketSourcePath, { '@bufbuild/protobuf': { create: (_schema, value) => value }, '@bufbuild/protobuf/wkt': {}, - './audioV2': { EventIdSchema: {}, ProcessingProfile: { SOURCE_NATIVE: 2 } }, + './audioV2': { + AudioCodec: { OPUS: 1 }, + AudioSpecSchema: {}, + CaptureBindingSchema: {}, + CaptureMediaPacketSchema: {}, + CaptureSourceIdSchema: {}, + ClientHelloSchema: {}, + ClientControlSchema: {}, + DataPurpose: { NORMAL_CAPTURE: 1 }, + DeliveryClass: { UNSPECIFIED: 0, LIVE: 1 }, + EventIdSchema: {}, + HeartbeatSchema: {}, + MediaEnvelopeSchema: {}, + PlaybackAcknowledgementSchema: {}, + ProcessingProfile: { SOURCE_NATIVE: 2 }, + ResponseIdSchema: {}, + StartCaptureSchema: {}, + StopCaptureSchema: {}, + StopReason: { USER_REQUESTED: 1 }, + VoiceReadySchema: {}, + decodeMediaEnvelope: value => value, + decodeServerControl: value => serverControls[value], + encodeClientControl: value => value, + encodeMediaEnvelope: value => value, + timestampFromUnixMs: value => value, + }, }); const fallbackEventId = createClientEventIdValue(null); const nextFallbackEventId = createClientEventIdValue(null); @@ -41,6 +81,57 @@ assert.equal( 'native randomUUID should be used when the runtime provides it', ); +global.WebSocket = { OPEN: 1 }; + +class FakeWebSocket { + constructor() { + this.readyState = 0; + this.sent = []; + } + + open() { + this.readyState = WebSocket.OPEN; + this.onopen(); + } + + receive(value) { + this.onmessage({ data: value }); + } + + send(value) { + this.sent.push(value); + } + + close() {} +} + +async function declaredUplinkDuration(frameDurationMs) { + const transport = new FakeWebSocket(); + const socket = new AudioV2Socket({ + url: 'wss://chronicle.invalid/ws/audio', + bearerToken: 'token', + sourceId: 'source', + displayName: 'source', + deviceKind: 4, + uplinkFrameDurationMs: frameDurationMs, + webSocketFactory: () => transport, + }); + const connecting = socket.connect(); + transport.open(); + transport.receive('hello'); + await connecting; + const starting = socket.beginCapture({ + captureEpoch: 0, + processingProfile: 2, + deliveryClass: 1, + }); + transport.receive('captureStarted'); + await starting; + return transport.sent.map(control => ( + control.event.value.audioSpec ?? control.event.value.supportedUplink?.[0] + )).filter(Boolean).map(spec => spec.frameDuration.nanos); +} + const sourcePath = path.join(__dirname, '../src/services/phoneAudioDiagnostics.ts'); const { PhoneAudioDiagnostics } = loadTypeScript(sourcePath, { '@/utils/logger': { @@ -133,11 +224,6 @@ const integrationSources = { assert.match(integrationSources.recorder, /setAudioLevel\(/, 'native levels must drive the UI meter'); assert.match(integrationSources.recorder, /native_frame_timeout/, 'a silent native engine must surface a diagnostic'); assert.match(integrationSources.streamer, /packetAccepted\(sequence\)/, 'backend acknowledgements must be logged'); -assert.match( - integrationSources.streamer, - /const optionsRef = useRef\(options\)/, - 'rerenders must not replace the WebSocket lifecycle callback through a fresh options object', -); assert.match( integrationSources.streamer, /onDiagnostic: event =>/, @@ -145,8 +231,8 @@ assert.match( ); assert.doesNotMatch( integrationSources.streamer, - /\}, \[drainRecovery, encodeBase64, options, packetAccepted\]\);/, - 'startStreaming must not close an active socket merely because its options object was recreated', + /autoReconnectEnabled|NetInfo/, + 'audio transport must expose failure instead of running a second reconnect state machine', ); assert.match(integrationSources.orchestrator, /beginAttempt\(\)/, 'the phone button must open a diagnostic attempt'); assert.match(integrationSources.ios, /"audioLevel": audioLevel/, 'iOS must emit PCM audio levels'); @@ -180,30 +266,30 @@ assert.match(integrationSources.android, /"audioLevel" to DuplexAudioPolicy\.aud const orchestratorPath = path.join(__dirname, '../src/hooks/useAudioStreamingOrchestrator.ts'); const noDiagnostics = new Proxy({}, { get: () => () => {} }); -global.WebSocket = { OPEN: 1 }; const { useAudioStreamingOrchestrator } = loadTypeScript(orchestratorPath, { react: { useCallback: (callback) => callback, useState: (value) => [value, () => {}], }, 'react-native': { Alert: { alert: () => {} } }, - 'friend-lite-react-native': {}, + 'friend-lite-react-native': { BleAudioCodec: { OPUS: 'opus' } }, '../services/phoneAudioDiagnostics': { phoneAudioDiagnostics: noDiagnostics }, }); (async () => { + assert.deepEqual(await declaredUplinkDuration(20), [20_000_000, 20_000_000]); + assert.deepEqual(await declaredUplinkDuration(60), [60_000_000, 60_000_000]); const queuedFrames = []; + const starts = []; const frame = { captureEpoch: 1, capturedAtMs: 1_780_000_000_000, opus: new Uint8Array([1, 2, 3]) }; const orchestrator = useAudioStreamingOrchestrator({ omiConnection: { isConnected: () => false }, deviceConnection: { connectedDeviceId: null }, audioStreamer: { isStreaming: false, - startStreaming: async () => {}, + startStreaming: async (url, source) => starts.push({ url, source }), stopStreaming: async () => {}, - sendDurableAudio: () => {}, - sendInteractiveFrame: (value) => queuedFrames.push(value), - getWebSocketReadyState: () => 0, + sendFrame: (source, value) => queuedFrames.push({ source, value }), }, phoneAudioRecorder: { isRecording: false, @@ -235,7 +321,70 @@ const { useAudioStreamingOrchestrator } = loadTypeScript(orchestratorPath, { }); await orchestrator.handleTogglePhoneAudio(); - assert.deepEqual(queuedFrames, [frame], 'phone frames must enter the durable spool while the socket reconnects'); + assert.deepEqual( + queuedFrames, + [{ source: 'phone', value: frame }], + 'phone frames must enter the one source-tagged queue', + ); + assert.equal(starts[0].url, 'wss://chronicle.invalid/ws/audio'); + assert.equal(starts[0].source.kind, 'phone'); + assert.equal(new URL(starts[0].url).search, '', 'audio credentials must never enter the URL'); + + let listenerStarts = 0; + let wearableSocketStarts = 0; + const nonOpus = useAudioStreamingOrchestrator({ + omiConnection: { + isConnected: () => true, + getAudioCodec: async () => 'pcm8', + }, + deviceConnection: { connectedDeviceId: 'neo-1' }, + audioStreamer: { + isStreaming: false, + startStreaming: async () => { wearableSocketStarts += 1; }, + stopStreaming: async () => {}, + sendFrame: () => {}, + }, + phoneAudioRecorder: { + isRecording: false, + startRecording: async () => { throw new Error('not used'); }, + stopRecording: async () => {}, + }, + originalStartAudioListener: async () => { listenerStarts += 1; }, + originalStopAudioListener: async () => {}, + settings: { webSocketUrl: 'https://chronicle.invalid' }, + }); + await nonOpus.handleStartAudioListeningAndStreaming(); + assert.equal(listenerStarts, 0, 'a non-Opus wearable must fail before capture starts'); + assert.equal(wearableSocketStarts, 0, 'a non-Opus wearable must not open Audio V2'); + + const wearableFrames = []; + const wearableStarts = []; + const opusWearable = useAudioStreamingOrchestrator({ + omiConnection: { + isConnected: () => true, + getAudioCodec: async () => 'opus', + }, + deviceConnection: { connectedDeviceId: 'neo-1' }, + audioStreamer: { + isStreaming: false, + startStreaming: async (url, source) => wearableStarts.push({ url, source }), + stopStreaming: async () => {}, + sendFrame: (source, value) => wearableFrames.push({ source, value }), + }, + phoneAudioRecorder: { + isRecording: false, + startRecording: async () => { throw new Error('not used'); }, + stopRecording: async () => {}, + }, + originalStartAudioListener: async onData => onData(new Uint8Array([4, 5, 6])), + originalStopAudioListener: async () => {}, + settings: { webSocketUrl: 'https://chronicle.invalid' }, + }); + await opusWearable.handleStartAudioListeningAndStreaming(); + assert.equal(wearableStarts[0].source.kind, 'wearable'); + assert.equal(wearableStarts[0].source.sourceId, 'neo-1'); + assert.equal(wearableFrames[0].source, 'wearable'); + assert.equal(wearableFrames[0].value.frameDurationMs, 60); console.log('phone audio diagnostics tests passed'); })().catch((error) => { console.error(error); diff --git a/app/scripts/test-phone-audio-recovery.cjs b/app/scripts/test-phone-audio-recovery.cjs index 0a8e7a46b..ffd8d701e 100644 --- a/app/scripts/test-phone-audio-recovery.cjs +++ b/app/scripts/test-phone-audio-recovery.cjs @@ -33,6 +33,8 @@ const ProcessingProfile = { const DeliveryClass = { LIVE: 1, RECOVERED: 2 }; const beginCaptureCalls = []; const socketBearerTokens = []; +const socketFrameDurations = []; +const pendingSources = []; let pendingReads = 0; class MockAudioV2Socket { @@ -40,6 +42,7 @@ class MockAudioV2Socket { this.options = options; this.activeBinding = null; socketBearerTokens.push(options.bearerToken); + socketFrameDurations.push(options.uplinkFrameDurationMs); } async connect() {} @@ -102,11 +105,11 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { '../protocol/audioV2Socket': { AudioV2Socket: MockAudioV2Socket }, '../services/auth': { getValidToken: async () => 'fresh-token', - isTokenExpired: (token) => token === 'expired-token', }, '../services/durableAudioSpool': { durableAudioSpool: { - async pendingPackets() { + async pendingPackets(source) { + pendingSources.push(source); pendingReads += 1; return pendingReads === 1 ? [{ @@ -129,11 +132,7 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { }); (async () => { - const refreshedTokens = []; - const streamer = useAudioStreamer({ - autoReconnectEnabled: false, - onTokenRefreshed: (token) => refreshedTokens.push(token), - }); + const streamer = useAudioStreamer(); const phoneVoice = { captureEpoch: 1, capabilities: { @@ -149,12 +148,13 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { }; await streamer.startStreaming( - 'https://chronicle.invalid/ws/audio?token=expired-token&device_name=phone-mic', - { phoneVoice }, + 'wss://chronicle.invalid/ws/audio', + { kind: 'phone', ...phoneVoice }, ); - assert.deepEqual(socketBearerTokens, ['fresh-token'], 'audio must replace an expired URL token before opening the socket'); - assert.deepEqual(refreshedTokens, ['fresh-token'], 'the refreshed token must propagate to app settings'); + assert.deepEqual(socketBearerTokens, ['fresh-token'], 'audio must use the managed token source'); + assert.deepEqual(socketFrameDurations, [20], 'phone capture must declare 20 ms Opus'); + assert.deepEqual(pendingSources, ['phone', 'phone'], 'recovery must read only the active source queue'); assert.equal(beginCaptureCalls.length, 2, 'queued audio must recover before live capture starts'); assert.deepEqual( { @@ -184,6 +184,14 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { ); await streamer.stopStreaming(); + + const wearableStreamer = useAudioStreamer(); + await wearableStreamer.startStreaming( + 'wss://chronicle.invalid/ws/audio', + { kind: 'wearable', sourceId: 'neo-1' }, + ); + assert.deepEqual(socketFrameDurations, [20, 60], 'wearable capture must declare 60 ms Opus'); + await wearableStreamer.stopStreaming(); console.log('phone audio recovery tests passed'); })().catch((error) => { console.error(error); diff --git a/app/src/hooks/useAppSettings.ts b/app/src/hooks/useAppSettings.ts index aed7379d2..8e6c311b5 100644 --- a/app/src/hooks/useAppSettings.ts +++ b/app/src/hooks/useAppSettings.ts @@ -11,6 +11,7 @@ import { } from '../utils/storage'; import { recoverBackendUrl } from '../services/serviceManager'; import { httpUrlToWebSocketUrl } from '../utils/urlConversion'; +import { onTokenRefreshed } from '../services/auth'; const DEFAULT_WS_URL = 'ws://localhost:8000/ws/audio'; @@ -35,6 +36,8 @@ export const useAppSettings = (): AppSettings => { const [jwtToken, setJwtToken] = useState(null); const [autoReconnectEnabled, setAutoReconnectEnabled] = useState(true); + useEffect(() => onTokenRefreshed(setJwtToken), []); + useEffect(() => { const loadSettings = async () => { const storedWsUrl = await getWebSocketUrl(); diff --git a/app/src/hooks/useAudioStreamer.ts b/app/src/hooks/useAudioStreamer.ts index 30318c180..b3eebfdc4 100644 --- a/app/src/hooks/useAudioStreamer.ts +++ b/app/src/hooks/useAudioStreamer.ts @@ -1,6 +1,5 @@ import { create } from '@bufbuild/protobuf'; -import NetInfo from '@react-native-community/netinfo'; -import { useCallback, useEffect, useRef, useState } from 'react'; +import { useCallback, useRef, useState } from 'react'; import { Platform } from 'react-native'; // @ts-ignore - no type declarations available import base64 from 'react-native-base64'; @@ -27,58 +26,52 @@ import { import { AudioV2Socket } from '../protocol/audioV2Socket'; import type { CapturedOpusFrame } from '../protocol/capturedOpusFrame'; import type { VoiceCapabilities } from '../protocol/audioCapabilities'; -import { getValidToken, isTokenExpired } from '../services/auth'; -import { durableAudioSpool, type SpoolPacket } from '../services/durableAudioSpool'; +import type { PhoneCaptureSession } from './usePhoneAudioRecorder'; +import { getValidToken } from '../services/auth'; +import { + durableAudioSpool, + type AudioSpoolSource, + type SpoolPacket, +} from '../services/durableAudioSpool'; import { phoneAudioDiagnostics } from '../services/phoneAudioDiagnostics'; -interface UseAudioStreamerOptions { - onTokenRefreshed?: (token: string) => void; - autoReconnectEnabled?: boolean; -} - -export interface StreamStartConfig { - phoneVoice?: { - captureEpoch: number; - capabilities: VoiceCapabilities; - restartCapture: () => Promise>; - stopCapture: () => Promise; - }; -} +export type AudioStreamSource = + | { + kind: 'wearable'; + sourceId: string; + } + | ({ kind: 'phone' } & PhoneCaptureSession); interface UseAudioStreamer { isStreaming: boolean; isConnecting: boolean; error: string | null; phonePlaybackState: 'started' | 'done' | 'cancelled' | 'failed' | null; - startStreaming: (url: string, config?: StreamStartConfig) => Promise; - getWebSocketReadyState: () => number | undefined; + startStreaming: (url: string, source: AudioStreamSource) => Promise; stopStreaming: () => Promise; - sendDurableAudio: (audioBytes: Uint8Array) => void; - sendInteractiveFrame: (frame: CapturedOpusFrame) => void; + sendFrame: (source: AudioSpoolSource, frame: CapturedOpusFrame) => void; } const HEARTBEAT_MS = 25_000; -const RECONNECT_BASE_MS = 3_000; -const RECONNECT_MAX_MS = 30_000; function recoveryBatchId(): string { return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 12)}`; } -function socketOptions(urlText: string, phone: boolean) { - const url = new URL(urlText); - const bearerToken = url.searchParams.get('token') ?? ''; - const displayName = url.searchParams.get('device_name') ?? (phone ? 'phone-mic' : 'omi'); - url.search = ''; - url.pathname = '/ws/audio'; +function socketSource(source: AudioStreamSource) { + if (source.kind === 'wearable') { + return { + sourceId: source.sourceId, + displayName: 'wearable', + deviceKind: DeviceKind.OMI, + uplinkFrameDurationMs: 60 as const, + }; + } return { - url: url.toString(), - bearerToken, - sourceId: displayName, - displayName, - deviceKind: phone - ? (Platform.OS === 'ios' ? DeviceKind.IOS_PHONE : DeviceKind.ANDROID_PHONE) - : DeviceKind.OMI, + sourceId: 'phone-mic', + displayName: 'phone-mic', + deviceKind: Platform.OS === 'ios' ? DeviceKind.IOS_PHONE : DeviceKind.ANDROID_PHONE, + uplinkFrameDurationMs: 20 as const, }; } @@ -122,25 +115,20 @@ function processingProfile(capabilities: VoiceCapabilities): ProcessingProfile { return ProcessingProfile.HALF_DUPLEX; } -export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStreamer => { +export const useAudioStreamer = (): UseAudioStreamer => { const [isStreaming, setIsStreaming] = useState(false); const [isConnecting, setIsConnecting] = useState(false); const [error, setError] = useState(null); const [phonePlaybackState, setPhonePlaybackState] = useState(null); const socketRef = useRef(null); - const optionsRef = useRef(options); - optionsRef.current = options; - const configRef = useRef(undefined); + const sourceRef = useRef(null); const urlRef = useRef(''); const stoppedRef = useRef(false); - const reconnectRef = useRef | null>(null); const heartbeatRef = useRef | null>(null); - const reconnectAttemptRef = useRef(0); const liveSequenceRef = useRef(0); const liveStartedAtRef = useRef(0); const acceptedRef = useRef(new Map()); const acceptedWaitersRef = useRef(new Map void>()); - const connectingRef = useRef | null>(null); const deliveryModeRef = useRef<'idle' | 'recovering' | 'live'>('idle'); const playbackRef = useRef<{ responseId: string; @@ -188,10 +176,13 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr return true; }, []); - const drainRecovery = useCallback(async (socket: AudioV2Socket) => { + const drainRecovery = useCallback(async ( + socket: AudioV2Socket, + source: AudioSpoolSource, + ) => { let recoverySequence = 0; while (true) { - const packets = await durableAudioSpool.pendingPackets(); + const packets = await durableAudioSpool.pendingPackets(source); if (!packets.length) return; await socket.beginCapture({ // Recovery is a source-native capture, whose protocol epoch is always zero. @@ -223,9 +214,7 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr const stopStreaming = useCallback(async () => { stoppedRef.current = true; deliveryModeRef.current = 'idle'; - if (reconnectRef.current) clearTimeout(reconnectRef.current); if (heartbeatRef.current) clearInterval(heartbeatRef.current); - reconnectRef.current = null; heartbeatRef.current = null; try { await socketRef.current?.stopCapture(); @@ -237,7 +226,10 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr routeSubscriptionRef.current?.remove(); routeSubscriptionRef.current = null; playbackRef.current = null; - await configRef.current?.phoneVoice?.stopCapture(); + if (sourceRef.current?.kind === 'phone') { + await sourceRef.current.stopCapture(); + } + sourceRef.current = null; durableAudioSpool.close(); setIsStreaming(false); setIsConnecting(false); @@ -246,33 +238,21 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr const startStreaming = useCallback(async ( url: string, - config?: StreamStartConfig, + source: AudioStreamSource, ): Promise => { - if (connectingRef.current) return connectingRef.current; - const operation = (async () => { - stoppedRef.current = false; - urlRef.current = url; - configRef.current = config ?? configRef.current; - setIsConnecting(true); - setError(null); - const phoneVoice = configRef.current?.phoneVoice; - let parsed = socketOptions(url, Boolean(phoneVoice)); - const managedToken = await getValidToken(); - const token = managedToken ?? ( - parsed.bearerToken && !isTokenExpired(parsed.bearerToken) - ? parsed.bearerToken - : null - ); + stoppedRef.current = false; + urlRef.current = url; + sourceRef.current = source; + setIsConnecting(true); + setError(null); + try { + const phoneVoice = source.kind === 'phone' ? source : null; + const token = await getValidToken(); if (!token) throw new Error('Audio authentication expired'); - if (token !== parsed.bearerToken) { - optionsRef.current?.onTokenRefreshed?.(token); - const refreshed = new URL(url); - refreshed.searchParams.set('token', token); - urlRef.current = refreshed.toString(); - parsed = socketOptions(urlRef.current, Boolean(phoneVoice)); - } const socket = new AudioV2Socket({ - ...parsed, + url, + bearerToken: token, + ...socketSource(source), onPacketAccepted: packetAccepted, onControl: control => { if (control.event.case === 'playbackOffer') { @@ -324,23 +304,9 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr }, onClosed: () => { if (phoneVoice) phoneAudioDiagnostics.socketClosed(stoppedRef.current); + deliveryModeRef.current = 'idle'; setIsStreaming(false); - if ( - !stoppedRef.current && - (optionsRef.current?.autoReconnectEnabled ?? true) && - !reconnectRef.current - ) { - const delay = Math.min( - RECONNECT_MAX_MS, - RECONNECT_BASE_MS * (2 ** reconnectAttemptRef.current++) - ); - reconnectRef.current = setTimeout(() => { - reconnectRef.current = null; - startStreaming(urlRef.current).catch(cause => { - setError(cause instanceof Error ? cause.message : 'Audio reconnect failed'); - }); - }, delay); - } + if (!stoppedRef.current) setError('Audio connection closed'); }, onDiagnostic: event => { if (phoneVoice) phoneAudioDiagnostics.socketStage(event.stage, event.detail); @@ -351,7 +317,7 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr await socket.connect(); if (phoneVoice) phoneAudioDiagnostics.socketOpen(); deliveryModeRef.current = 'recovering'; - await drainRecovery(socket); + await drainRecovery(socket, source.kind); liveStartedAtRef.current = Date.now(); liveSequenceRef.current = 0; const capabilities = phoneVoice @@ -396,30 +362,24 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr if (heartbeatRef.current) clearInterval(heartbeatRef.current); heartbeatRef.current = null; socket.stopCapture() - .catch(() => undefined) .then(() => { socket.close(); return phoneVoice.restartCapture(); }) - .then(restarted => startStreaming(urlRef.current, { phoneVoice: restarted })) + .then(restarted => startStreaming(urlRef.current, { kind: 'phone', ...restarted })) .catch(cause => { setError(cause instanceof Error ? cause.message : 'Audio route restart failed'); }); }); } - reconnectAttemptRef.current = 0; heartbeatRef.current = setInterval( () => socket.heartbeat(performance.now()), HEARTBEAT_MS, ); setIsConnecting(false); setIsStreaming(true); - })(); - connectingRef.current = operation; - try { - await operation; } catch (cause) { - if (configRef.current?.phoneVoice) phoneAudioDiagnostics.failure('websocket_start', cause); + if (source.kind === 'phone') phoneAudioDiagnostics.failure('websocket_start', cause); setIsConnecting(false); setIsStreaming(false); const message = cause instanceof Error ? cause.message : 'Audio V2 connection failed'; @@ -432,56 +392,28 @@ export const useAudioStreamer = (options?: UseAudioStreamerOptions): UseAudioStr routeSubscriptionRef.current = null; deliveryModeRef.current = 'idle'; throw cause; - } finally { - connectingRef.current = null; } }, [drainRecovery, encodeBase64, packetAccepted]); - const enqueueLive = useCallback((opus: Uint8Array, capturedAtMs: number) => { - if (!opus.length) return; - const packet = durableAudioSpool.append(opus, capturedAtMs); - if (deliveryModeRef.current === 'live') { + const sendFrame = useCallback(( + source: AudioSpoolSource, + frame: CapturedOpusFrame, + ) => { + if (!frame.opus.length) return; + if (source === 'phone') phoneAudioDiagnostics.frameEnqueued(frame.opus.length); + const packet = durableAudioSpool.append(source, frame.opus, frame.capturedAtMs); + if (deliveryModeRef.current === 'live' && sourceRef.current?.kind === source) { sendSpoolPacket(packet, liveSequenceRef.current++); } }, [sendSpoolPacket]); - const sendDurableAudio = useCallback((audioBytes: Uint8Array) => { - enqueueLive(audioBytes, Date.now()); - }, [enqueueLive]); - - const sendInteractiveFrame = useCallback((frame: CapturedOpusFrame) => { - phoneAudioDiagnostics.frameEnqueued(frame.opus.length); - enqueueLive(frame.opus, frame.capturedAtMs); - }, [enqueueLive]); - - useEffect(() => { - const unsubscribe = NetInfo.addEventListener(state => { - if ( - state.isConnected && - state.isInternetReachable && - !stoppedRef.current && - socketRef.current?.readyState !== WebSocket.OPEN && - urlRef.current - ) { - startStreaming(urlRef.current).catch(() => undefined); - } - }); - return () => { - unsubscribe(); - stoppedRef.current = true; - socketRef.current?.close(); - }; - }, [startStreaming]); - return { isStreaming, isConnecting, error, phonePlaybackState, startStreaming, - getWebSocketReadyState: () => socketRef.current?.readyState, stopStreaming, - sendDurableAudio, - sendInteractiveFrame, + sendFrame, }; }; diff --git a/app/src/hooks/useAudioStreamingOrchestrator.ts b/app/src/hooks/useAudioStreamingOrchestrator.ts index 695d9f69c..5dc24aa5a 100644 --- a/app/src/hooks/useAudioStreamingOrchestrator.ts +++ b/app/src/hooks/useAudioStreamingOrchestrator.ts @@ -1,8 +1,8 @@ -import { useState, useCallback } from 'react'; +import { useCallback } from 'react'; import { Alert } from 'react-native'; -import { OmiConnection } from 'friend-lite-react-native'; -import { AppSettings } from './useAppSettings'; -import type { StreamStartConfig } from './useAudioStreamer'; +import { BleAudioCodec, OmiConnection } from 'friend-lite-react-native'; +import type { AppSettings } from './useAppSettings'; +import type { AudioStreamSource } from './useAudioStreamer'; import type { PhoneCaptureSession } from './usePhoneAudioRecorder'; import type { CapturedOpusFrame } from '../protocol/capturedOpusFrame'; import { phoneAudioDiagnostics } from '../services/phoneAudioDiagnostics'; @@ -13,28 +13,23 @@ interface OrchestratorParams { connectedDeviceId: string | null; }; audioStreamer: { - isStreaming: boolean; - startStreaming: (url: string, config?: StreamStartConfig) => Promise; + startStreaming: (url: string, source: AudioStreamSource) => Promise; stopStreaming: () => Promise; - sendDurableAudio: (audioBytes: Uint8Array) => void; - sendInteractiveFrame: (frame: CapturedOpusFrame) => void; - getWebSocketReadyState: () => number | undefined; + sendFrame: (source: 'phone' | 'wearable', frame: CapturedOpusFrame) => void; }; phoneAudioRecorder: { isRecording: boolean; startRecording: ( onData: (frame: CapturedOpusFrame) => Promise ) => Promise; - stopRecording: () => Promise; }; originalStartAudioListener: (onAudioData: (bytes: Uint8Array) => void) => Promise; originalStopAudioListener: () => Promise; - settings: AppSettings; + settings: Pick; } export interface AudioOrchestrator { isPhoneAudioMode: boolean; - setIsPhoneAudioMode: (mode: boolean) => void; handleStartAudioListeningAndStreaming: () => Promise; handleStopAudioListeningAndStreaming: () => Promise; handleTogglePhoneAudio: () => Promise; @@ -49,47 +44,14 @@ export const useAudioStreamingOrchestrator = ({ originalStopAudioListener, settings, }: OrchestratorParams): AudioOrchestrator => { - const [isPhoneAudioMode, setIsPhoneAudioMode] = useState(false); - - const buildWebSocketUrl = useCallback((baseUrl: string): string => { + const buildAudioWebSocketUrl = useCallback((baseUrl: string): string => { let url = baseUrl.trim(); url = url.replace(/^http:/, 'ws:').replace(/^https:/, 'wss:'); const parsed = new URL(url); parsed.pathname = '/ws/audio'; parsed.search = ''; - url = parsed.toString(); - - const isAdvanced = settings.jwtToken && settings.isAuthenticated; - if (isAdvanced) { - const params = new URLSearchParams(); - params.append('token', settings.jwtToken!); - const deviceName = settings.userId?.trim() || 'phone'; - params.append('device_name', deviceName); - const separator = url.includes('?') ? '&' : '?'; - url = `${url}${separator}${params.toString()}`; - } - return url; - }, [settings.jwtToken, settings.isAuthenticated, settings.userId]); - - const buildPhoneWebSocketUrl = useCallback((baseUrl: string): string => { - let url = baseUrl.trim(); - url = url.replace(/^http:/, 'ws:').replace(/^https:/, 'wss:'); - const parsed = new URL(url); - parsed.pathname = '/ws/audio'; - parsed.search = ''; - url = parsed.toString(); - - const isAdvanced = settings.jwtToken && settings.isAuthenticated; - if (isAdvanced) { - const params = new URLSearchParams(); - params.append('token', settings.jwtToken!); - const deviceName = 'phone-mic'; - params.append('device_name', deviceName); - const separator = url.includes('?') ? '&' : '?'; - url = `${url}${separator}${params.toString()}`; - } - return url; - }, [settings.jwtToken, settings.isAuthenticated]); + return parsed.toString(); + }, []); const handleStartAudioListeningAndStreaming = useCallback(async () => { if (!settings.webSocketUrl?.trim()) { @@ -102,22 +64,30 @@ export const useAudioStreamingOrchestrator = ({ } try { - const finalUrl = buildWebSocketUrl(settings.webSocketUrl); + const codec = await omiConnection.getAudioCodec(); + if (codec !== BleAudioCodec.OPUS) { + throw new Error(`Wearable must stream Opus; device reported ${codec}`); + } + const sourceId = deviceConnection.connectedDeviceId; + const finalUrl = buildAudioWebSocketUrl(settings.webSocketUrl); await originalStartAudioListener(async (audioBytes) => { if (audioBytes.length > 0) { - audioStreamer.sendDurableAudio(audioBytes); + audioStreamer.sendFrame('wearable', { + captureEpoch: 0, + capturedAtMs: Date.now(), + monotonicTimestampMs: performance.now(), + frameDurationMs: 60, + opus: audioBytes, + }); } }); - // BLE capture is independent of network availability. The durable spool above - // keeps receiving while this connection attempt fails or reconnects. - audioStreamer.startStreaming(finalUrl).catch((error) => { - console.warn('[AudioOrchestrator] Initial WebSocket connection failed; buffering locally:', error); - }); + await audioStreamer.startStreaming(finalUrl, { kind: 'wearable', sourceId }); } catch (error) { Alert.alert('Error', 'Could not start audio listening or streaming.'); - if (audioStreamer.isStreaming) audioStreamer.stopStreaming(); + await originalStopAudioListener(); + await audioStreamer.stopStreaming(); } - }, [originalStartAudioListener, audioStreamer, settings.webSocketUrl, omiConnection, deviceConnection.connectedDeviceId, buildWebSocketUrl]); + }, [originalStartAudioListener, audioStreamer, settings.webSocketUrl, omiConnection, deviceConnection.connectedDeviceId, buildAudioWebSocketUrl]); const handleStopAudioListeningAndStreaming = useCallback(async () => { await originalStopAudioListener(); @@ -131,47 +101,35 @@ export const useAudioStreamingOrchestrator = ({ } try { - const finalUrl = buildPhoneWebSocketUrl(settings.webSocketUrl); + const finalUrl = buildAudioWebSocketUrl(settings.webSocketUrl); const capture = await phoneAudioRecorder.startRecording(async (frame) => { if (frame.opus.length === 0) return; - const wsReady = audioStreamer.getWebSocketReadyState(); - if (wsReady !== WebSocket.OPEN) { - phoneAudioDiagnostics.socketUnavailable(wsReady); - } - // Native capture owns durability. Queue every frame first; the streamer - // sends immediately while live and replays the spool after reconnecting. - audioStreamer.sendInteractiveFrame(frame); + audioStreamer.sendFrame('phone', frame); }); - await audioStreamer.startStreaming(finalUrl, { phoneVoice: capture }); - setIsPhoneAudioMode(true); + await audioStreamer.startStreaming(finalUrl, { kind: 'phone', ...capture }); } catch (error) { phoneAudioDiagnostics.failure('orchestrator_start', error); Alert.alert('Error', 'Could not start phone audio streaming.'); - if (audioStreamer.isStreaming) audioStreamer.stopStreaming(); - if (phoneAudioRecorder.isRecording) await phoneAudioRecorder.stopRecording(); - setIsPhoneAudioMode(false); + await audioStreamer.stopStreaming(); } - }, [audioStreamer, phoneAudioRecorder, settings.webSocketUrl, buildPhoneWebSocketUrl]); + }, [audioStreamer, phoneAudioRecorder, settings.webSocketUrl, buildAudioWebSocketUrl]); const handleStopPhoneAudioStreaming = useCallback(async () => { await audioStreamer.stopStreaming(); - await phoneAudioRecorder.stopRecording(); - setIsPhoneAudioMode(false); phoneAudioDiagnostics.stopped(); - }, [phoneAudioRecorder, audioStreamer]); + }, [audioStreamer]); const handleTogglePhoneAudio = useCallback(async () => { - if (isPhoneAudioMode || phoneAudioRecorder.isRecording) { + if (phoneAudioRecorder.isRecording) { await handleStopPhoneAudioStreaming(); } else { phoneAudioDiagnostics.beginAttempt(); await handleStartPhoneAudioStreaming(); } - }, [isPhoneAudioMode, phoneAudioRecorder.isRecording, handleStartPhoneAudioStreaming, handleStopPhoneAudioStreaming]); + }, [phoneAudioRecorder.isRecording, handleStartPhoneAudioStreaming, handleStopPhoneAudioStreaming]); return { - isPhoneAudioMode, - setIsPhoneAudioMode, + isPhoneAudioMode: phoneAudioRecorder.isRecording, handleStartAudioListeningAndStreaming, handleStopAudioListeningAndStreaming, handleTogglePhoneAudio, diff --git a/app/src/protocol/audioV2Socket.ts b/app/src/protocol/audioV2Socket.ts index b274f42ef..390771a3b 100644 --- a/app/src/protocol/audioV2Socket.ts +++ b/app/src/protocol/audioV2Socket.ts @@ -76,6 +76,7 @@ export interface AudioV2SocketOptions { sourceId: string; displayName: string; deviceKind: DeviceKind; + uplinkFrameDurationMs: 20 | 60; onPacketAccepted?: (sequence: number) => void; onPlaybackPacket?: (packet: PlaybackMediaPacket) => void; onControl?: (control: ServerControl) => void; @@ -114,12 +115,12 @@ function eventId() { return create(EventIdSchema, { value: createClientEventIdValue() }); } -function uplinkSpec() { +function uplinkSpec(frameDurationMs: 20 | 60) { return create(AudioSpecSchema, { codec: AudioCodec.OPUS, sampleRateHz: 16_000, channelCount: 1, - frameDuration: create(DurationSchema, { nanos: 20_000_000 }), + frameDuration: create(DurationSchema, { nanos: frameDurationMs * 1_000_000 }), bitrateBps: 24_000, }); } @@ -175,7 +176,7 @@ export class AudioV2Socket { sourceId: create(CaptureSourceIdSchema, { value: this.options.sourceId }), deviceKind: this.options.deviceKind, displayName: this.options.displayName, - supportedUplink: [uplinkSpec()], + supportedUplink: [uplinkSpec(this.options.uplinkFrameDurationMs)], supportedDownlink: [downlinkSpec()], }), }); @@ -212,7 +213,7 @@ export class AudioV2Socket { processingProfile: options.processingProfile, dataPurpose: options.dataPurpose ?? DataPurpose.NORMAL_CAPTURE, deliveryClass: options.deliveryClass, - audioSpec: uplinkSpec(), + audioSpec: uplinkSpec(this.options.uplinkFrameDurationMs), capabilities: options.capabilities, recoveryBatchId: options.recoveryBatchId ?? '', }), diff --git a/app/src/services/auth.ts b/app/src/services/auth.ts index 285f78b59..5714dc93b 100644 --- a/app/src/services/auth.ts +++ b/app/src/services/auth.ts @@ -53,7 +53,6 @@ export const isTokenExpired = (token: string | null, skewSeconds = 60): boolean return Date.now() / 1000 >= exp - skewSeconds; }; -/** Listeners notified whenever a fresh token is obtained (login or refresh). */ type TokenListener = (token: string) => void; const tokenListeners = new Set(); @@ -62,16 +61,6 @@ export const onTokenRefreshed = (listener: TokenListener): (() => void) => { return () => tokenListeners.delete(listener); }; -const notifyToken = (token: string) => { - tokenListeners.forEach(l => { - try { - l(token); - } catch (e) { - console.warn('[Auth] token listener error:', e); - } - }); -}; - // De-dupe concurrent refreshes: many callers may hit a 401 at once. let refreshInFlight: Promise | null = null; @@ -106,7 +95,7 @@ export const login = async ( await saveAuthEmail(email); await saveAuthPassword(password); await saveJwtToken(token); - notifyToken(token); + tokenListeners.forEach(listener => listener(token)); return token; }; diff --git a/app/src/services/durableAudioSpool.ts b/app/src/services/durableAudioSpool.ts index ab1d49997..0d7973719 100644 --- a/app/src/services/durableAudioSpool.ts +++ b/app/src/services/durableAudioSpool.ts @@ -6,6 +6,8 @@ const SEGMENT_MS = 30_000; const HEADER_BYTES = 16; const ACK_PREFIX = 'chronicle.audioSpool.ack.'; +export type AudioSpoolSource = 'phone' | 'wearable'; + export interface SpoolPacket { fileName: string; /** @@ -23,6 +25,7 @@ export interface SpoolPacket { interface ActiveSegment { file: File; handle: FileHandle; + source: AudioSpoolSource; segmentId: string; startedAtMs: number; nextSequence: number; @@ -34,9 +37,10 @@ const makeSegmentId = (): string => /** * Append-only, document-directory audio spool. * - * Every BLE packet reaches this file before it is offered to the WebSocket. Files - * are retained until the backend acknowledges the packet sequence after writing - * the decoded audio to its Redis WAL. Old files are discovered after app restart. + * Every captured packet reaches a source-tagged segment before it is offered to the + * WebSocket. Files remain until the backend accepts their decoded PCM into Redis. + * Phone and wearable packets never share a recovery capture because their declared + * Opus durations differ. */ export class DurableAudioSpool { private readonly directory = new Directory(Paths.document, 'chronicle-audio-spool'); @@ -55,14 +59,15 @@ export class DurableAudioSpool { this.active = null; } - private startSegment(capturedAtMs: number): ActiveSegment { + private startSegment(source: AudioSpoolSource, capturedAtMs: number): ActiveSegment { this.ensureDirectory(); const segmentId = makeSegmentId(); - const file = new File(this.directory, `${segmentId}.spool`); + const file = new File(this.directory, `${source}-${segmentId}.spool`); file.create({ overwrite: false, intermediates: true }); const active = { file, handle: file.open(), + source, segmentId, startedAtMs: capturedAtMs, nextSequence: 0, @@ -71,11 +76,19 @@ export class DurableAudioSpool { return active; } - append(payload: Uint8Array, capturedAtMs = Date.now()): SpoolPacket { + append( + source: AudioSpoolSource, + payload: Uint8Array, + capturedAtMs = Date.now(), + ): SpoolPacket { let segment = this.active; - if (!segment || capturedAtMs - segment.startedAtMs >= SEGMENT_MS) { + if ( + !segment || + segment.source !== source || + capturedAtMs - segment.startedAtMs >= SEGMENT_MS + ) { this.closeActive(); - segment = this.startSegment(capturedAtMs); + segment = this.startSegment(source, capturedAtMs); } const sequence = segment.nextSequence++; @@ -96,12 +109,14 @@ export class DurableAudioSpool { }; } - async pendingPackets(): Promise { + async pendingPackets(source: AudioSpoolSource): Promise { this.ensureDirectory(); const packets: SpoolPacket[] = []; const files = this.directory .list() - .filter((entry): entry is File => entry instanceof File && entry.name.endsWith('.spool')); + .filter((entry): entry is File => ( + entry instanceof File && entry.name.startsWith(`${source}-`) && entry.name.endsWith('.spool') + )); for (const file of files) { await this.acknowledgmentChains.get(file.name)?.catch(() => undefined); diff --git a/app/src/services/phoneAudioSelfTest.ts b/app/src/services/phoneAudioSelfTest.ts index 71207f712..59735cb6c 100644 --- a/app/src/services/phoneAudioSelfTest.ts +++ b/app/src/services/phoneAudioSelfTest.ts @@ -256,6 +256,7 @@ async function runNetworkProbe( sourceId: 'phone-audio-diagnostics', displayName: 'phone-audio-diagnostics', deviceKind: Platform.OS === 'ios' ? DeviceKind.IOS_PHONE : DeviceKind.ANDROID_PHONE, + uplinkFrameDurationMs: 20, onPacketAccepted: sequence => { acked.add(sequence); if (acked.size >= payload.packets.length) finishAcknowledgements(); diff --git a/backends/advanced/src/advanced_omi_backend/audio_contract/v2/codec.py b/backends/advanced/src/advanced_omi_backend/audio_contract/v2/codec.py index 77d6f3a21..98ef4abe8 100644 --- a/backends/advanced/src/advanced_omi_backend/audio_contract/v2/codec.py +++ b/backends/advanced/src/advanced_omi_backend/audio_contract/v2/codec.py @@ -23,19 +23,40 @@ class AudioProtocolV2Error(ValueError): """A v2 envelope failed schema or Chronicle invariant validation.""" -class RawOpusDecoder: - """Stateful decoder for Chronicle's 16 kHz mono, 20 ms raw-Opus uplink.""" +CANONICAL_FRAME_MS = 20 +UPLINK_FRAME_MS = frozenset({20, 60}) - def __init__(self) -> None: + +class RawOpusNormalizer: + """Decode one declared uplink packet into canonical 20 ms PCM frames.""" + + def __init__(self, frame_duration_ms: int) -> None: + if frame_duration_ms not in UPLINK_FRAME_MS: + raise AudioProtocolV2Error("uplink Opus requires 20 or 60 ms packets") + self._frame_duration_ms = frame_duration_ms self._decoder = opuslib.Decoder(16_000, 1) - def decode_packet(self, payload: bytes) -> bytes: + def decode_frames(self, payload: bytes) -> tuple[bytes, ...]: if not payload: raise AudioProtocolV2Error("capture packet has no Opus payload") try: - return self._decoder.decode(payload, 320, decode_fec=False) + pcm = self._decoder.decode( + payload, + 16 * self._frame_duration_ms, + decode_fec=False, + ) except Exception as error: raise AudioProtocolV2Error("invalid raw Opus packet") from error + expected_bytes = 32 * self._frame_duration_ms + if len(pcm) != expected_bytes: + raise AudioProtocolV2Error( + f"Opus packet decoded to {len(pcm)} bytes; expected {expected_bytes}" + ) + canonical_bytes = 32 * CANONICAL_FRAME_MS + return tuple( + pcm[offset : offset + canonical_bytes] + for offset in range(0, len(pcm), canonical_bytes) + ) def _require_id(value: str, label: str) -> None: @@ -117,16 +138,25 @@ def serialize_server_control_json(message: audio_pb2.ServerControl) -> str: ) +def frame_duration_ms(spec: audio_pb2.AudioSpec) -> int: + if not spec.HasField("frame_duration"): + raise AudioProtocolV2Error("audio_spec requires frame_duration") + if spec.frame_duration.seconds != 0: + raise AudioProtocolV2Error("audio frame duration must be below one second") + return spec.frame_duration.nanos // 1_000_000 + + def validate_audio_spec(spec: audio_pb2.AudioSpec, *, live_uplink: bool) -> None: if spec.codec != audio_pb2.AUDIO_CODEC_OPUS: raise AudioProtocolV2Error("live audio requires raw Opus") expected_rate = 16_000 if live_uplink else 24_000 if spec.sample_rate_hz != expected_rate or spec.channel_count != 1: raise AudioProtocolV2Error(f"live audio requires {expected_rate} Hz mono Opus") - if not spec.HasField("frame_duration"): - raise AudioProtocolV2Error("audio_spec requires frame_duration") - if spec.frame_duration.seconds != 0 or spec.frame_duration.nanos != 20_000_000: - raise AudioProtocolV2Error("live Opus requires 20 ms packets") + duration_ms = frame_duration_ms(spec) + allowed = UPLINK_FRAME_MS if live_uplink else {CANONICAL_FRAME_MS} + if spec.frame_duration.nanos % 1_000_000 or duration_ms not in allowed: + expected = "20 or 60 ms" if live_uplink else "20 ms" + raise AudioProtocolV2Error(f"live Opus requires {expected} packets") def validate_start_capture(message: audio_pb2.StartCapture) -> None: diff --git a/backends/advanced/src/advanced_omi_backend/controllers/audio_v2_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/audio_v2_controller.py index 70fbb9dc4..06f2d771b 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/audio_v2_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/audio_v2_controller.py @@ -11,7 +11,7 @@ import json import logging import uuid -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from functools import partial from fastapi import WebSocket, WebSocketDisconnect @@ -20,7 +20,8 @@ from advanced_omi_backend.audio_contract.v2 import audio_pb2 from advanced_omi_backend.audio_contract.v2.codec import ( AudioProtocolV2Error, - RawOpusDecoder, + RawOpusNormalizer, + frame_duration_ms, parse_client_control_json, parse_media_envelope, serialize_media_envelope, @@ -224,10 +225,12 @@ async def _subscribe_v2_transcripts( await pubsub.close() -async def _decode_opus(decoder: RawOpusDecoder, payload: bytes) -> bytes: +async def _decode_opus_frames( + normalizer: RawOpusNormalizer, payload: bytes +) -> tuple[bytes, ...]: return await asyncio.get_running_loop().run_in_executor( DECODER_EXECUTOR, - partial(decoder.decode_packet, payload), + partial(normalizer.decode_frames, payload), ) @@ -319,15 +322,13 @@ def _voice_capabilities( async def ingest_capture_packet( *, - websocket: WebSocket, packet: audio_pb2.CaptureMediaPacket, client_state, - audio_stream_producer, - user, - decoder: RawOpusDecoder, + normalizer: RawOpusNormalizer, v2_streams: AudioV2Streams, -) -> None: - """Decode one bound Opus packet and cross the durable/realtime seams.""" + canonical_sequence: int, +) -> int: + """Normalize one bound Opus packet and publish canonical 20 ms frames.""" session_id = client_state.stream_session_id if session_id is None: @@ -340,34 +341,30 @@ async def ingest_capture_packet( if packet.binding.voice_session_id.value != expected_voice: raise AudioProtocolV2Error("capture packet has a stale voice binding") - pcm = await _decode_opus(decoder, packet.opus_payload) - expected_bytes = 16_000 * 1 * 2 * 20 // 1_000 - if not pcm or len(pcm) != expected_bytes: - raise AudioProtocolV2Error( - f"Opus packet decoded to {len(pcm) if pcm else 0} bytes; " - f"expected {expected_bytes}" + frames = await _decode_opus_frames(normalizer, packet.opus_payload) + for index, pcm in enumerate(frames): + captured_at = timestamp_pb2.Timestamp() + captured_at.FromDatetime( + packet.captured_at.ToDatetime(tzinfo=timezone.utc) + + timedelta(milliseconds=index * 20) ) - - await v2_streams.publish_frame( - audio_pb2.CaptureStreamEvent( - frame=audio_pb2.CanonicalPcmFrame( - binding=packet.binding, - sequence=packet.sequence, - captured_at=packet.captured_at, - monotonic_offset_us=packet.monotonic_offset_us, - delivery_class=packet.delivery_class, - pcm_s16le=pcm, - data_purpose={ - "normal_capture": audio_pb2.DATA_PURPOSE_NORMAL_CAPTURE, - "annotation": audio_pb2.DATA_PURPOSE_ANNOTATION, - }[client_state.data_purpose], + await v2_streams.publish_frame( + audio_pb2.CaptureStreamEvent( + frame=audio_pb2.CanonicalPcmFrame( + binding=packet.binding, + sequence=canonical_sequence + index, + captured_at=captured_at, + monotonic_offset_us=packet.monotonic_offset_us + index * 20_000, + delivery_class=packet.delivery_class, + pcm_s16le=pcm, + data_purpose={ + "normal_capture": audio_pb2.DATA_PURPOSE_NORMAL_CAPTURE, + "annotation": audio_pb2.DATA_PURPOSE_ANNOTATION, + }[client_state.data_purpose], + ) ) ) - ) - if packet.delivery_class == audio_pb2.DELIVERY_CLASS_RECOVERED: - # Recovery is durable-only by construction. The typed persistence consumer - # will promote this event to Mongo; it must never enter the old live stream. - return + return canonical_sequence + len(frames) async def handle_audio_v2_websocket(websocket: WebSocket) -> None: @@ -383,6 +380,7 @@ async def handle_audio_v2_websocket(websocket: WebSocket) -> None: interim_task = None downlink_task = None v2_streams = None + active_binding = None try: first = await websocket.receive_text() hello_control = parse_client_control_json(first) @@ -428,9 +426,10 @@ async def handle_audio_v2_websocket(websocket: WebSocket) -> None: ), ) - decoder = RawOpusDecoder() + normalizer = None active_delivery_class = audio_pb2.DELIVERY_CLASS_UNSPECIFIED last_sequence = -1 + canonical_sequence = 0 while True: incoming = await websocket.receive() if incoming.get("type") == "websocket.disconnect": @@ -443,6 +442,8 @@ async def handle_audio_v2_websocket(websocket: WebSocket) -> None: raise AudioProtocolV2Error("capture is already active") start = control.start_capture active_delivery_class = start.delivery_class + source_frame_duration_ms = frame_duration_ms(start.audio_spec) + normalizer = RawOpusNormalizer(source_frame_duration_ms) provenance = _start_provenance(start) if provenance.memory_space_id: try: @@ -465,7 +466,7 @@ async def handle_audio_v2_websocket(websocket: WebSocket) -> None: "rate": 16_000, "width": 2, "channels": 1, - "frame_duration_ms": 20, + "frame_duration_ms": source_frame_duration_ms, "mode": "streaming", }, provenance=provenance, @@ -479,6 +480,7 @@ async def handle_audio_v2_websocket(websocket: WebSocket) -> None: ), capture_epoch=client_state.capture_epoch, ) + active_binding = binding v2_streams = await AudioV2Streams.open( producer.redis_client, event=audio_pb2.CaptureStreamEvent( @@ -557,7 +559,10 @@ async def handle_audio_v2_websocket(websocket: WebSocket) -> None: interim_task = None active_delivery_class = audio_pb2.DELIVERY_CLASS_UNSPECIFIED last_sequence = -1 + canonical_sequence = 0 + normalizer = None v2_streams = None + active_binding = None elif event == "heartbeat": await _send_control(websocket, heartbeat=control.heartbeat) elif event == "voice_ready": @@ -638,14 +643,14 @@ async def handle_audio_v2_websocket(websocket: WebSocket) -> None: raise AudioProtocolV2Error("packet sequence is not increasing") if v2_streams is None: raise AudioProtocolV2Error("media arrived before stream open") - await ingest_capture_packet( - websocket=websocket, + if normalizer is None: + raise AudioProtocolV2Error("media arrived before capture start") + canonical_sequence = await ingest_capture_packet( packet=packet, client_state=client_state, - audio_stream_producer=producer, - user=user, - decoder=decoder, + normalizer=normalizer, v2_streams=v2_streams, + canonical_sequence=canonical_sequence, ) await _send_control( websocket, @@ -666,6 +671,29 @@ async def handle_audio_v2_websocket(websocket: WebSocket) -> None: getattr(client_state, "stream_session_id", None), error, ) + if ( + client_state is not None + and client_state.stream_session_id is not None + and producer is not None + and v2_streams is not None + and active_binding is not None + ): + await finalize_capture_session( + client_state=client_state, + producer=producer, + user_id=user.user_id, + client_id=client_id, + completion_reason="protocol_error", + failure=str(error), + ) + await v2_streams.end( + audio_pb2.CaptureStreamEvent( + ended=audio_pb2.CaptureStreamEnded( + binding=active_binding, + reason=audio_pb2.STOP_REASON_AUDIO_DISCONNECT, + ) + ) + ) try: await _send_control( websocket, diff --git a/backends/advanced/src/advanced_omi_backend/controllers/capture_lifecycle.py b/backends/advanced/src/advanced_omi_backend/controllers/capture_lifecycle.py index bb2cf8085..a54d359b4 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/capture_lifecycle.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/capture_lifecycle.py @@ -14,7 +14,10 @@ track_client_user_relationship_async, ) from advanced_omi_backend.model_registry import get_models_registry -from advanced_omi_backend.models.audio_capture import CaptureStartProvenance +from advanced_omi_backend.models.audio_capture import ( + AudioCaptureSession, + CaptureStartProvenance, +) from advanced_omi_backend.plugins.events import BUTTON_STATE_TO_EVENT, ButtonState from advanced_omi_backend.redis_factory import create_async_redis from advanced_omi_backend.services.audio_stream.durability import ( @@ -25,6 +28,7 @@ get_audio_stream_producer, ) from advanced_omi_backend.services.audio_stream.session_store import ( + CompletionReason, SessionStatus, SessionStore, ) @@ -201,20 +205,37 @@ async def initialize_capture_session( async def finalize_capture_session( - *, client_state, producer, user_id: str, client_id: str + *, + client_state, + producer, + user_id: str, + client_id: str, + completion_reason: CompletionReason = "user_stopped", + failure: str | None = None, ) -> None: session_id = client_state.stream_session_id if session_id is None: return - await producer.finalize_session(session_id, completion_reason="user_stopped") + await producer.finalize_session(session_id, completion_reason=completion_reason) + if failure is not None: + capture = await AudioCaptureSession.find_one( + AudioCaptureSession.capture_session_id == session_id + ) + if capture is None: + raise RuntimeError(f"Capture session {session_id} disappeared") + await capture.set({"status": "failed", "failure": failure}) if client_state.markers: await producer.store.set_markers(session_id, client_state.markers) client_state.markers.clear() - await producer.store.mark_complete(session_id, "user_stopped") + await producer.store.mark_complete(session_id, completion_reason) await publish_sse_event_async( user_id, "session.ended", - {"session_id": session_id, "client_id": client_id, "reason": "user_stopped"}, + { + "session_id": session_id, + "client_id": client_id, + "reason": completion_reason, + }, ) client_state.stream_session_id = None client_state.last_persistence_healthcheck = 0.0 diff --git a/backends/advanced/src/advanced_omi_backend/services/audio_stream/session_store.py b/backends/advanced/src/advanced_omi_backend/services/audio_stream/session_store.py index 94ecafdc3..1269a2273 100644 --- a/backends/advanced/src/advanced_omi_backend/services/audio_stream/session_store.py +++ b/backends/advanced/src/advanced_omi_backend/services/audio_stream/session_store.py @@ -45,6 +45,7 @@ "inactivity_timeout", "max_duration", "all_jobs_complete", + "protocol_error", ] diff --git a/backends/advanced/src/advanced_omi_backend/workers/audio_jobs.py b/backends/advanced/src/advanced_omi_backend/workers/audio_jobs.py index 2cd859af5..14ac3c4e2 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/audio_jobs.py +++ b/backends/advanced/src/advanced_omi_backend/workers/audio_jobs.py @@ -513,7 +513,15 @@ async def persistence_group_drained() -> bool: # stale object here would replace the producer's wall-clock ``ended_at`` with # an absent or recorded-audio timestamp. Update only the field this worker # owns; audio evidence keeps its independent captured clock on each chunk. - await capture.set({"status": "complete"}) + completed_capture = await AudioCaptureSession.find_one( + AudioCaptureSession.capture_session_id == session_id + ) + if completed_capture is None: + raise AudioPersistenceInvariantError( + f"Capture session {session_id} disappeared before completion" + ) + if completed_capture.failure is None: + await completed_capture.set({"status": "complete"}) await redis_client.delete(f"audio_persistence:session:{session_id}") logger.info( diff --git a/backends/advanced/tests/test_audio_durability.py b/backends/advanced/tests/test_audio_durability.py index cb811af12..0303dde32 100644 --- a/backends/advanced/tests/test_audio_durability.py +++ b/backends/advanced/tests/test_audio_durability.py @@ -123,6 +123,7 @@ class _PersistedCapture: status = "active" time_basis = "received" ended_at = None + failure = None async def save(self): return None diff --git a/backends/advanced/tests/test_audio_persistence_lifecycle.py b/backends/advanced/tests/test_audio_persistence_lifecycle.py index 6fadf62b6..1ce7a79b3 100644 --- a/backends/advanced/tests/test_audio_persistence_lifecycle.py +++ b/backends/advanced/tests/test_audio_persistence_lifecycle.py @@ -87,6 +87,7 @@ class _PersistedCapture: time_basis = "received" status = "active" ended_at = None + failure = None async def save(self): return None @@ -211,3 +212,54 @@ async def insert(self): "status": "complete", "ended_at": producer_ended_at, } + + +@pytest.mark.asyncio +async def test_persistence_completion_preserves_protocol_failure(monkeypatch): + redis = _RaceRedis() + persisted = {"status": "failed", "failure": "invalid raw Opus packet"} + + class FailedCapture(_PersistedCapture): + status = "failed" + failure = "invalid raw Opus packet" + + async def set(self, updates): + persisted.update(updates) + + class FakeCapture: + capture_session_id = _QueryField() + find_one = AsyncMock(return_value=FailedCapture()) + + class FakeAudioChunk: + source_stream = _QueryField() + source_first_message_id = _QueryField() + capture_session_id = _QueryField() + find_one = AsyncMock(return_value=None) + find = staticmethod(lambda *_args, **_kwargs: _EmptyFind()) + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + async def insert(self): + return None + + monkeypatch.setattr(audio_jobs, "SessionStore", _RaceSessionStore) + monkeypatch.setattr(audio_jobs, "AudioCaptureSession", FakeCapture) + monkeypatch.setattr(audio_jobs, "AudioChunkDocument", FakeAudioChunk) + monkeypatch.setattr(audio_jobs, "get_current_job", lambda: object()) + monkeypatch.setattr(audio_jobs, "check_job_alive", AsyncMock(return_value=True)) + monkeypatch.setattr( + audio_jobs, "encode_pcm_to_opus", AsyncMock(return_value=b"opus") + ) + + await audio_jobs.audio_streaming_persistence_job.__wrapped__( + "session-1", + "user-1", + "client-1", + redis_client=redis, + ) + + assert persisted == { + "status": "failed", + "failure": "invalid raw Opus packet", + } diff --git a/backends/advanced/tests/test_audio_protocol_v2.py b/backends/advanced/tests/test_audio_protocol_v2.py index 7353cccc5..3fe3c00c3 100644 --- a/backends/advanced/tests/test_audio_protocol_v2.py +++ b/backends/advanced/tests/test_audio_protocol_v2.py @@ -9,11 +9,12 @@ from advanced_omi_backend.audio_contract.v2 import audio_pb2 from advanced_omi_backend.audio_contract.v2.codec import ( AudioProtocolV2Error, - RawOpusDecoder, + RawOpusNormalizer, parse_client_control_json, parse_media_envelope, serialize_client_control_json, serialize_media_envelope, + validate_audio_spec, ) from advanced_omi_backend.controllers.audio_v2_controller import ( _subscribe_v2_transcripts, @@ -42,7 +43,28 @@ def test_raw_opus_decoder_accepts_three_byte_silence_packet(): packet = Encoder(16_000, 1, "audio").encode(bytes(640), 320) assert len(packet) == 3 - assert len(RawOpusDecoder().decode_packet(packet)) == 640 + assert RawOpusNormalizer(20).decode_frames(packet) == (bytes(640),) + + +def test_raw_opus_normalizer_splits_one_neo_packet_into_canonical_frames(): + packet = Encoder(16_000, 1, "audio").encode(bytes(1_920), 960) + + frames = RawOpusNormalizer(60).decode_frames(packet) + + assert len(frames) == 3 + assert all(len(frame) == 640 for frame in frames) + + +def test_uplink_spec_accepts_only_declared_20_or_60_ms_packets(): + validate_audio_spec(_spec(), live_uplink=True) + sixty_ms = _spec() + sixty_ms.frame_duration.nanos = 60_000_000 + validate_audio_spec(sixty_ms, live_uplink=True) + + forty_ms = _spec() + forty_ms.frame_duration.nanos = 40_000_000 + with pytest.raises(AudioProtocolV2Error, match="20 or 60 ms"): + validate_audio_spec(forty_ms, live_uplink=True) def test_generated_control_json_round_trips_without_dictionary_contracts(): diff --git a/backends/advanced/tests/test_audio_v2_ingress.py b/backends/advanced/tests/test_audio_v2_ingress.py index cd3c88f03..16cc4222e 100644 --- a/backends/advanced/tests/test_audio_v2_ingress.py +++ b/backends/advanced/tests/test_audio_v2_ingress.py @@ -7,15 +7,15 @@ from advanced_omi_backend.audio_contract.v2 import audio_pb2 from advanced_omi_backend.audio_contract.v2.codec import AudioProtocolV2Error -from advanced_omi_backend.controllers import audio_v2_controller +from advanced_omi_backend.controllers import audio_v2_controller, capture_lifecycle pytestmark = pytest.mark.unit class Decoder: - def decode_packet(self, payload): + def decode_frames(self, payload): assert payload == b"raw-opus" - return b"\x00\x00" * 320 + return (b"\x00\x00" * 320,) def _packet(delivery_class=audio_pb2.DELIVERY_CLASS_LIVE): @@ -98,8 +98,8 @@ async def test_v2_opus_decodes_once_then_crosses_realtime_and_durable_seams( ): monkeypatch.setattr( audio_v2_controller, - "_decode_opus", - AsyncMock(return_value=b"\x00\x00" * 320), + "_decode_opus_frames", + AsyncMock(return_value=(b"\x00\x00" * 320,)), ) state = SimpleNamespace( stream_session_id="capture-1", @@ -112,25 +112,25 @@ async def test_v2_opus_decodes_once_then_crosses_realtime_and_durable_seams( user = SimpleNamespace(user_id="user-1", email="user@example.com") streams = SimpleNamespace(publish_frame=AsyncMock()) - await audio_v2_controller.ingest_capture_packet( - websocket=object(), + next_sequence = await audio_v2_controller.ingest_capture_packet( packet=_packet(), client_state=state, - audio_stream_producer=producer, - user=user, - decoder=Decoder(), + normalizer=Decoder(), v2_streams=streams, + canonical_sequence=7, ) + assert next_sequence == 8 assert streams.publish_frame.await_args.args[0].WhichOneof("event") == "frame" + assert streams.publish_frame.await_args.args[0].frame.sequence == 7 assert streams.publish_frame.await_args.args[0].frame.pcm_s16le == b"\x00\x00" * 320 async def test_recovered_packet_enters_only_typed_durable_stream(monkeypatch): monkeypatch.setattr( audio_v2_controller, - "_decode_opus", - AsyncMock(return_value=b"\x00\x00" * 320), + "_decode_opus_frames", + AsyncMock(return_value=(b"\x00\x00" * 320,)), ) state = SimpleNamespace( stream_session_id="capture-1", @@ -142,13 +142,11 @@ async def test_recovered_packet_enters_only_typed_durable_stream(monkeypatch): streams = SimpleNamespace(publish_frame=AsyncMock()) await audio_v2_controller.ingest_capture_packet( - websocket=object(), packet=_packet(audio_pb2.DELIVERY_CLASS_RECOVERED), client_state=state, - audio_stream_producer=SimpleNamespace(redis_client=object()), - user=SimpleNamespace(user_id="user-1", email="user@example.com"), - decoder=Decoder(), + normalizer=Decoder(), v2_streams=streams, + canonical_sequence=0, ) streams.publish_frame.assert_awaited_once() @@ -160,7 +158,6 @@ async def test_v2_media_rejects_stale_connection_binding(): with pytest.raises(AudioProtocolV2Error, match="stale session"): await audio_v2_controller.ingest_capture_packet( - websocket=object(), packet=packet, client_state=SimpleNamespace( stream_session_id="capture-1", @@ -169,8 +166,91 @@ async def test_v2_media_rejects_stale_connection_binding(): data_purpose="normal_capture", client_id="client-1", ), - audio_stream_producer=SimpleNamespace(redis_client=object()), - user=SimpleNamespace(user_id="user-1", email="user@example.com"), - decoder=Decoder(), + normalizer=Decoder(), v2_streams=SimpleNamespace(publish_frame=AsyncMock()), + canonical_sequence=0, ) + + +async def test_v2_60_ms_packet_publishes_three_contiguous_canonical_frames( + monkeypatch, +): + monkeypatch.setattr( + audio_v2_controller, + "_decode_opus_frames", + AsyncMock( + return_value=( + b"\x01\x00" * 320, + b"\x02\x00" * 320, + b"\x03\x00" * 320, + ) + ), + ) + state = SimpleNamespace( + stream_session_id="capture-1", + voice_session_id="voice-1", + capture_epoch=9, + data_purpose="normal_capture", + ) + streams = SimpleNamespace(publish_frame=AsyncMock()) + + next_sequence = await audio_v2_controller.ingest_capture_packet( + packet=_packet(), + client_state=state, + normalizer=Decoder(), + v2_streams=streams, + canonical_sequence=30, + ) + + assert next_sequence == 33 + frames = [call.args[0].frame for call in streams.publish_frame.await_args_list] + assert [frame.sequence for frame in frames] == [30, 31, 32] + assert [frame.monotonic_offset_us for frame in frames] == [ + 240_000, + 260_000, + 280_000, + ] + assert [frame.captured_at.nanos for frame in frames] == [0, 20_000_000, 40_000_000] + + +async def test_protocol_rejection_marks_the_capture_failed(monkeypatch): + updates = [] + + class QueryField: + def __eq__(self, value): + return value + + class Capture: + async def set(self, values): + updates.append(values) + + class CaptureModel: + capture_session_id = QueryField() + find_one = AsyncMock(return_value=Capture()) + + monkeypatch.setattr(capture_lifecycle, "AudioCaptureSession", CaptureModel) + monkeypatch.setattr(capture_lifecycle, "publish_sse_event_async", AsyncMock()) + state = SimpleNamespace( + stream_session_id="capture-1", + markers=[], + last_persistence_healthcheck=5.0, + ) + producer = SimpleNamespace( + finalize_session=AsyncMock(), + store=SimpleNamespace(mark_complete=AsyncMock(), set_markers=AsyncMock()), + ) + + await capture_lifecycle.finalize_capture_session( + client_state=state, + producer=producer, + user_id="user-1", + client_id="client-1", + completion_reason="protocol_error", + failure="invalid raw Opus packet", + ) + + producer.finalize_session.assert_awaited_once_with( + "capture-1", completion_reason="protocol_error" + ) + assert updates == [{"status": "failed", "failure": "invalid raw Opus packet"}] + assert state.stream_session_id is None diff --git a/docs/backend/audio-interface-map.md b/docs/backend/audio-interface-map.md index 8e5339a91..8cf24777c 100644 --- a/docs/backend/audio-interface-map.md +++ b/docs/backend/audio-interface-map.md @@ -12,7 +12,7 @@ test and a deployed trace both exist. | IOS-CAPTURE | AVAudioEngine → app transport | PCM base64 map | native Opus `CaptureMediaPacket` | amber: source complete, native build pending | Expo typecheck; TestFlight/Xcode compile required | | ANDROID-CAPTURE | AudioRecord → app transport | PCM base64 map | native Opus `CaptureMediaPacket` | amber: source complete, Gradle/device pending | MediaCodec Opus adapter; Expo typecheck | | WEB-CAPTURE | Web Audio → backend | paired-header PCM | WebCodecs raw Opus packets | green source/build; browser E2E pending | `RecordingContext.test.tsx`; WebUI production build | -| OMI-NEO | BLE → app/tray → backend | raw Opus via Wyoming | Opus packet adapter | green app + tray adapters; physical device pending | `AudioV2Socket`; shared Python `AudioV2Client` test | +| OMI-NEO | BLE → app/tray → backend | raw Opus via Wyoming | declared 60 ms Opus normalized to canonical frames | amber: source/tests green; physical retest pending | real 60 ms Opus decode; app and shared-client duration tests | | HAVPE | firmware → relay → backend | device-local JSONL/PCM → relay V2 adapter | generated V2 at backend boundary | green source; physical pending | PCM normalizer/raw-Opus round trip; typed button/playback adapter | | RECOVERY | phone spool → backend | inferred bare packets | typed recovered packets | green app + ingress + persistence | typed packet ACK; `test_audio_durability.py` | | DURABLE-REDIS | ingress → persistence | string fields/magic end marker | `CaptureStreamEvent` binary | green V2 producer + persistence consumer | `test_audio_v2_streams.py`, `test_audio_durability.py` | @@ -30,7 +30,8 @@ test and a deployed trace both exist. ## Fixed invariants -- Live uplink is 16 kHz mono raw Opus in 20 ms packets. +- Live uplink is declared 16 kHz mono raw Opus: 20 ms from phone/web/HAVPE or + 60 ms from OMI/Neo. Ingress normalizes both to canonical 20 ms PCM frames. - Live downlink is 24 kHz mono raw Opus in 20 ms packets. - PCM S16LE is internal only. - Recovered packets never enter a live wake, turn, or action path. @@ -40,6 +41,24 @@ test and a deployed trace both exist. ## Worklog +### 2026-09-08 + +- Collapsed mobile capture to one explicit three-operation interface: start a named + source, enqueue a source-tagged frame, and stop. Removed URL-carried credentials, + optional phone/wearable inference, duplicate URL builders, caller-side socket + readiness checks, and the second reconnect state machine. +- Made source packet duration part of the socket interface. Phone declares 20 ms; + OMI/Neo declares its native 60 ms packets; HAVPE remains explicit 20 ms. The + backend decodes one declared packet and publishes one or three contiguous + canonical 20 ms frames. +- Source-tagged spool segments prevent a queued phone packet from being replayed + through a wearable decoder, or vice versa. The app refuses non-Opus wearable + capture before installing the BLE listener. +- Protocol rejection now finalizes the technical stream with `failure` and status + `failed`; the Mongo persistence worker preserves that result instead of replacing + it with `complete`. Verification is source-level only until backend deployment and + a physical Neo trace prove Redis and Mongo persistence. + ### 2026-08-29 - Added the Protobuf v2 source and generated Python/TypeScript bindings. diff --git a/extras/chronicle-client/chronicle_client/audio_v2.py b/extras/chronicle-client/chronicle_client/audio_v2.py index e7b5e0a2e..069bc54b0 100644 --- a/extras/chronicle-client/chronicle_client/audio_v2.py +++ b/extras/chronicle-client/chronicle_client/audio_v2.py @@ -21,6 +21,16 @@ def _timestamp(epoch_seconds: float) -> timestamp_pb2.Timestamp: return value +def _opus_spec(sample_rate_hz: int, frame_duration_ms: int) -> audio_pb2.AudioSpec: + return audio_pb2.AudioSpec( + codec=audio_pb2.AUDIO_CODEC_OPUS, + sample_rate_hz=sample_rate_hz, + channel_count=1, + frame_duration=duration_pb2.Duration(nanos=frame_duration_ms * 1_000_000), + bitrate_bps=24_000, + ) + + class AudioV2Client: """Own connection, capture binding, sequencing, and generated wire encoding.""" @@ -32,15 +42,19 @@ def __init__( source_id: str, display_name: str, device_kind: int, + uplink_frame_duration_ms: int, ssl_context=None, on_control: ControlHandler | None = None, on_playback: PlaybackHandler | None = None, ) -> None: + if uplink_frame_duration_ms not in {20, 60}: + raise ValueError("uplink_frame_duration_ms must be 20 or 60") self.websocket_url = websocket_url self.bearer_token = bearer_token self.source_id = source_id self.display_name = display_name self.device_kind = device_kind + self.uplink_frame_duration_ms = uplink_frame_duration_ms self.ssl_context = ssl_context self.on_control = on_control self.on_playback = on_playback @@ -53,16 +67,6 @@ def __init__( self._started_monotonic = 0.0 self._send_lock = asyncio.Lock() - @staticmethod - def opus_spec(sample_rate_hz: int) -> audio_pb2.AudioSpec: - return audio_pb2.AudioSpec( - codec=audio_pb2.AUDIO_CODEC_OPUS, - sample_rate_hz=sample_rate_hz, - channel_count=1, - frame_duration=duration_pb2.Duration(nanos=20_000_000), - bitrate_bps=24_000, - ) - async def connect(self) -> None: connect_options = dict( subprotocols=["chronicle.audio.v2"], @@ -81,8 +85,8 @@ async def connect(self) -> None: source_id=audio_pb2.CaptureSourceId(value=self.source_id), device_kind=self.device_kind, display_name=self.display_name, - supported_uplink=[self.opus_spec(16_000)], - supported_downlink=[self.opus_spec(24_000)], + supported_uplink=[_opus_spec(16_000, self.uplink_frame_duration_ms)], + supported_downlink=[_opus_spec(24_000, 20)], ) ) await hello @@ -104,7 +108,7 @@ async def start_capture( processing_profile=processing_profile, data_purpose=data_purpose, delivery_class=delivery_class, - audio_spec=self.opus_spec(16_000), + audio_spec=_opus_spec(16_000, self.uplink_frame_duration_ms), capabilities=capabilities, recovery_batch_id=recovery_batch_id, ) diff --git a/extras/chronicle-client/tests/test_audio_v2.py b/extras/chronicle-client/tests/test_audio_v2.py index 501e329b2..ebbdfb449 100644 --- a/extras/chronicle-client/tests/test_audio_v2.py +++ b/extras/chronicle-client/tests/test_audio_v2.py @@ -43,6 +43,18 @@ def _control(**event): ) +async def test_client_requires_an_explicit_supported_uplink_duration(): + with pytest.raises(ValueError, match="20 or 60"): + AudioV2Client( + websocket_url="wss://chronicle/ws/audio", + bearer_token="token", + source_id="bad", + display_name="bad", + device_kind=audio_pb2.DEVICE_KIND_PROBE, + uplink_frame_duration_ms=40, + ) + + async def test_client_sends_atomic_bound_opus_packet(monkeypatch): socket = FakeSocket() @@ -56,6 +68,7 @@ async def connect(*_args, **_kwargs): source_id="neo", display_name="Neo", device_kind=audio_pb2.DEVICE_KIND_NEO, + uplink_frame_duration_ms=60, ) connect_task = asyncio.create_task(client.connect()) await asyncio.sleep(0) @@ -77,7 +90,7 @@ async def connect(*_args, **_kwargs): await socket.incoming.put( _control( capture_started=audio_pb2.CaptureStarted( - binding=binding, audio_spec=client.opus_spec(16_000) + binding=binding, audio_spec=audio_pb2.AudioSpec() ) ) ) @@ -89,6 +102,10 @@ async def connect(*_args, **_kwargs): assert envelope.capture.binding == binding assert envelope.capture.sequence == 0 assert envelope.capture.opus_payload == b"raw-opus" - assert json.loads(socket.sent[0])["hello"]["bearer_token"] == "token" + hello = json.loads(socket.sent[0])["hello"] + assert hello["bearer_token"] == "token" + assert hello["supported_uplink"][0]["frame_duration"] == "0.060s" + start = json.loads(socket.sent[1])["start_capture"] + assert start["audio_spec"]["frame_duration"] == "0.060s" await socket.close() await asyncio.gather(client._receive_task, return_exceptions=True) diff --git a/extras/havpe-relay/relay_core.py b/extras/havpe-relay/relay_core.py index 69d895ed3..824197550 100644 --- a/extras/havpe-relay/relay_core.py +++ b/extras/havpe-relay/relay_core.py @@ -106,6 +106,7 @@ async def run_device_session( source_id=config.device_name, display_name=config.device_name, device_kind=audio_pb2.DEVICE_KIND_HAVPE, + uplink_frame_duration_ms=20, ) playback = HavpePlayback(client, device) client.on_control = playback.control diff --git a/extras/local-wearable-client/chronicle_wearable/backend.py b/extras/local-wearable-client/chronicle_wearable/backend.py index c3ac25db6..f081f60f5 100644 --- a/extras/local-wearable-client/chronicle_wearable/backend.py +++ b/extras/local-wearable-client/chronicle_wearable/backend.py @@ -87,6 +87,7 @@ async def stream_to_backend( if "neo" in device_name.lower() else audio_pb2.DEVICE_KIND_OMI ), + uplink_frame_duration_ms=60, ssl_context=_ssl_context(), ) connected_at = None diff --git a/tests/libs/audio_stream_library.py b/tests/libs/audio_stream_library.py index ee53dcca0..b75150352 100644 --- a/tests/libs/audio_stream_library.py +++ b/tests/libs/audio_stream_library.py @@ -69,6 +69,7 @@ def start_stream( source_id=device_name, display_name=device_name, device_kind=audio_pb2.DEVICE_KIND_PROBE, + uplink_frame_duration_ms=20, ) session = _Session(loop=loop, thread=thread, client=client) stream_id = str(uuid.uuid4()) @@ -251,4 +252,5 @@ def get_audio_stream_client( source_id=device_name, display_name=device_name, device_kind=audio_pb2.DEVICE_KIND_PROBE, + uplink_frame_duration_ms=20, ) From 318572ad9269de3a2b80f51c61d6f81b12e50a84 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:53:59 +0000 Subject: [PATCH 10/12] fix(app): use one live phone audio path --- .github/workflows/ios-testflight.yml | 1 - app/package.json | 3 +- app/scripts/test-durable-audio-spool.cjs | 96 -------- app/scripts/test-phone-audio-diagnostics.cjs | 13 +- ...ery.cjs => test-phone-audio-streaming.cjs} | 130 +++++------ app/src/hooks/useAudioStreamer.ts | 137 ++--------- app/src/hooks/usePhoneAudioRecorder.ts | 2 - app/src/services/durableAudioSpool.ts | 215 ------------------ app/src/services/phoneAudioDiagnostics.ts | 25 +- .../advanced/tests/test_spool_ack_contract.py | 71 ------ docs/backend/audio-interface-map.md | 14 +- scripts/rainbow-testflight-handoff.sh | 2 +- 12 files changed, 103 insertions(+), 606 deletions(-) delete mode 100644 app/scripts/test-durable-audio-spool.cjs rename app/scripts/{test-phone-audio-recovery.cjs => test-phone-audio-streaming.cjs} (62%) delete mode 100644 app/src/services/durableAudioSpool.ts delete mode 100644 backends/advanced/tests/test_spool_ack_contract.py diff --git a/.github/workflows/ios-testflight.yml b/.github/workflows/ios-testflight.yml index 26d1e2310..6f1a0ea21 100644 --- a/.github/workflows/ios-testflight.yml +++ b/.github/workflows/ios-testflight.yml @@ -51,7 +51,6 @@ jobs: run: | npm run test:wearable-activation npm run typecheck - npm run test:durable-audio-spool npm run test:phone-audio-diagnostics npm run test:push-notifications npm run check:theme diff --git a/app/package.json b/app/package.json index 60e425424..91f5efd87 100644 --- a/app/package.json +++ b/app/package.json @@ -11,8 +11,7 @@ "web": "expo start --web", "typecheck": "tsc --noEmit", "test:wearable-activation": "node scripts/test-wearable-activation.cjs", - "test:durable-audio-spool": "node scripts/test-durable-audio-spool.cjs", - "test:phone-audio-diagnostics": "node scripts/test-phone-audio-diagnostics.cjs && node scripts/test-phone-audio-recovery.cjs && node scripts/test-phone-audio-self-test.cjs && node scripts/test-client-diagnostic-metadata.cjs", + "test:phone-audio-diagnostics": "node scripts/test-phone-audio-diagnostics.cjs && node scripts/test-phone-audio-streaming.cjs && node scripts/test-phone-audio-self-test.cjs && node scripts/test-client-diagnostic-metadata.cjs", "test:push-notifications": "node scripts/test-push-notifications.cjs", "check:theme": "node scripts/check-theme-tokens.mjs" }, diff --git a/app/scripts/test-durable-audio-spool.cjs b/app/scripts/test-durable-audio-spool.cjs deleted file mode 100644 index 2e9c60455..000000000 --- a/app/scripts/test-durable-audio-spool.cjs +++ /dev/null @@ -1,96 +0,0 @@ -const assert = require('node:assert/strict'); -const fs = require('node:fs'); -const Module = require('node:module'); -const path = require('node:path'); -const ts = require('typescript'); - -function loadTypeScript(sourcePath, mocks) { - const source = fs.readFileSync(sourcePath, 'utf8'); - const compiled = ts.transpileModule(source, { - compilerOptions: { - esModuleInterop: true, - module: ts.ModuleKind.CommonJS, - target: ts.ScriptTarget.ES2020, - strict: true, - }, - fileName: sourcePath, - }); - const loaded = new Module(sourcePath, module); - loaded.filename = sourcePath; - loaded.paths = Module._nodeModulePaths(path.dirname(sourcePath)); - const originalRequire = loaded.require.bind(loaded); - loaded.require = (request) => mocks[request] ?? originalRequire(request); - loaded._compile(compiled.outputText, sourcePath); - return loaded.exports; -} - -const values = new Map(); -const wait = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); -const storage = { - async getItem(key) { - return values.get(key) ?? null; - }, - async setItem(key, value) { - // Make a lower ACK finish last. Concurrent read-modify-write logic regresses - // the durable watermark to 3 even though sequence 10 was already accepted. - await wait(value === '10' ? 5 : 20); - values.set(key, value); - }, - async removeItem(key) { - values.delete(key); - }, -}; - -class MockDirectory { - constructor() { - this.exists = true; - } - - list() { - return []; - } -} - -class MockFile { - constructor(_directory, name) { - this.name = name; - this.exists = false; - } -} - -const sourcePath = path.join(__dirname, '../src/services/durableAudioSpool.ts'); -const { DurableAudioSpool } = loadTypeScript(sourcePath, { - '@react-native-async-storage/async-storage': storage, - 'expo-file-system': { - Directory: MockDirectory, - File: MockFile, - Paths: { document: '/documents' }, - }, -}); - -(async () => { - const spool = new DurableAudioSpool(); - spool.active = { file: { name: 'segment.spool' } }; - const packet = (sequence) => ({ - fileName: 'segment.spool', - segmentId: 'segment', - sequence, - capturedAtMs: 1_770_000_000_000 + sequence, - payload: new Uint8Array([sequence]), - }); - - await Promise.all([ - spool.acknowledge(packet(10)), - spool.acknowledge(packet(3)), - ]); - - assert.equal( - values.get('chronicle.audioSpool.ack.segment.spool'), - '10', - 'out-of-order concurrent ACKs must never lower the durable watermark', - ); - console.log('durable audio spool tests passed'); -})().catch((error) => { - console.error(error); - process.exitCode = 1; -}); diff --git a/app/scripts/test-phone-audio-diagnostics.cjs b/app/scripts/test-phone-audio-diagnostics.cjs index 18d2d281f..a7139cd80 100644 --- a/app/scripts/test-phone-audio-diagnostics.cjs +++ b/app/scripts/test-phone-audio-diagnostics.cjs @@ -156,15 +156,13 @@ diagnostics.nativeStage({ captureEpoch: 1, stage: 'opus_encoded', monotonicTimes diagnostics.nativeFrame({ captureEpoch: 1, opusBytes: 42, audioLevel: 0.25 }); diagnostics.nativeFrame({ captureEpoch: 1, opusBytes: 43, audioLevel: 0.5 }); diagnostics.audioLevelActive(0.5); -diagnostics.socketUnavailable(0); -diagnostics.socketUnavailable(0); diagnostics.socketConnecting(); diagnostics.socketStage('transport_open'); diagnostics.socketStage('client_hello_sent'); diagnostics.socketStage('transport_error', 'wss://chronicle/ws/audio?token=secret-value'); diagnostics.socketOpen(); diagnostics.captureStarted('capture-secret-id'); -diagnostics.frameEnqueued(44); +diagnostics.frameSent(44); diagnostics.packetAccepted(0); diagnostics.timeout('meter_stalled'); diagnostics.failure('connect', 'wss://chronicle/ws/audio?token=secret-value'); @@ -180,7 +178,6 @@ assert.deepEqual( ['info', 'PhoneAudio'], ['info', 'PhoneAudio'], ['info', 'PhoneAudio'], - ['warn', 'PhoneAudio'], ['info', 'PhoneAudio'], ['info', 'PhoneAudio'], ['info', 'PhoneAudio'], @@ -192,7 +189,7 @@ assert.deepEqual( ['warn', 'PhoneAudio'], ['error', 'PhoneAudio'], ], - 'each lifecycle boundary must be exported once while repeated frames/drops become counters', + 'each lifecycle boundary must be exported once while repeated frames become counters', ); const text = writes.map(({ message }) => message).join('\n'); assert.match(text, /button_pressed attempt=1/); @@ -201,15 +198,14 @@ assert.match(text, /native_tap_received.*capture_epoch=1/); assert.match(text, /native_pcm_converted.*frames=320/); assert.match(text, /native_opus_encoded.*bytes=42/); assert.match(text, /audio_level_active.*audio_level=0\.500/); -assert.match(text, /frame_buffered_socket_not_open.*ready_state=0/); assert.match(text, /websocket_transport_open/); assert.match(text, /websocket_client_hello_sent/); assert.match(text, /websocket_transport_error.*token=/); -assert.match(text, /first_frame_enqueued.*opus_bytes=44/); +assert.match(text, /first_frame_sent.*opus_bytes=44/); assert.match(text, /first_packet_accepted.*sequence=0/); assert.match( text, - /meter_stalled.*native_frames=2.*buffered_while_disconnected=2.*enqueued_frames=1.*acked_packets=1.*last_audio_level=0\.500/, + /meter_stalled.*native_frames=2.*sent_frames=1.*acked_packets=1.*last_audio_level=0\.500/, ); assert.doesNotMatch(text, /capture-secret-id/, 'server-issued identifiers must be abbreviated'); assert.doesNotMatch(text, /secret-value/, 'credentials must be redacted from exported diagnostics'); @@ -305,7 +301,6 @@ const { useAudioStreamingOrchestrator } = loadTypeScript(orchestratorPath, { aec: { requested: true, available: true, enabled: true }, noise_suppression: { requested: true, available: true, enabled: true }, }, - restartCapture: async () => {}, stopCapture: async () => {}, }; }, diff --git a/app/scripts/test-phone-audio-recovery.cjs b/app/scripts/test-phone-audio-streaming.cjs similarity index 62% rename from app/scripts/test-phone-audio-recovery.cjs rename to app/scripts/test-phone-audio-streaming.cjs index ffd8d701e..7388cd058 100644 --- a/app/scripts/test-phone-audio-recovery.cjs +++ b/app/scripts/test-phone-audio-streaming.cjs @@ -32,10 +32,12 @@ const ProcessingProfile = { }; const DeliveryClass = { LIVE: 1, RECOVERED: 2 }; const beginCaptureCalls = []; +const sentPackets = []; const socketBearerTokens = []; const socketFrameDurations = []; -const pendingSources = []; -let pendingReads = 0; +const diagnostics = []; +let backendStops = 0; +let socketCloses = 0; class MockAudioV2Socket { constructor(options) { @@ -51,42 +53,48 @@ class MockAudioV2Socket { beginCaptureCalls.push(options); this.activeBinding = { captureSessionId: { value: `capture-${beginCaptureCalls.length}` }, - voiceSessionId: { value: beginCaptureCalls.length === 2 ? 'voice-live' : '' }, + voiceSessionId: { value: options.deliveryClass === DeliveryClass.LIVE ? 'voice-live' : '' }, captureEpoch: options.captureEpoch, }; return this.activeBinding; } sendPacket(packet) { + sentPackets.push(packet); this.options.onPacketAccepted(packet.sequence); } async stopCapture() { + backendStops += 1; this.activeBinding = null; } voiceReady() {} heartbeat() {} - close() {} acknowledgePlayback() {} + close() { + socketCloses += 1; + this.activeBinding = null; + } } -const noDiagnostics = new Proxy({}, { get: () => () => {} }); +const noDiagnostics = new Proxy({ + frameSent: bytes => diagnostics.push(['sent', bytes]), + packetAccepted: sequence => diagnostics.push(['accepted', sequence]), +}, { get: (target, property) => target[property] ?? (() => {}) }); + const sourcePath = path.join(__dirname, '../src/hooks/useAudioStreamer.ts'); const { useAudioStreamer } = loadTypeScript(sourcePath, { '@bufbuild/protobuf': { create: (_schema, value) => value }, - '@react-native-community/netinfo': { addEventListener: () => () => {} }, react: { - useCallback: (callback) => callback, - useEffect: () => {}, - useRef: (value) => ({ current: value }), - useState: (value) => [value, () => {}], + useCallback: callback => callback, + useRef: value => ({ current: value }), + useState: value => [value, () => {}], }, 'react-native': { Platform: { OS: 'ios' } }, - 'react-native-base64': { encode: (value) => value }, + 'react-native-base64': { encode: value => value }, '../../modules/chronicle-duplex-audio': { addPlaybackStateListener: () => ({ remove() {} }), - addRouteChangeListener: () => ({ remove() {} }), cancelResponse: async () => {}, scheduleResponse: async () => {}, }, @@ -103,38 +111,14 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { ProcessingProfile, }, '../protocol/audioV2Socket': { AudioV2Socket: MockAudioV2Socket }, - '../services/auth': { - getValidToken: async () => 'fresh-token', - }, - '../services/durableAudioSpool': { - durableAudioSpool: { - async pendingPackets(source) { - pendingSources.push(source); - pendingReads += 1; - return pendingReads === 1 - ? [{ - fileName: 'old.spool', - segmentId: 'old', - sequence: 7, - capturedAtMs: 1_780_000_000_000, - payload: new Uint8Array([1, 2, 3]), - }] - : []; - }, - async acknowledge() {}, - close() {}, - append() { - throw new Error('not used by this test'); - }, - }, - }, + '../services/auth': { getValidToken: async () => 'fresh-token' }, '../services/phoneAudioDiagnostics': { phoneAudioDiagnostics: noDiagnostics }, }); (async () => { - const streamer = useAudioStreamer(); + let nativeStops = 0; const phoneVoice = { - captureEpoch: 1, + captureEpoch: 7, capabilities: { mode: 'duplex_full', input_route: 'built_in_mic', @@ -143,9 +127,9 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { aec: { requested: true, available: true, enabled: true }, noise_suppression: { requested: true, available: true, enabled: true }, }, - restartCapture: async () => phoneVoice, - stopCapture: async () => {}, + stopCapture: async () => { nativeStops += 1; }, }; + const streamer = useAudioStreamer(); await streamer.startStreaming( 'wss://chronicle.invalid/ws/audio', @@ -154,36 +138,37 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { assert.deepEqual(socketBearerTokens, ['fresh-token'], 'audio must use the managed token source'); assert.deepEqual(socketFrameDurations, [20], 'phone capture must declare 20 ms Opus'); - assert.deepEqual(pendingSources, ['phone', 'phone'], 'recovery must read only the active source queue'); - assert.equal(beginCaptureCalls.length, 2, 'queued audio must recover before live capture starts'); - assert.deepEqual( - { - captureEpoch: beginCaptureCalls[0].captureEpoch, - processingProfile: beginCaptureCalls[0].processingProfile, - deliveryClass: beginCaptureCalls[0].deliveryClass, - }, - { - captureEpoch: 0, - processingProfile: ProcessingProfile.SOURCE_NATIVE, - deliveryClass: DeliveryClass.RECOVERED, - }, - 'recovered source-native audio must use epoch zero', - ); - assert.deepEqual( - { - captureEpoch: beginCaptureCalls[1].captureEpoch, - processingProfile: beginCaptureCalls[1].processingProfile, - deliveryClass: beginCaptureCalls[1].deliveryClass, - }, - { - captureEpoch: 1, - processingProfile: ProcessingProfile.DUPLEX_AEC, - deliveryClass: DeliveryClass.LIVE, - }, - 'the following live duplex capture must retain the native phone epoch', - ); + assert.equal(beginCaptureCalls.length, 1, 'one button press must create exactly one backend capture'); + assert.equal(beginCaptureCalls[0].deliveryClass, DeliveryClass.LIVE); + assert.equal(beginCaptureCalls[0].captureEpoch, 7); + assert.equal(beginCaptureCalls[0].processingProfile, ProcessingProfile.DUPLEX_AEC); + assert.equal(beginCaptureCalls[0].recoveryBatchId, undefined, 'the clean path has no recovery capture'); + + const now = performance.now(); + streamer.sendFrame('phone', { + captureEpoch: 6, + capturedAtMs: 1_780_000_000_000, + monotonicTimestampMs: now, + frameDurationMs: 20, + opus: new Uint8Array([9]), + }); + streamer.sendFrame('phone', { + captureEpoch: 7, + capturedAtMs: 1_780_000_000_020, + monotonicTimestampMs: now + 20, + frameDurationMs: 20, + opus: new Uint8Array([1, 2, 3]), + }); + assert.equal(sentPackets.length, 1, 'only the active native epoch may send'); + assert.equal(sentPackets[0].sequence, 0); + assert.equal(sentPackets[0].capturedAtMs, 1_780_000_000_020); + assert.deepEqual(Array.from(sentPackets[0].opus), [1, 2, 3]); + assert.deepEqual(diagnostics, [['sent', 3], ['accepted', 0]]); await streamer.stopStreaming(); + assert.equal(backendStops, 1, 'the capture has one stop owner'); + assert.equal(nativeStops, 1, 'stopping the stream also stops the native phone session'); + assert.equal(socketCloses, 1); const wearableStreamer = useAudioStreamer(); await wearableStreamer.startStreaming( @@ -191,9 +176,12 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { { kind: 'wearable', sourceId: 'neo-1' }, ); assert.deepEqual(socketFrameDurations, [20, 60], 'wearable capture must declare 60 ms Opus'); + assert.equal(beginCaptureCalls.length, 2, 'wearable also uses one live capture'); + assert.equal(beginCaptureCalls[1].deliveryClass, DeliveryClass.LIVE); await wearableStreamer.stopStreaming(); - console.log('phone audio recovery tests passed'); -})().catch((error) => { + + console.log('phone audio streaming tests passed'); +})().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/app/src/hooks/useAudioStreamer.ts b/app/src/hooks/useAudioStreamer.ts index b3eebfdc4..a76f7a03d 100644 --- a/app/src/hooks/useAudioStreamer.ts +++ b/app/src/hooks/useAudioStreamer.ts @@ -6,7 +6,6 @@ import base64 from 'react-native-base64'; import { addPlaybackStateListener, - addRouteChangeListener, cancelResponse, scheduleResponse, } from '../../modules/chronicle-duplex-audio'; @@ -28,11 +27,6 @@ import type { CapturedOpusFrame } from '../protocol/capturedOpusFrame'; import type { VoiceCapabilities } from '../protocol/audioCapabilities'; import type { PhoneCaptureSession } from './usePhoneAudioRecorder'; import { getValidToken } from '../services/auth'; -import { - durableAudioSpool, - type AudioSpoolSource, - type SpoolPacket, -} from '../services/durableAudioSpool'; import { phoneAudioDiagnostics } from '../services/phoneAudioDiagnostics'; export type AudioStreamSource = @@ -49,15 +43,11 @@ interface UseAudioStreamer { phonePlaybackState: 'started' | 'done' | 'cancelled' | 'failed' | null; startStreaming: (url: string, source: AudioStreamSource) => Promise; stopStreaming: () => Promise; - sendFrame: (source: AudioSpoolSource, frame: CapturedOpusFrame) => void; + sendFrame: (source: AudioStreamSource['kind'], frame: CapturedOpusFrame) => void; } const HEARTBEAT_MS = 25_000; -function recoveryBatchId(): string { - return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 12)}`; -} - function socketSource(source: AudioStreamSource) { if (source.kind === 'wearable') { return { @@ -122,14 +112,10 @@ export const useAudioStreamer = (): UseAudioStreamer => { const [phonePlaybackState, setPhonePlaybackState] = useState(null); const socketRef = useRef(null); const sourceRef = useRef(null); - const urlRef = useRef(''); const stoppedRef = useRef(false); const heartbeatRef = useRef | null>(null); const liveSequenceRef = useRef(0); const liveStartedAtRef = useRef(0); - const acceptedRef = useRef(new Map()); - const acceptedWaitersRef = useRef(new Map void>()); - const deliveryModeRef = useRef<'idle' | 'recovering' | 'live'>('idle'); const playbackRef = useRef<{ responseId: string; generation: number; @@ -137,7 +123,6 @@ export const useAudioStreamer = (): UseAudioStreamer => { nextSequence: number; } | null>(null); const playbackSubscriptionRef = useRef<{ remove: () => void } | null>(null); - const routeSubscriptionRef = useRef<{ remove: () => void } | null>(null); const encodeBase64 = useCallback((bytes: Uint8Array) => { let binary = ''; @@ -149,88 +134,25 @@ export const useAudioStreamer = (): UseAudioStreamer => { const packetAccepted = useCallback((sequence: number) => { phoneAudioDiagnostics.packetAccepted(sequence); - const packet = acceptedRef.current.get(sequence); - if (packet) { - acceptedRef.current.delete(sequence); - durableAudioSpool.acknowledge(packet).catch(cause => { - setError(cause instanceof Error ? cause.message : 'Could not retire audio spool packet'); - }); - } - acceptedWaitersRef.current.get(sequence)?.(); - acceptedWaitersRef.current.delete(sequence); - }, []); - - const sendSpoolPacket = useCallback((packet: SpoolPacket, sequence: number) => { - const socket = socketRef.current; - if (!socket?.activeBinding) return false; - acceptedRef.current.set(sequence, packet); - socket.sendPacket({ - sequence, - capturedAtMs: packet.capturedAtMs, - monotonicOffsetUs: Math.max( - 0, - Math.round((packet.capturedAtMs - liveStartedAtRef.current) * 1000) - ), - opus: packet.payload, - }); - return true; - }, []); - - const drainRecovery = useCallback(async ( - socket: AudioV2Socket, - source: AudioSpoolSource, - ) => { - let recoverySequence = 0; - while (true) { - const packets = await durableAudioSpool.pendingPackets(source); - if (!packets.length) return; - await socket.beginCapture({ - // Recovery is a source-native capture, whose protocol epoch is always zero. - // The native phone epoch belongs only to the subsequent live voice session. - captureEpoch: 0, - processingProfile: ProcessingProfile.SOURCE_NATIVE, - dataPurpose: DataPurpose.NORMAL_CAPTURE, - deliveryClass: DeliveryClass.RECOVERED, - recoveryBatchId: recoveryBatchId(), - }); - const acknowledgements = packets.map(packet => { - const sequence = recoverySequence++; - return new Promise(resolve => { - acceptedWaitersRef.current.set(sequence, resolve); - acceptedRef.current.set(sequence, packet); - socket.sendPacket({ - sequence, - capturedAtMs: packet.capturedAtMs, - monotonicOffsetUs: 0, - opus: packet.payload, - }); - }); - }); - await Promise.all(acknowledgements); - await socket.stopCapture(); - } }, []); const stopStreaming = useCallback(async () => { stoppedRef.current = true; - deliveryModeRef.current = 'idle'; if (heartbeatRef.current) clearInterval(heartbeatRef.current); heartbeatRef.current = null; + const socket = socketRef.current; try { - await socketRef.current?.stopCapture(); + await socket?.stopCapture(); } finally { - socketRef.current?.close(); + socket?.close(); socketRef.current = null; playbackSubscriptionRef.current?.remove(); playbackSubscriptionRef.current = null; - routeSubscriptionRef.current?.remove(); - routeSubscriptionRef.current = null; playbackRef.current = null; if (sourceRef.current?.kind === 'phone') { await sourceRef.current.stopCapture(); } sourceRef.current = null; - durableAudioSpool.close(); setIsStreaming(false); setIsConnecting(false); } @@ -241,7 +163,6 @@ export const useAudioStreamer = (): UseAudioStreamer => { source: AudioStreamSource, ): Promise => { stoppedRef.current = false; - urlRef.current = url; sourceRef.current = source; setIsConnecting(true); setError(null); @@ -304,7 +225,6 @@ export const useAudioStreamer = (): UseAudioStreamer => { }, onClosed: () => { if (phoneVoice) phoneAudioDiagnostics.socketClosed(stoppedRef.current); - deliveryModeRef.current = 'idle'; setIsStreaming(false); if (!stoppedRef.current) setError('Audio connection closed'); }, @@ -316,9 +236,7 @@ export const useAudioStreamer = (): UseAudioStreamer => { if (phoneVoice) phoneAudioDiagnostics.socketConnecting(); await socket.connect(); if (phoneVoice) phoneAudioDiagnostics.socketOpen(); - deliveryModeRef.current = 'recovering'; - await drainRecovery(socket, source.kind); - liveStartedAtRef.current = Date.now(); + liveStartedAtRef.current = performance.now(); liveSequenceRef.current = 0; const capabilities = phoneVoice ? typedCapabilities(phoneVoice.capabilities) @@ -335,7 +253,6 @@ export const useAudioStreamer = (): UseAudioStreamer => { if (phoneVoice) { phoneAudioDiagnostics.captureStarted(binding.captureSessionId?.value ?? ''); } - deliveryModeRef.current = 'live'; if (capabilities) socket.voiceReady(capabilities); if (phoneVoice) { playbackSubscriptionRef.current?.remove(); @@ -355,22 +272,6 @@ export const useAudioStreamer = (): UseAudioStreamer => { event.monotonicTimestampMs, ); }); - routeSubscriptionRef.current?.remove(); - routeSubscriptionRef.current = addRouteChangeListener(event => { - if (event.captureEpoch !== phoneVoice.captureEpoch) return; - stoppedRef.current = true; - if (heartbeatRef.current) clearInterval(heartbeatRef.current); - heartbeatRef.current = null; - socket.stopCapture() - .then(() => { - socket.close(); - return phoneVoice.restartCapture(); - }) - .then(restarted => startStreaming(urlRef.current, { kind: 'phone', ...restarted })) - .catch(cause => { - setError(cause instanceof Error ? cause.message : 'Audio route restart failed'); - }); - }); } heartbeatRef.current = setInterval( () => socket.heartbeat(performance.now()), @@ -388,24 +289,32 @@ export const useAudioStreamer = (): UseAudioStreamer => { socketRef.current = null; playbackSubscriptionRef.current?.remove(); playbackSubscriptionRef.current = null; - routeSubscriptionRef.current?.remove(); - routeSubscriptionRef.current = null; - deliveryModeRef.current = 'idle'; throw cause; } - }, [drainRecovery, encodeBase64, packetAccepted]); + }, [encodeBase64, packetAccepted]); const sendFrame = useCallback(( - source: AudioSpoolSource, + source: AudioStreamSource['kind'], frame: CapturedOpusFrame, ) => { if (!frame.opus.length) return; - if (source === 'phone') phoneAudioDiagnostics.frameEnqueued(frame.opus.length); - const packet = durableAudioSpool.append(source, frame.opus, frame.capturedAtMs); - if (deliveryModeRef.current === 'live' && sourceRef.current?.kind === source) { - sendSpoolPacket(packet, liveSequenceRef.current++); + const activeSource = sourceRef.current; + const socket = socketRef.current; + if (!socket?.activeBinding || activeSource?.kind !== source) return; + if (source === 'phone') { + if (activeSource.kind !== 'phone' || frame.captureEpoch !== activeSource.captureEpoch) return; + phoneAudioDiagnostics.frameSent(frame.opus.length); } - }, [sendSpoolPacket]); + socket.sendPacket({ + sequence: liveSequenceRef.current++, + capturedAtMs: frame.capturedAtMs, + monotonicOffsetUs: Math.max( + 0, + Math.round((frame.monotonicTimestampMs - liveStartedAtRef.current) * 1000), + ), + opus: frame.opus, + }); + }, []); return { isStreaming, diff --git a/app/src/hooks/usePhoneAudioRecorder.ts b/app/src/hooks/usePhoneAudioRecorder.ts index 95b7fb2ef..da1f5e979 100644 --- a/app/src/hooks/usePhoneAudioRecorder.ts +++ b/app/src/hooks/usePhoneAudioRecorder.ts @@ -23,7 +23,6 @@ const ACTIVE_AUDIO_LEVEL = 0.01; export interface PhoneCaptureSession { captureEpoch: number; capabilities: VoiceCapabilities; - restartCapture: () => Promise; stopCapture: () => Promise; } @@ -112,7 +111,6 @@ export const usePhoneAudioRecorder = (): UsePhoneAudioRecorder => { return { captureEpoch, capabilities, - restartCapture: startNativeCapture, stopCapture: stopRecording, }; }, [stopRecording]); diff --git a/app/src/services/durableAudioSpool.ts b/app/src/services/durableAudioSpool.ts deleted file mode 100644 index 0d7973719..000000000 --- a/app/src/services/durableAudioSpool.ts +++ /dev/null @@ -1,215 +0,0 @@ -import AsyncStorage from '@react-native-async-storage/async-storage'; -import { Directory, File, Paths } from 'expo-file-system'; -import type { FileHandle } from 'expo-file-system'; - -const SEGMENT_MS = 30_000; -const HEADER_BYTES = 16; -const ACK_PREFIX = 'chronicle.audioSpool.ack.'; - -export type AudioSpoolSource = 'phone' | 'wearable'; - -export interface SpoolPacket { - fileName: string; - /** - * Identity of the spool *file* this packet was written to, not the backend audio - * session. It was called `sessionId` and sent as `durable_session_id`, which the - * backend echoed back as `session_id` — three names for a spool segment, all of - * them colliding with the real WebSocket SessionId that means something else. - */ - segmentId: string; - sequence: number; - capturedAtMs: number; - payload: Uint8Array; -} - -interface ActiveSegment { - file: File; - handle: FileHandle; - source: AudioSpoolSource; - segmentId: string; - startedAtMs: number; - nextSequence: number; -} - -const makeSegmentId = (): string => - `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 12)}`; - -/** - * Append-only, document-directory audio spool. - * - * Every captured packet reaches a source-tagged segment before it is offered to the - * WebSocket. Files remain until the backend accepts their decoded PCM into Redis. - * Phone and wearable packets never share a recovery capture because their declared - * Opus durations differ. - */ -export class DurableAudioSpool { - private readonly directory = new Directory(Paths.document, 'chronicle-audio-spool'); - private active: ActiveSegment | null = null; - private readonly acknowledgmentChains = new Map>(); - - private ensureDirectory(): void { - if (!this.directory.exists) { - this.directory.create({ idempotent: true, intermediates: true }); - } - } - - private closeActive(): void { - if (!this.active) return; - this.active.handle.close(); - this.active = null; - } - - private startSegment(source: AudioSpoolSource, capturedAtMs: number): ActiveSegment { - this.ensureDirectory(); - const segmentId = makeSegmentId(); - const file = new File(this.directory, `${source}-${segmentId}.spool`); - file.create({ overwrite: false, intermediates: true }); - const active = { - file, - handle: file.open(), - source, - segmentId, - startedAtMs: capturedAtMs, - nextSequence: 0, - }; - this.active = active; - return active; - } - - append( - source: AudioSpoolSource, - payload: Uint8Array, - capturedAtMs = Date.now(), - ): SpoolPacket { - let segment = this.active; - if ( - !segment || - segment.source !== source || - capturedAtMs - segment.startedAtMs >= SEGMENT_MS - ) { - this.closeActive(); - segment = this.startSegment(source, capturedAtMs); - } - - const sequence = segment.nextSequence++; - const frame = new Uint8Array(HEADER_BYTES + payload.length); - const view = new DataView(frame.buffer); - view.setUint32(0, sequence, false); - view.setFloat64(4, capturedAtMs, false); - view.setUint32(12, payload.length, false); - frame.set(payload, HEADER_BYTES); - segment.handle.writeBytes(frame); - - return { - fileName: segment.file.name, - segmentId: segment.segmentId, - sequence, - capturedAtMs, - payload, - }; - } - - async pendingPackets(source: AudioSpoolSource): Promise { - this.ensureDirectory(); - const packets: SpoolPacket[] = []; - const files = this.directory - .list() - .filter((entry): entry is File => ( - entry instanceof File && entry.name.startsWith(`${source}-`) && entry.name.endsWith('.spool') - )); - - for (const file of files) { - await this.acknowledgmentChains.get(file.name)?.catch(() => undefined); - const segmentId = file.name.slice(0, -'.spool'.length); - const acknowledged = Number(await AsyncStorage.getItem(`${ACK_PREFIX}${file.name}`) ?? '-1'); - const bytes = file.bytesSync(); - let offset = 0; - let finalSequence = -1; - while (offset + HEADER_BYTES <= bytes.length) { - const view = new DataView(bytes.buffer, bytes.byteOffset + offset, HEADER_BYTES); - const sequence = view.getUint32(0, false); - const capturedAtMs = view.getFloat64(4, false); - const length = view.getUint32(12, false); - const end = offset + HEADER_BYTES + length; - if (end > bytes.length) break; // Ignore a final partial frame after a hard crash. - finalSequence = sequence; - if (sequence > acknowledged) { - packets.push({ - fileName: file.name, - segmentId, - sequence, - capturedAtMs, - payload: bytes.slice(offset + HEADER_BYTES, end), - }); - } - offset = end; - } - if ( - finalSequence >= 0 && - acknowledged >= finalSequence && - this.active?.file.name !== file.name - ) { - file.delete(); - await AsyncStorage.removeItem(`${ACK_PREFIX}${file.name}`); - } - } - return packets.sort((a, b) => a.capturedAtMs - b.capturedAtMs); - } - - private async acknowledgeInOrder(packet: SpoolPacket): Promise { - const ackKey = `${ACK_PREFIX}${packet.fileName}`; - const isActive = this.active?.file.name === packet.fileName; - const file = isActive ? null : new File(this.directory, packet.fileName); - if (file && !file.exists) { - // A higher ACK may already have retired this closed segment. Do not let a - // later, lower ACK recreate its watermark after the file is gone. - await AsyncStorage.removeItem(ackKey); - return; - } - - const previous = Number(await AsyncStorage.getItem(ackKey) ?? '-1'); - const acknowledged = Math.max(previous, packet.sequence); - await AsyncStorage.setItem(ackKey, String(acknowledged)); - if (isActive) return; - - const bytes = file!.bytesSync(); - let offset = 0; - let finalSequence = -1; - while (offset + HEADER_BYTES <= bytes.length) { - const view = new DataView(bytes.buffer, bytes.byteOffset + offset, HEADER_BYTES); - const sequence = view.getUint32(0, false); - const length = view.getUint32(12, false); - const end = offset + HEADER_BYTES + length; - if (end > bytes.length) break; - finalSequence = sequence; - offset = end; - } - if (finalSequence >= 0 && acknowledged >= finalSequence) { - file!.delete(); - await AsyncStorage.removeItem(ackKey); - } - } - - async acknowledge(packet: SpoolPacket): Promise { - const previous = this.acknowledgmentChains.get(packet.fileName) ?? Promise.resolve(); - const operation = previous - // A transient storage failure must be reported to its own caller, but it - // must not permanently poison retirement for every later ACK in the file. - .catch(() => undefined) - .then(() => this.acknowledgeInOrder(packet)); - this.acknowledgmentChains.set(packet.fileName, operation); - try { - await operation; - } finally { - if (this.acknowledgmentChains.get(packet.fileName) === operation) { - this.acknowledgmentChains.delete(packet.fileName); - } - } - } - - close(): void { - this.closeActive(); - } -} - -export const durableAudioSpool = new DurableAudioSpool(); diff --git a/app/src/services/phoneAudioDiagnostics.ts b/app/src/services/phoneAudioDiagnostics.ts index 9ae0b727e..7aae0fd12 100644 --- a/app/src/services/phoneAudioDiagnostics.ts +++ b/app/src/services/phoneAudioDiagnostics.ts @@ -28,8 +28,7 @@ export class PhoneAudioDiagnostics { private startedAtMs = 0; private milestones = new Set(); private nativeFrames = 0; - private bufferedWhileDisconnected = 0; - private enqueuedFrames = 0; + private sentFrames = 0; private ackedPackets = 0; private lastAudioLevel = 0; @@ -54,8 +53,7 @@ export class PhoneAudioDiagnostics { this.startedAtMs = this.now(); this.milestones.clear(); this.nativeFrames = 0; - this.bufferedWhileDisconnected = 0; - this.enqueuedFrames = 0; + this.sentFrames = 0; this.ackedPackets = 0; this.lastAudioLevel = 0; this.write('info', 'button_pressed'); @@ -113,16 +111,6 @@ export class PhoneAudioDiagnostics { this.once('warn', 'native_frame_rejected', `reason=${reason}`); } - socketUnavailable(readyState: number | undefined): void { - if (!this.active) return; - this.bufferedWhileDisconnected += 1; - this.once( - 'warn', - 'frame_buffered_socket_not_open', - `ready_state=${readyState ?? 'undefined'}`, - ); - } - socketConnecting(): void { this.once('info', 'websocket_connecting'); } @@ -150,10 +138,10 @@ export class PhoneAudioDiagnostics { this.once('info', 'backend_capture_started', `capture_id=${shortId(captureSessionId)}`); } - frameEnqueued(opusBytes: number): void { + frameSent(opusBytes: number): void { if (!this.active) return; - this.enqueuedFrames += 1; - this.once('info', 'first_frame_enqueued', `opus_bytes=${opusBytes}`); + this.sentFrames += 1; + this.once('info', 'first_frame_sent', `opus_bytes=${opusBytes}`); } packetAccepted(sequence: number): void { @@ -182,8 +170,7 @@ export class PhoneAudioDiagnostics { return [ `elapsed_ms=${Math.max(0, this.now() - this.startedAtMs)}`, `native_frames=${this.nativeFrames}`, - `buffered_while_disconnected=${this.bufferedWhileDisconnected}`, - `enqueued_frames=${this.enqueuedFrames}`, + `sent_frames=${this.sentFrames}`, `acked_packets=${this.ackedPackets}`, `last_audio_level=${this.lastAudioLevel.toFixed(3)}`, ].join(' '); diff --git a/backends/advanced/tests/test_spool_ack_contract.py b/backends/advanced/tests/test_spool_ack_contract.py deleted file mode 100644 index cb1cbe958..000000000 --- a/backends/advanced/tests/test_spool_ack_contract.py +++ /dev/null @@ -1,71 +0,0 @@ -"""The mobile spool's segment id is not the backend's audio session id. - -The phone spools every BLE packet to a file before offering it to the WebSocket, and -that file has an id. It used to travel as ``durable_session_id`` and come back as -``session_id`` — so a spool-file identity wore the name of the backend ``SessionId``, -which is a different thing with a different lifetime (one WebSocket connection, minted -for each authenticated audio-v2 connection as ``{client_id}-{uuid4}``). - -Nothing was mis-routed by it, because the receipt key is namespaced by user and client -and the value is only ever echoed back. That is exactly why it is worth pinning: the -names are the only thing keeping the two apart, and the wire is where they meet. - -These assert the shape both ends agree on, without a live socket. -""" - -from pathlib import Path - -import pytest -from google.protobuf.descriptor import FieldDescriptor - -from advanced_omi_backend.audio_contract.v2 import audio_pb2 - -SRC = Path(__file__).resolve().parents[1] / "src" / "advanced_omi_backend" -APP = Path(__file__).resolve().parents[3] / "app" / "src" - -AUDIO_STREAMER = APP / "hooks" / "useAudioStreamer.ts" -SPOOL = APP / "services" / "durableAudioSpool.ts" - - -def _code(path: Path) -> str: - """File text with comment lines removed, so prose about the old names is ignored.""" - - lines = [] - for line in path.read_text(encoding="utf-8").splitlines(): - stripped = line.strip() - if stripped.startswith(("#", "//", "*", "/*")): - continue - lines.append(line) - return "\n".join(lines) - - -def test_v2_ack_is_bound_to_the_backend_capture_and_exact_packet_sequence(): - """The V2 receipt cannot confuse a local spool file with a capture session.""" - - fields = audio_pb2.CapturePacketAccepted.DESCRIPTOR.fields_by_name - assert fields["binding"].message_type.full_name.endswith("CaptureBinding") - assert fields["sequence"].type == FieldDescriptor.TYPE_UINT64 - assert "spool_segment_id" not in fields - assert "session_id" not in fields - - -@pytest.mark.parametrize("path", [AUDIO_STREAMER, SPOOL]) -def test_the_app_uses_segment_naming_end_to_end(path): - if not path.exists(): # pragma: no cover - app tree absent in some checkouts - pytest.skip(f"{path} not present") - code = _code(path) - - assert "durable_session_id" not in code - assert "sessionId" not in code - - -def test_the_app_sends_and_matches_the_same_wire_fields_the_backend_reads(): - if not AUDIO_STREAMER.exists(): # pragma: no cover - pytest.skip("app tree not present") - app_code = _code(AUDIO_STREAMER) - # Spool identity stays local. The client maps a generated packet sequence back - # to its pending file and retires it only on CapturePacketAccepted. - assert "acceptedRef" in app_code - assert "onPacketAccepted" in app_code - assert "durableAudioSpool.acknowledge(packet)" in app_code - assert "spool_segment_id" not in app_code diff --git a/docs/backend/audio-interface-map.md b/docs/backend/audio-interface-map.md index 8cf24777c..ac6ca070a 100644 --- a/docs/backend/audio-interface-map.md +++ b/docs/backend/audio-interface-map.md @@ -14,7 +14,7 @@ test and a deployed trace both exist. | WEB-CAPTURE | Web Audio → backend | paired-header PCM | WebCodecs raw Opus packets | green source/build; browser E2E pending | `RecordingContext.test.tsx`; WebUI production build | | OMI-NEO | BLE → app/tray → backend | raw Opus via Wyoming | declared 60 ms Opus normalized to canonical frames | amber: source/tests green; physical retest pending | real 60 ms Opus decode; app and shared-client duration tests | | HAVPE | firmware → relay → backend | device-local JSONL/PCM → relay V2 adapter | generated V2 at backend boundary | green source; physical pending | PCM normalizer/raw-Opus round trip; typed button/playback adapter | -| RECOVERY | phone spool → backend | inferred bare packets | typed recovered packets | green app + ingress + persistence | typed packet ACK; `test_audio_durability.py` | +| RECOVERY | external recovery client → backend | inferred bare packets | typed recovered packets | backend supported; removed from Expo live path | `test_audio_durability.py` | | DURABLE-REDIS | ingress → persistence | string fields/magic end marker | `CaptureStreamEvent` binary | green V2 producer + persistence consumer | `test_audio_v2_streams.py`, `test_audio_durability.py` | | REALTIME-REDIS | ingress → ASR/wake/turns | wildcard string-field streams | typed `CanonicalPcmFrame` | green producer + all three consumers | backend streaming tests; wakeword consumer tests | | MONGO-AUDIO | persistence → claims | fixed Opus chunks | retained; stricter domain types | green deployed | live completed session + 1 canonical 1.1 s chunk | @@ -43,6 +43,12 @@ test and a deployed trace both exist. ### 2026-09-08 +- Removed the Expo durable spool/recovery state machine after a physical iPhone + trace caught recovery and user-stop competing for the same `captureStopped` + waiter. One button press now owns one live capture and one stop; frames flow + directly through that binding, with captured/sent/accepted counts retained in + the exported diagnostics. This deliberately drops offline/reconnect replay from + the app instead of layering another lifecycle guard around it. - Collapsed mobile capture to one explicit three-operation interface: start a named source, enqueue a source-tagged frame, and stop. Removed URL-carried credentials, optional phone/wearable inference, duplicate URL builders, caller-side socket @@ -51,9 +57,7 @@ test and a deployed trace both exist. OMI/Neo declares its native 60 ms packets; HAVPE remains explicit 20 ms. The backend decodes one declared packet and publishes one or three contiguous canonical 20 ms frames. -- Source-tagged spool segments prevent a queued phone packet from being replayed - through a wearable decoder, or vice versa. The app refuses non-Opus wearable - capture before installing the BLE listener. +- The app refuses non-Opus wearable capture before installing the BLE listener. - Protocol rejection now finalizes the technical stream with `failure` and status `failed`; the Mongo persistence worker preserves that result instead of replacing it with `complete`. Verification is source-level only until backend deployment and @@ -258,7 +262,7 @@ test and a deployed trace both exist. ## Remaining verification gates -- Record one physical iPhone capture/recovery/playback trace and one physical +- Record one physical iPhone capture/playback trace and one physical OMI/Neo or HAVPE capture trace through ingress, Redis, Mongo, inference, and action. - Restore or replace the exhausted OpenRouter allowance, then verify short summary, detailed summary, and memory extraction on one of the E2E-created Conversations. diff --git a/scripts/rainbow-testflight-handoff.sh b/scripts/rainbow-testflight-handoff.sh index ba0c2ea71..0270f1dd9 100755 --- a/scripts/rainbow-testflight-handoff.sh +++ b/scripts/rainbow-testflight-handoff.sh @@ -52,7 +52,7 @@ fi npm ci npm run typecheck -npm run test:durable-audio-spool +npm run test:phone-audio-diagnostics npm run check:theme npx --no-install expo config --type public --json >/dev/null npx --no-install expo-modules-autolinking verify --platform ios --verbose From 01b8bb82d7a8037dcd3a98f1e2c3ba353690b708 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:43:15 +0000 Subject: [PATCH 11/12] fix(audio): gate the phone WebSocket lifecycle --- .github/workflows/ios-testflight.yml | 46 ++++ .../controllers/audio_v2_controller.py | 2 +- .../test_audio_v2_websocket_entrypoint.py | 231 ++++++++++++++++++ 3 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_audio_v2_websocket_entrypoint.py diff --git a/.github/workflows/ios-testflight.yml b/.github/workflows/ios-testflight.yml index 4ab77bccf..174b3af6b 100644 --- a/.github/workflows/ios-testflight.yml +++ b/.github/workflows/ios-testflight.yml @@ -15,7 +15,53 @@ on: type: string jobs: + audio-v2-release-gate: + if: github.event_name == 'push' || inputs.build_id == '' + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: ./backend + + steps: + - name: Setup repo + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Setup uv + uses: astral-sh/setup-uv@v4 + with: + version: latest + + - name: Install Opus runtime + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libopus0 + + - name: Install backend test dependencies + run: uv sync --locked --group test + + - name: Verify Audio V2 phone path + run: >- + uv run --group test pytest + tests/test_audio_v2_websocket_entrypoint.py + tests/test_audio_protocol_v2.py + tests/test_audio_v2_ingress.py + tests/test_audio_v2_streams.py + tests/test_audio_persistence_lifecycle.py + tests/test_audio_durability.py + -q + build-and-submit: + needs: audio-v2-release-gate + if: >- + ${{ + always() && + (needs.audio-v2-release-gate.result == 'success' || + needs.audio-v2-release-gate.result == 'skipped') + }} runs-on: macos-26 timeout-minutes: 120 defaults: diff --git a/backend/src/backend/controllers/audio_v2_controller.py b/backend/src/backend/controllers/audio_v2_controller.py index aa20cf9c5..10f4b120e 100644 --- a/backend/src/backend/controllers/audio_v2_controller.py +++ b/backend/src/backend/controllers/audio_v2_controller.py @@ -466,7 +466,6 @@ async def handle_audio_v2_websocket(websocket: WebSocket) -> None: }, provenance=provenance, ) - active_binding = binding binding = audio_pb2.CaptureBinding( capture_session_id=audio_pb2.CaptureSessionId( value=client_state.stream_session_id @@ -476,6 +475,7 @@ async def handle_audio_v2_websocket(websocket: WebSocket) -> None: ), capture_epoch=client_state.capture_epoch, ) + active_binding = binding v2_streams = await AudioV2Streams.open( producer.redis_client, event=audio_pb2.CaptureStreamEvent( diff --git a/backend/tests/test_audio_v2_websocket_entrypoint.py b/backend/tests/test_audio_v2_websocket_entrypoint.py new file mode 100644 index 000000000..9d496f369 --- /dev/null +++ b/backend/tests/test_audio_v2_websocket_entrypoint.py @@ -0,0 +1,231 @@ +import asyncio +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from fakeredis import aioredis as fake_aioredis +from google.protobuf import duration_pb2, json_format, timestamp_pb2 +from opuslib import Encoder + +from backend.audio_contract.v2 import audio_pb2 +from backend.audio_contract.v2.codec import ( + serialize_client_control_json, + serialize_media_envelope, +) +from backend.controllers import audio_v2_controller +from backend.routers.modules.websocket_routes import audio_v2_endpoint +from backend.services.audio_stream.v2_streams import parse_stream_event + +pytestmark = pytest.mark.unit + + +def _timestamp() -> timestamp_pb2.Timestamp: + value = timestamp_pb2.Timestamp() + value.FromDatetime(datetime(2026, 9, 8, 18, 27, 29, tzinfo=timezone.utc)) + return value + + +def _control(**event) -> str: + return serialize_client_control_json( + audio_pb2.ClientControl( + event_id=audio_pb2.EventId(value=f"event-{next(_EVENT_IDS)}"), + sent_at=_timestamp(), + **event, + ) + ) + + +def _binding() -> audio_pb2.CaptureBinding: + return audio_pb2.CaptureBinding( + capture_session_id=audio_pb2.CaptureSessionId(value="capture-1"), + voice_session_id=audio_pb2.VoiceSessionId(value="voice-1"), + capture_epoch=1, + ) + + +def _parse_server_control(raw: str) -> audio_pb2.ServerControl: + control = audio_pb2.ServerControl() + json_format.Parse(raw, control) + return control + + +_EVENT_IDS = iter(range(1, 20)) + + +class PhoneWebSocket: + def __init__(self, messages): + self.scope = {"subprotocols": ["chronicle.audio.v2"]} + self._messages = iter(messages) + self.accepted_subprotocol = None + self.sent_text = [] + self.closed = None + + async def accept(self, *, subprotocol): + self.accepted_subprotocol = subprotocol + + async def receive_text(self): + return _control( + hello=audio_pb2.ClientHello( + bearer_token="phone-token", + source_id=audio_pb2.CaptureSourceId(value="a421c9-phone"), + device_kind=audio_pb2.DEVICE_KIND_IOS_PHONE, + display_name="phone", + ) + ) + + async def receive(self): + return next(self._messages, {"type": "websocket.disconnect"}) + + async def send_text(self, value): + self.sent_text.append(value) + + async def send_bytes(self, _value): + raise AssertionError("the capture path must not send binary downlink") + + async def close(self, *, code, reason): + self.closed = (code, reason) + + +@pytest.mark.asyncio +async def test_registered_audio_websocket_accepts_phone_frame_and_stops(monkeypatch): + """The server-testable phone path must survive hello through durable ingress.""" + + audio_spec = audio_pb2.AudioSpec( + codec=audio_pb2.AUDIO_CODEC_OPUS, + sample_rate_hz=16_000, + channel_count=1, + frame_duration=duration_pb2.Duration(nanos=20_000_000), + bitrate_bps=24_000, + ) + start = _control( + start_capture=audio_pb2.StartCapture( + capture_epoch=1, + processing_profile=audio_pb2.PROCESSING_PROFILE_DUPLEX_AEC, + data_purpose=audio_pb2.DATA_PURPOSE_NORMAL_CAPTURE, + delivery_class=audio_pb2.DELIVERY_CLASS_LIVE, + audio_spec=audio_spec, + capabilities=audio_pb2.CaptureCapabilities( + duplex_mode=audio_pb2.DUPLEX_MODE_FULL, + input_route=audio_pb2.INPUT_ROUTE_BUILT_IN_MIC, + output_route=audio_pb2.OUTPUT_ROUTE_SPEAKERPHONE, + native_sample_rate_hz=48_000, + acoustic_echo_cancellation=audio_pb2.EffectStatus( + requested=True, available=True, enabled=True + ), + noise_suppression=audio_pb2.EffectStatus( + requested=True, available=True, enabled=True + ), + ), + ) + ) + opus_silence = Encoder(16_000, 1, "audio").encode(bytes(640), 320) + media = serialize_media_envelope( + audio_pb2.MediaEnvelope( + capture=audio_pb2.CaptureMediaPacket( + binding=_binding(), + sequence=0, + captured_at=_timestamp(), + delivery_class=audio_pb2.DELIVERY_CLASS_LIVE, + opus_payload=opus_silence, + ) + ) + ) + stop = _control( + stop_capture=audio_pb2.StopCapture( + binding=_binding(), reason=audio_pb2.STOP_REASON_USER_REQUESTED + ) + ) + websocket = PhoneWebSocket( + [ + {"type": "websocket.receive", "text": start}, + {"type": "websocket.receive", "bytes": media}, + {"type": "websocket.receive", "text": stop}, + ] + ) + + redis_client = fake_aioredis.FakeRedis() + state = SimpleNamespace( + stream_session_id=None, + voice_session_id=None, + capture_epoch=0, + data_purpose=None, + socket_id=None, + ) + producer = SimpleNamespace( + redis_client=redis_client, + update_session_job_ids=AsyncMock(), + ) + + async def initialize_capture_session(**kwargs): + initialized_state = kwargs["client_state"] + initialized_state.stream_session_id = "capture-1" + initialized_state.voice_session_id = "voice-1" + initialized_state.capture_epoch = 1 + initialized_state.data_purpose = "normal_capture" + + async def finalize_capture_session(**kwargs): + kwargs["client_state"].stream_session_id = None + + monkeypatch.setattr( + audio_v2_controller, + "websocket_auth", + AsyncMock( + return_value=( + SimpleNamespace( + id="69b80e5894aa9ec334a421c9", + user_id="69b80e5894aa9ec334a421c9", + email="phone@example.test", + ), + None, + ) + ), + ) + monkeypatch.setattr( + audio_v2_controller, "create_client_state", AsyncMock(return_value=state) + ) + monkeypatch.setattr( + audio_v2_controller, "get_audio_stream_producer", lambda: producer + ) + monkeypatch.setattr( + audio_v2_controller, "initialize_capture_session", initialize_capture_session + ) + monkeypatch.setattr( + audio_v2_controller, "finalize_capture_session", finalize_capture_session + ) + monkeypatch.setattr( + audio_v2_controller, + "start_streaming_jobs", + lambda **_kwargs: { + "speech_detection": "speech-job-1", + "audio_persistence": "persistence-job-1", + }, + ) + monkeypatch.setattr( + audio_v2_controller, "cleanup_client_state", AsyncMock(return_value=True) + ) + + await asyncio.wait_for(audio_v2_endpoint(websocket), timeout=2) + + controls = [_parse_server_control(raw) for raw in websocket.sent_text] + assert [control.WhichOneof("event") for control in controls] == [ + "hello", + "capture_started", + "capture_packet_accepted", + "capture_stopped", + ] + assert controls[1].capture_started.binding == _binding() + assert controls[2].capture_packet_accepted.sequence == 0 + assert websocket.accepted_subprotocol == "chronicle.audio.v2" + assert websocket.closed is None + + durable_entries = await redis_client.xrange("audio:v2:durable:capture-1") + durable_events = [ + parse_stream_event(fields) for _entry_id, fields in durable_entries + ] + assert [event.WhichOneof("event") for event in durable_events] == [ + "opened", + "frame", + "ended", + ] + assert durable_events[1].frame.pcm_s16le == bytes(640) From 43337a94af6f8006af5a74bc35374b7f1e7d9a6a Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Wed, 9 Sep 2026 01:13:00 +0000 Subject: [PATCH 12/12] fix(app): stop capture before closing and break push token recursion --- app/scripts/test-phone-audio-diagnostics.cjs | 10 ++++++ app/scripts/test-phone-audio-streaming.cjs | 36 ++++++++++++++++++++ app/scripts/test-push-notifications.cjs | 11 ++++-- app/src/hooks/useAudioStreamer.ts | 8 ++--- app/src/services/pushNotifications.ts | 11 +++--- 5 files changed, 66 insertions(+), 10 deletions(-) diff --git a/app/scripts/test-phone-audio-diagnostics.cjs b/app/scripts/test-phone-audio-diagnostics.cjs index a7139cd80..6bfe86eb0 100644 --- a/app/scripts/test-phone-audio-diagnostics.cjs +++ b/app/scripts/test-phone-audio-diagnostics.cjs @@ -28,6 +28,7 @@ const writes = []; const socketSourcePath = path.join(__dirname, '../src/protocol/audioV2Socket.ts'); const serverControls = { hello: { event: { case: 'hello', value: {} } }, + captureStopped: { event: { case: 'captureStopped', value: {} } }, captureStarted: { event: { case: 'captureStarted', @@ -127,6 +128,15 @@ async function declaredUplinkDuration(frameDurationMs) { }); transport.receive('captureStarted'); await starting; + let stopped = false; + const stopping = socket.stopCapture().then(() => { stopped = true; }); + await Promise.resolve(); + assert.equal(stopped, false, 'stop must await the server acknowledgement'); + assert.equal(transport.sent.at(-1).event.case, 'stopCapture'); + transport.receive('captureStopped'); + await stopping; + assert.equal(socket.activeBinding, null, 'acknowledged stop must release the capture'); + socket.close(); return transport.sent.map(control => ( control.event.value.audioSpec ?? control.event.value.supportedUplink?.[0] )).filter(Boolean).map(spec => spec.frameDuration.nanos); diff --git a/app/scripts/test-phone-audio-streaming.cjs b/app/scripts/test-phone-audio-streaming.cjs index 7388cd058..23b911adb 100644 --- a/app/scripts/test-phone-audio-streaming.cjs +++ b/app/scripts/test-phone-audio-streaming.cjs @@ -38,6 +38,7 @@ const socketFrameDurations = []; const diagnostics = []; let backendStops = 0; let socketCloses = 0; +let duringBackendStop = () => {}; class MockAudioV2Socket { constructor(options) { @@ -66,6 +67,7 @@ class MockAudioV2Socket { async stopCapture() { backendStops += 1; + await duringBackendStop(); this.activeBinding = null; } @@ -165,7 +167,19 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { assert.deepEqual(Array.from(sentPackets[0].opus), [1, 2, 3]); assert.deepEqual(diagnostics, [['sent', 3], ['accepted', 0]]); + duringBackendStop = async () => { + const before = sentPackets.length; + streamer.sendFrame('phone', { + captureEpoch: 7, + capturedAtMs: 1_780_000_000_040, + monotonicTimestampMs: now + 40, + opus: new Uint8Array([4]), + }); + assert.equal(sentPackets.length, before, 'queued native callbacks must not send after stop'); + assert.equal(nativeStops, 1, 'microphone must stop before waiting for the backend'); + }; await streamer.stopStreaming(); + duringBackendStop = () => {}; assert.equal(backendStops, 1, 'the capture has one stop owner'); assert.equal(nativeStops, 1, 'stopping the stream also stops the native phone session'); assert.equal(socketCloses, 1); @@ -180,6 +194,28 @@ const { useAudioStreamer } = loadTypeScript(sourcePath, { assert.equal(beginCaptureCalls[1].deliveryClass, DeliveryClass.LIVE); await wearableStreamer.stopStreaming(); + for (const failureAt of ['native', 'backend']) { + const failingStreamer = useAudioStreamer(); + const expected = new Error(`${failureAt} stop failed`); + await failingStreamer.startStreaming('wss://chronicle.invalid/ws/audio', { + kind: 'phone', + ...phoneVoice, + stopCapture: async () => { + if (failureAt === 'native') throw expected; + }, + }); + const closesBefore = socketCloses; + duringBackendStop = async () => { throw expected; }; + await assert.rejects(failingStreamer.stopStreaming(), error => error === expected); + assert.equal(socketCloses, closesBefore + 1, 'stop failure must still close the transport'); + const packetsBefore = sentPackets.length; + failingStreamer.sendFrame('phone', { + captureEpoch: 7, capturedAtMs: 1_780_000_000_060, + monotonicTimestampMs: now + 60, opus: new Uint8Array([5]), + }); + assert.equal(sentPackets.length, packetsBefore, 'failed stop must leave capture inactive'); + } + console.log('phone audio streaming tests passed'); })().catch(error => { console.error(error); diff --git a/app/scripts/test-push-notifications.cjs b/app/scripts/test-push-notifications.cjs index 2b2520b39..224eb7edb 100644 --- a/app/scripts/test-push-notifications.cjs +++ b/app/scripts/test-push-notifications.cjs @@ -31,13 +31,19 @@ const requests = []; const opened = []; const alerts = []; let pushTokenListener = null; +let nativeTokenFetches = 0; const notifications = { PermissionStatus: { DENIED: 'denied' }, AndroidImportance: { HIGH: 4, DEFAULT: 3 }, getPermissionsAsync: async () => permission, requestPermissionsAsync: async () => permission, - getExpoPushTokenAsync: async ({ projectId }) => { + getExpoPushTokenAsync: async ({ projectId, devicePushToken }) => { assert.equal(projectId, 'project-one'); + // Expo requests a native token if none is supplied; iOS emits the listener + // again when that request completes. Bound the buggy loop in this fixture. + if (!devicePushToken && pushTokenListener && ++nativeTokenFetches < 5) { + pushTokenListener({ type: 'ios', data: 'rotated' }); + } return { data: 'ExpoPushToken[abcdefghijklmnopqrstuvwxyz]' }; }, setNotificationHandler() {}, @@ -100,8 +106,9 @@ const push = loadTypeScript(sourcePath, { await push.refreshPushRegistration('wss://chronicle/ws/audio'); assert.equal(requests.length, 2, 'authenticated launch refreshes the registration'); const stopTokenListener = push.listenForPushTokenChanges('wss://chronicle/ws/audio'); - pushTokenListener({ type: 'expo', data: 'rotated' }); + pushTokenListener({ type: 'ios', data: 'rotated' }); await new Promise(resolve => setImmediate(resolve)); + assert.equal(nativeTokenFetches, 0, 'rotation must use the supplied native token without requesting another'); assert.equal(requests.length, 3, 'native token changes refresh the Expo registration'); stopTokenListener(); diff --git a/app/src/hooks/useAudioStreamer.ts b/app/src/hooks/useAudioStreamer.ts index a76f7a03d..0f136f766 100644 --- a/app/src/hooks/useAudioStreamer.ts +++ b/app/src/hooks/useAudioStreamer.ts @@ -142,6 +142,9 @@ export const useAudioStreamer = (): UseAudioStreamer => { heartbeatRef.current = null; const socket = socketRef.current; try { + if (sourceRef.current?.kind === 'phone') { + await sourceRef.current.stopCapture(); + } await socket?.stopCapture(); } finally { socket?.close(); @@ -149,9 +152,6 @@ export const useAudioStreamer = (): UseAudioStreamer => { playbackSubscriptionRef.current?.remove(); playbackSubscriptionRef.current = null; playbackRef.current = null; - if (sourceRef.current?.kind === 'phone') { - await sourceRef.current.stopCapture(); - } sourceRef.current = null; setIsStreaming(false); setIsConnecting(false); @@ -297,7 +297,7 @@ export const useAudioStreamer = (): UseAudioStreamer => { source: AudioStreamSource['kind'], frame: CapturedOpusFrame, ) => { - if (!frame.opus.length) return; + if (stoppedRef.current || !frame.opus.length) return; const activeSource = sourceRef.current; const socket = socketRef.current; if (!socket?.activeBinding || activeSource?.kind !== source) return; diff --git a/app/src/services/pushNotifications.ts b/app/src/services/pushNotifications.ts index c35448e38..845f900eb 100644 --- a/app/src/services/pushNotifications.ts +++ b/app/src/services/pushNotifications.ts @@ -45,9 +45,12 @@ export const configureNotificationPresentation = async (): Promise => { } }; -const registerToken = async (backendUrl: string): Promise => { +const registerToken = async ( + backendUrl: string, + devicePushToken?: Notifications.DevicePushToken, +): Promise => { if (!projectId) throw new Error('This build has no EAS project ID.'); - const token = await Notifications.getExpoPushTokenAsync({ projectId }); + const token = await Notifications.getExpoPushTokenAsync({ projectId, devicePushToken }); const installationId = await getOrCreateInstallationId(); const response = await fetchAuthed( `${deriveBaseUrl(backendUrl)}/api/notifications/devices/${encodeURIComponent(installationId)}`, @@ -123,8 +126,8 @@ export const startNotificationTapHandling = async (): Promise<() => void> => { /** Native-token rotation means the Expo token must be fetched and registered again. */ export const listenForPushTokenChanges = (backendUrl: string): (() => void) => { if (Platform.OS === 'web') return () => {}; - const subscription = Notifications.addPushTokenListener(() => { - void refreshPushRegistration(backendUrl).catch(error => { + const subscription = Notifications.addPushTokenListener(devicePushToken => { + void registerToken(backendUrl, devicePushToken).catch(error => { console.warn('[Notifications] token refresh failed:', error); }); });