{"version":3,"sources":["../../src/voice/realtime_message_metrics.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { ReadableStream } from 'node:stream/web';\nimport { describe, expect, it, vi } from 'vitest';\nimport { type JobContext, runWithJobContextAsync } from '../job.js';\nimport { ChatContext, type FunctionCall } from '../llm/chat_context.js';\nimport {\n type GenerationCreatedEvent,\n type MessageGeneration,\n type RealtimeCapabilities,\n RealtimeModel,\n RealtimeSession,\n} from '../llm/realtime.js';\nimport { type ToolChoice, ToolContext } from '../llm/tool_context.js';\nimport { initializeLogger, log } from '../log.js';\nimport type { RealtimeModelMetrics } from '../metrics/base.js';\nimport { FakeSTT } from '../stt/testing/fake_stt.js';\nimport { Agent } from './agent.js';\nimport { AgentSession } from './agent_session.js';\nimport { AgentSessionEventTypes, type ConversationItemAddedEvent } from './events.js';\n\ninitializeLogger({ pretty: false, level: 'silent' });\n\nfunction emptyStream(): ReadableStream {\n return new ReadableStream({\n start(controller) {\n controller.close();\n },\n });\n}\n\nfunction oneItemStream(item: T): ReadableStream {\n return new ReadableStream({\n start(controller) {\n controller.enqueue(item);\n controller.close();\n },\n });\n}\n\nclass FakeRealtimeSession extends RealtimeSession {\n closingMetrics?: RealtimeModelMetrics;\n private _chatCtx = ChatContext.empty();\n private _tools = ToolContext.empty();\n\n get chatCtx(): ChatContext {\n return this._chatCtx;\n }\n\n get tools(): ToolContext {\n return this._tools;\n }\n\n async updateInstructions(_instructions: string): Promise {}\n\n async updateChatCtx(chatCtx: ChatContext): Promise {\n this._chatCtx = chatCtx.copy();\n }\n\n async updateTools(tools: ToolContext): Promise {\n this._tools = tools.copy();\n }\n\n updateOptions(_options: { toolChoice?: ToolChoice | null }): void {}\n\n pushAudio(_frame: AudioFrame): void {}\n\n async generateReply(): Promise {\n const message: MessageGeneration = {\n messageId: 'message-id',\n textStream: oneItemStream('Hello'),\n audioStream: emptyStream(),\n modalities: Promise.resolve(['text']),\n };\n\n return {\n messageStream: oneItemStream(message),\n functionStream: emptyStream(),\n userInitiated: true,\n responseId: 'provider-response-id',\n };\n }\n\n async commitAudio(): Promise {}\n\n async clearAudio(): Promise {}\n\n async interrupt(): Promise {}\n\n async truncate(): Promise {}\n\n async close(): Promise {\n if (this.closingMetrics) this.emit('metrics_collected', this.closingMetrics);\n await super.close();\n }\n}\n\nclass FakeRealtimeModel extends RealtimeModel {\n readonly activeSession: FakeRealtimeSession;\n\n constructor(turnDetection = false) {\n const capabilities: RealtimeCapabilities = {\n messageTruncation: false,\n turnDetection,\n userTranscription: false,\n autoToolReplyGeneration: false,\n audioOutput: false,\n manualFunctionCalls: false,\n midSessionChatCtxUpdate: true,\n midSessionInstructionsUpdate: true,\n midSessionToolsUpdate: true,\n perResponseToolChoice: false,\n };\n super(capabilities);\n this.activeSession = new FakeRealtimeSession(this);\n }\n\n get model(): string {\n return 'fake-realtime';\n }\n\n session(): RealtimeSession {\n return this.activeSession;\n }\n\n async close(): Promise {}\n}\n\nconst REALTIME_REDACTION_WARNING =\n 'RealtimeModel user turns lack complete speech timestamps, so audio redaction may be inaccurate; disable audio recording to prevent redaction leak.';\n\nfunction fakeJobContext(enableRedaction: boolean): JobContext {\n return {\n job: { enableRecording: true, enableRedaction },\n simulationContext: () => undefined,\n initRecording: vi.fn(async () => {}),\n _primaryAgentSession: undefined,\n sessionDirectory: undefined,\n } as unknown as JobContext;\n}\n\nasync function runRealtimeSession({\n projectRedaction,\n sessionRedaction,\n stt,\n serverTurnDetection = true,\n}: {\n projectRedaction: boolean;\n sessionRedaction: boolean;\n stt?: FakeSTT;\n serverTurnDetection?: boolean;\n}): Promise {\n const session = new AgentSession({\n llm: new FakeRealtimeModel(serverTurnDetection),\n stt,\n vad: null,\n turnHandling: { turnDetection: null },\n });\n\n await runWithJobContextAsync(fakeJobContext(projectRedaction), async () => {\n await session.start({\n agent: new Agent({ instructions: 'test' }),\n record: sessionRedaction ? { redaction: true } : true,\n });\n await session.close();\n });\n}\n\ndescribe('Realtime message metrics', () => {\n it('collects usage reported while the provider session closes', async () => {\n const model = new FakeRealtimeModel();\n const session = new AgentSession({\n llm: model,\n vad: null,\n turnHandling: { turnDetection: null },\n });\n await session.start({ agent: new Agent({ instructions: 'test' }) });\n model.activeSession.closingMetrics = {\n type: 'realtime_model_metrics',\n label: 'fake',\n requestId: 'sess_1',\n timestamp: Date.now(),\n durationMs: 0,\n ttftMs: -1,\n cancelled: false,\n tokensPerSecond: 0,\n sessionDurationMs: 12_500,\n inputTokens: 100,\n outputTokens: 50,\n totalTokens: 150,\n inputTokenDetails: { audioTokens: 90, textTokens: 10, imageTokens: 0, cachedTokens: 0 },\n outputTokenDetails: { audioTokens: 40, textTokens: 10, imageTokens: 0 },\n metadata: { modelName: 'fake-live', modelProvider: 'fake.provider' },\n };\n await session.close();\n expect(session.usage.modelUsage).toContainEqual(\n expect.objectContaining({\n provider: 'fake.provider',\n inputTokens: 100,\n inputAudioTokens: 90,\n outputTokens: 50,\n outputAudioTokens: 40,\n sessionDurationMs: 12_500,\n }),\n );\n expect(model.activeSession.listenerCount('metrics_collected')).toBe(0);\n });\n\n it('makes realtime response IDs available on assistant messages', async () => {\n const llm = new FakeRealtimeModel();\n const session = new AgentSession({ llm, vad: null, turnHandling: { turnDetection: null } });\n const conversationEvents: ConversationItemAddedEvent[] = [];\n\n session.on(AgentSessionEventTypes.ConversationItemAdded, (ev) => {\n conversationEvents.push(ev);\n });\n\n await session.start({ agent: new Agent({ instructions: 'test' }) });\n try {\n await session.generateReply().waitForPlayout();\n } finally {\n await session.close();\n }\n\n const assistantMessages = conversationEvents\n .map((event) => event.item)\n .filter((item) => item.type === 'message' && item.role === 'assistant');\n\n expect(assistantMessages).toHaveLength(1);\n expect(assistantMessages[0]?.metrics.providerRequestIds).toEqual(['provider-response-id']);\n });\n});\n\ndescribe('Realtime audio redaction warning', () => {\n it.each([\n { source: 'project', projectRedaction: true, sessionRedaction: false },\n { source: 'session', projectRedaction: false, sessionRedaction: true },\n ])('warns when $source redaction is enabled', async (options) => {\n const warn = vi.spyOn(log(), 'warn').mockImplementation(() => undefined);\n\n try {\n await runRealtimeSession(options);\n expect(warn).toHaveBeenCalledWith(REALTIME_REDACTION_WARNING);\n } finally {\n warn.mockRestore();\n }\n });\n\n it('warns when STT is configured', async () => {\n const warn = vi.spyOn(log(), 'warn').mockImplementation(() => undefined);\n\n try {\n await runRealtimeSession({\n projectRedaction: true,\n sessionRedaction: false,\n stt: new FakeSTT(),\n });\n expect(warn).toHaveBeenCalledWith(REALTIME_REDACTION_WARNING);\n } finally {\n warn.mockRestore();\n }\n });\n\n it('warns when server-side turn detection is disabled', async () => {\n const warn = vi.spyOn(log(), 'warn').mockImplementation(() => undefined);\n\n try {\n await runRealtimeSession({\n projectRedaction: true,\n sessionRedaction: false,\n serverTurnDetection: false,\n });\n expect(warn).toHaveBeenCalledWith(REALTIME_REDACTION_WARNING);\n } finally {\n warn.mockRestore();\n }\n });\n\n it('does not warn when redaction is disabled', async () => {\n const warn = vi.spyOn(log(), 'warn').mockImplementation(() => undefined);\n\n try {\n await runRealtimeSession({\n projectRedaction: false,\n sessionRedaction: false,\n });\n expect(warn).not.toHaveBeenCalledWith(REALTIME_REDACTION_WARNING);\n } finally {\n warn.mockRestore();\n }\n });\n});\n"],"mappings":";AAIA,iBAA+B;AAC/B,oBAAyC;AACzC,iBAAwD;AACxD,0BAA+C;AAC/C,sBAMO;AACP,0BAA6C;AAC7C,iBAAsC;AAEtC,sBAAwB;AACxB,mBAAsB;AACtB,2BAA6B;AAC7B,oBAAwE;AAAA,IAExE,6BAAiB,EAAE,QAAQ,OAAO,OAAO,SAAS,CAAC;AAEnD,SAAS,cAAoC;AAC3C,SAAO,IAAI,0BAAkB;AAAA,IAC3B,MAAM,YAAY;AAChB,iBAAW,MAAM;AAAA,IACnB;AAAA,EACF,CAAC;AACH;AAEA,SAAS,cAAiB,MAA4B;AACpD,SAAO,IAAI,0BAAkB;AAAA,IAC3B,MAAM,YAAY;AAChB,iBAAW,QAAQ,IAAI;AACvB,iBAAW,MAAM;AAAA,IACnB;AAAA,EACF,CAAC;AACH;AAEA,MAAM,4BAA4B,gCAAgB;AAAA,EAChD;AAAA,EACQ,WAAW,gCAAY,MAAM;AAAA,EAC7B,SAAS,gCAAY,MAAM;AAAA,EAEnC,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,QAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,mBAAmB,eAAsC;AAAA,EAAC;AAAA,EAEhE,MAAM,cAAc,SAAqC;AACvD,SAAK,WAAW,QAAQ,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,YAAY,OAAmC;AACnD,SAAK,SAAS,MAAM,KAAK;AAAA,EAC3B;AAAA,EAEA,cAAc,UAAoD;AAAA,EAAC;AAAA,EAEnE,UAAU,QAA0B;AAAA,EAAC;AAAA,EAErC,MAAM,gBAAiD;AACrD,UAAM,UAA6B;AAAA,MACjC,WAAW;AAAA,MACX,YAAY,cAAc,OAAO;AAAA,MACjC,aAAa,YAAY;AAAA,MACzB,YAAY,QAAQ,QAAQ,CAAC,MAAM,CAAC;AAAA,IACtC;AAEA,WAAO;AAAA,MACL,eAAe,cAAc,OAAO;AAAA,MACpC,gBAAgB,YAA0B;AAAA,MAC1C,eAAe;AAAA,MACf,YAAY;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,cAA6B;AAAA,EAAC;AAAA,EAEpC,MAAM,aAA4B;AAAA,EAAC;AAAA,EAEnC,MAAM,YAA2B;AAAA,EAAC;AAAA,EAElC,MAAM,WAA0B;AAAA,EAAC;AAAA,EAEjC,MAAM,QAAuB;AAC3B,QAAI,KAAK,eAAgB,MAAK,KAAK,qBAAqB,KAAK,cAAc;AAC3E,UAAM,MAAM,MAAM;AAAA,EACpB;AACF;AAEA,MAAM,0BAA0B,8BAAc;AAAA,EACnC;AAAA,EAET,YAAY,gBAAgB,OAAO;AACjC,UAAM,eAAqC;AAAA,MACzC,mBAAmB;AAAA,MACnB;AAAA,MACA,mBAAmB;AAAA,MACnB,yBAAyB;AAAA,MACzB,aAAa;AAAA,MACb,qBAAqB;AAAA,MACrB,yBAAyB;AAAA,MACzB,8BAA8B;AAAA,MAC9B,uBAAuB;AAAA,MACvB,uBAAuB;AAAA,IACzB;AACA,UAAM,YAAY;AAClB,SAAK,gBAAgB,IAAI,oBAAoB,IAAI;AAAA,EACnD;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA,EAEA,UAA2B;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAuB;AAAA,EAAC;AAChC;AAEA,MAAM,6BACJ;AAEF,SAAS,eAAe,iBAAsC;AAC5D,SAAO;AAAA,IACL,KAAK,EAAE,iBAAiB,MAAM,gBAAgB;AAAA,IAC9C,mBAAmB,MAAM;AAAA,IACzB,eAAe,iBAAG,GAAG,YAAY;AAAA,IAAC,CAAC;AAAA,IACnC,sBAAsB;AAAA,IACtB,kBAAkB;AAAA,EACpB;AACF;AAEA,eAAe,mBAAmB;AAAA,EAChC;AAAA,EACA;AAAA,EACA;AAAA,EACA,sBAAsB;AACxB,GAKkB;AAChB,QAAM,UAAU,IAAI,kCAAa;AAAA,IAC/B,KAAK,IAAI,kBAAkB,mBAAmB;AAAA,IAC9C;AAAA,IACA,KAAK;AAAA,IACL,cAAc,EAAE,eAAe,KAAK;AAAA,EACtC,CAAC;AAED,YAAM,mCAAuB,eAAe,gBAAgB,GAAG,YAAY;AACzE,UAAM,QAAQ,MAAM;AAAA,MAClB,OAAO,IAAI,mBAAM,EAAE,cAAc,OAAO,CAAC;AAAA,MACzC,QAAQ,mBAAmB,EAAE,WAAW,KAAK,IAAI;AAAA,IACnD,CAAC;AACD,UAAM,QAAQ,MAAM;AAAA,EACtB,CAAC;AACH;AAAA,IAEA,wBAAS,4BAA4B,MAAM;AACzC,wBAAG,6DAA6D,YAAY;AAC1E,UAAM,QAAQ,IAAI,kBAAkB;AACpC,UAAM,UAAU,IAAI,kCAAa;AAAA,MAC/B,KAAK;AAAA,MACL,KAAK;AAAA,MACL,cAAc,EAAE,eAAe,KAAK;AAAA,IACtC,CAAC;AACD,UAAM,QAAQ,MAAM,EAAE,OAAO,IAAI,mBAAM,EAAE,cAAc,OAAO,CAAC,EAAE,CAAC;AAClE,UAAM,cAAc,iBAAiB;AAAA,MACnC,MAAM;AAAA,MACN,OAAO;AAAA,MACP,WAAW;AAAA,MACX,WAAW,KAAK,IAAI;AAAA,MACpB,YAAY;AAAA,MACZ,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,iBAAiB;AAAA,MACjB,mBAAmB;AAAA,MACnB,aAAa;AAAA,MACb,cAAc;AAAA,MACd,aAAa;AAAA,MACb,mBAAmB,EAAE,aAAa,IAAI,YAAY,IAAI,aAAa,GAAG,cAAc,EAAE;AAAA,MACtF,oBAAoB,EAAE,aAAa,IAAI,YAAY,IAAI,aAAa,EAAE;AAAA,MACtE,UAAU,EAAE,WAAW,aAAa,eAAe,gBAAgB;AAAA,IACrE;AACA,UAAM,QAAQ,MAAM;AACpB,8BAAO,QAAQ,MAAM,UAAU,EAAE;AAAA,MAC/B,qBAAO,iBAAiB;AAAA,QACtB,UAAU;AAAA,QACV,aAAa;AAAA,QACb,kBAAkB;AAAA,QAClB,cAAc;AAAA,QACd,mBAAmB;AAAA,QACnB,mBAAmB;AAAA,MACrB,CAAC;AAAA,IACH;AACA,8BAAO,MAAM,cAAc,cAAc,mBAAmB,CAAC,EAAE,KAAK,CAAC;AAAA,EACvE,CAAC;AAED,wBAAG,+DAA+D,YAAY;AAlNhF;AAmNI,UAAM,MAAM,IAAI,kBAAkB;AAClC,UAAM,UAAU,IAAI,kCAAa,EAAE,KAAK,KAAK,MAAM,cAAc,EAAE,eAAe,KAAK,EAAE,CAAC;AAC1F,UAAM,qBAAmD,CAAC;AAE1D,YAAQ,GAAG,qCAAuB,uBAAuB,CAAC,OAAO;AAC/D,yBAAmB,KAAK,EAAE;AAAA,IAC5B,CAAC;AAED,UAAM,QAAQ,MAAM,EAAE,OAAO,IAAI,mBAAM,EAAE,cAAc,OAAO,CAAC,EAAE,CAAC;AAClE,QAAI;AACF,YAAM,QAAQ,cAAc,EAAE,eAAe;AAAA,IAC/C,UAAE;AACA,YAAM,QAAQ,MAAM;AAAA,IACtB;AAEA,UAAM,oBAAoB,mBACvB,IAAI,CAAC,UAAU,MAAM,IAAI,EACzB,OAAO,CAAC,SAAS,KAAK,SAAS,aAAa,KAAK,SAAS,WAAW;AAExE,8BAAO,iBAAiB,EAAE,aAAa,CAAC;AACxC,+BAAO,uBAAkB,CAAC,MAAnB,mBAAsB,QAAQ,kBAAkB,EAAE,QAAQ,CAAC,sBAAsB,CAAC;AAAA,EAC3F,CAAC;AACH,CAAC;AAAA,IAED,wBAAS,oCAAoC,MAAM;AACjD,mBAAG,KAAK;AAAA,IACN,EAAE,QAAQ,WAAW,kBAAkB,MAAM,kBAAkB,MAAM;AAAA,IACrE,EAAE,QAAQ,WAAW,kBAAkB,OAAO,kBAAkB,KAAK;AAAA,EACvE,CAAC,EAAE,2CAA2C,OAAO,YAAY;AAC/D,UAAM,OAAO,iBAAG,UAAM,gBAAI,GAAG,MAAM,EAAE,mBAAmB,MAAM,MAAS;AAEvE,QAAI;AACF,YAAM,mBAAmB,OAAO;AAChC,gCAAO,IAAI,EAAE,qBAAqB,0BAA0B;AAAA,IAC9D,UAAE;AACA,WAAK,YAAY;AAAA,IACnB;AAAA,EACF,CAAC;AAED,wBAAG,gCAAgC,YAAY;AAC7C,UAAM,OAAO,iBAAG,UAAM,gBAAI,GAAG,MAAM,EAAE,mBAAmB,MAAM,MAAS;AAEvE,QAAI;AACF,YAAM,mBAAmB;AAAA,QACvB,kBAAkB;AAAA,QAClB,kBAAkB;AAAA,QAClB,KAAK,IAAI,wBAAQ;AAAA,MACnB,CAAC;AACD,gCAAO,IAAI,EAAE,qBAAqB,0BAA0B;AAAA,IAC9D,UAAE;AACA,WAAK,YAAY;AAAA,IACnB;AAAA,EACF,CAAC;AAED,wBAAG,qDAAqD,YAAY;AAClE,UAAM,OAAO,iBAAG,UAAM,gBAAI,GAAG,MAAM,EAAE,mBAAmB,MAAM,MAAS;AAEvE,QAAI;AACF,YAAM,mBAAmB;AAAA,QACvB,kBAAkB;AAAA,QAClB,kBAAkB;AAAA,QAClB,qBAAqB;AAAA,MACvB,CAAC;AACD,gCAAO,IAAI,EAAE,qBAAqB,0BAA0B;AAAA,IAC9D,UAAE;AACA,WAAK,YAAY;AAAA,IACnB;AAAA,EACF,CAAC;AAED,wBAAG,4CAA4C,YAAY;AACzD,UAAM,OAAO,iBAAG,UAAM,gBAAI,GAAG,MAAM,EAAE,mBAAmB,MAAM,MAAS;AAEvE,QAAI;AACF,YAAM,mBAAmB;AAAA,QACvB,kBAAkB;AAAA,QAClB,kBAAkB;AAAA,MACpB,CAAC;AACD,gCAAO,IAAI,EAAE,IAAI,qBAAqB,0BAA0B;AAAA,IAClE,UAAE;AACA,WAAK,YAAY;AAAA,IACnB;AAAA,EACF,CAAC;AACH,CAAC;","names":[]}