{"version":3,"sources":["../../src/voice/realtime_overlapping_speech.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AudioFrame } from '@livekit/rtc-node';\nimport { ReadableStream, type ReadableStreamDefaultController } from 'node:stream/web';\nimport { setImmediate } from 'node:timers/promises';\nimport { describe, expect, it, vi } from 'vitest';\nimport { ChatContext, type FunctionCall } from '../llm/chat_context.js';\nimport { type GenerationCreatedEvent, RealtimeModel, RealtimeSession } from '../llm/realtime.js';\nimport { ToolContext } from '../llm/tool_context.js';\nimport { log } from '../log.js';\nimport { type VADEvent, VADEventType } from '../vad.js';\nimport { Agent } from './agent.js';\nimport { AgentSession } from './agent_session.js';\nimport { AudioOutput } from './io.js';\n\nclass TracingAudioOutput extends AudioOutput {\n  frames = 0;\n  clears = 0;\n  pauses = 0;\n  constructor() {\n    super(24_000, undefined, { pause: true });\n  }\n  async captureFrame(frame: AudioFrame): Promise<void> {\n    await super.captureFrame(frame);\n    this.frames++;\n    this.onPlaybackStarted(Date.now());\n  }\n  flush(): void {\n    super.flush();\n    this.onPlaybackFinished({ playbackPosition: this.frames * 0.1, interrupted: false });\n  }\n  clearBuffer(): void {\n    this.clears++;\n    this.onPlaybackFinished({ playbackPosition: this.frames * 0.1, interrupted: true });\n  }\n  pause(): void {\n    this.pauses++;\n  }\n  resume(): void {}\n}\n\nfunction empty<T>(): ReadableStream<T> {\n  return new ReadableStream<T>({\n    start(controller) {\n      controller.close();\n    },\n  });\n}\n\nclass FakeRealtimeSession extends RealtimeSession {\n  chatCtx = ChatContext.empty();\n  tools = ToolContext.empty();\n  asks = 0;\n  audio?: ReadableStreamDefaultController<AudioFrame>;\n  text?: ReadableStreamDefaultController<string>;\n  async updateInstructions(): Promise<void> {}\n  async updateChatCtx(context: ChatContext): Promise<void> {\n    this.chatCtx = context;\n  }\n  async updateTools(tools: ToolContext): Promise<void> {\n    this.tools = tools;\n  }\n  updateOptions(): void {}\n  pushAudio(): void {}\n  async generateReply(): Promise<GenerationCreatedEvent> {\n    this.asks++;\n    return this.generation(true);\n  }\n  generation(userInitiated: boolean): GenerationCreatedEvent {\n    const audioStream = new ReadableStream<AudioFrame>({\n      start: (controller) => {\n        this.audio = controller;\n      },\n      cancel: () => {\n        this.audio = undefined;\n      },\n    });\n    const textStream = new ReadableStream<string>({\n      start: (controller) => {\n        this.text = controller;\n      },\n      cancel: () => {\n        this.text = undefined;\n      },\n    });\n    this.audio!.enqueue(new AudioFrame(new Int16Array(2400), 24_000, 1, 2400));\n    this.text!.enqueue('The weather today is');\n    return {\n      userInitiated,\n      messageStream: new ReadableStream({\n        start(controller) {\n          controller.enqueue({\n            messageId: 'msg-1',\n            audioStream,\n            textStream,\n            modalities: Promise.resolve(['audio', 'text'] as ('audio' | 'text')[]),\n          });\n          controller.close();\n        },\n      }),\n      functionStream: empty<FunctionCall>(),\n    };\n  }\n  endSpeech(): void {\n    this.audio?.close();\n    this.text?.close();\n  }\n  async commitAudio(): Promise<void> {}\n  async clearAudio(): Promise<void> {}\n  async interrupt(): Promise<void> {}\n  async truncate(): Promise<void> {}\n}\n\nclass FakeRealtimeModel extends RealtimeModel {\n  activeSession!: FakeRealtimeSession;\n  constructor(overlap: boolean, turnDetection = true) {\n    super({\n      messageTruncation: true,\n      turnDetection,\n      supportsOverlappingSpeech: overlap,\n      userTranscription: true,\n      autoToolReplyGeneration: true,\n      audioOutput: true,\n      manualFunctionCalls: true,\n    });\n  }\n  get model(): string {\n    return 'fake';\n  }\n  session(): FakeRealtimeSession {\n    return (this.activeSession = new FakeRealtimeSession(this));\n  }\n  async close(): Promise<void> {}\n}\n\nconst speechStart: VADEvent = {\n  type: VADEventType.START_OF_SPEECH,\n  samplesIndex: 0,\n  timestamp: 0,\n  speechDuration: 0,\n  silenceDuration: 0,\n  frames: [],\n  probability: 1,\n  inferenceDuration: 0,\n  speaking: true,\n  rawAccumulatedSilence: 0,\n  rawAccumulatedSpeech: 0,\n};\n\nasync function start(overlap: boolean, turnDetection = true) {\n  const model = new FakeRealtimeModel(overlap, turnDetection);\n  const session = new AgentSession({\n    llm: model,\n    vad: null,\n    aecWarmupDuration: null,\n    turnHandling: { turnDetection: turnDetection ? 'realtime_llm' : 'manual' },\n  });\n  const output = new TracingAudioOutput();\n  session.output.audio = output;\n  const agent = new Agent({ instructions: 'be concise' });\n  await session.start({ agent });\n  return { model, session, output, activity: agent.getActivityOrThrow() };\n}\n\ndescribe('realtime overlapping speech', () => {\n  it.each([false, true])('handles server speech start with overlap=%s', async (overlap) => {\n    const { model, session, output } = await start(overlap);\n    try {\n      const speech = session.generateReply();\n      await setImmediate();\n      expect(output.frames).toBeGreaterThan(0);\n      model.activeSession.emit('input_speech_started', {});\n      await setImmediate();\n      expect(speech.interrupted).toBe(!overlap);\n      expect(output.clears).toBe(overlap ? 0 : 1);\n      expect(session.userState).toBe('speaking');\n    } finally {\n      model.activeSession.endSpeech();\n      await session.close();\n    }\n  });\n\n  it.each([false, true])(\n    'does not wait for caller silence or pause output (requested: %s)',\n    async (requested) => {\n      const { model, session, output, activity } = await start(true);\n      try {\n        activity.onStartOfSpeech(speechStart);\n        if (requested) session.generateReply();\n        else model.activeSession.emit('generation_created', model.activeSession.generation(false));\n        await setImmediate();\n        expect(output.frames).toBeGreaterThan(0);\n        expect(session.agentState).toBe('speaking');\n        expect(output.pauses).toBe(0);\n        expect(output.clears).toBe(0);\n      } finally {\n        model.activeSession.endSpeech();\n        await session.close();\n      }\n    },\n  );\n\n  it.each([\n    { overlap: false, turnDetection: true },\n    { overlap: true, turnDetection: false },\n  ])(\n    'keeps the silence gate for $overlap overlap and $turnDetection server turn detection',\n    async ({ overlap, turnDetection }) => {\n      const { model, session, output, activity } = await start(overlap, turnDetection);\n      try {\n        activity.onStartOfSpeech(speechStart);\n        session.generateReply();\n        await setImmediate();\n        expect(model.activeSession.asks).toBe(0);\n        expect(output.frames).toBe(0);\n        activity.onEndOfSpeech();\n        await setImmediate();\n        expect(model.activeSession.asks).toBe(1);\n        expect(output.frames).toBeGreaterThan(0);\n      } finally {\n        model.activeSession.endSpeech();\n        await session.close();\n      }\n    },\n  );\n\n  it.each([false, true])(\n    'warns when interruptions are disabled with server turn detection (overlap: %s)',\n    async (overlap) => {\n      const session = new AgentSession({\n        llm: new FakeRealtimeModel(overlap),\n        vad: null,\n        turnHandling: { interruption: { enabled: false } },\n      });\n      const warn = vi.spyOn(log(), 'warn');\n      try {\n        await session.start({ agent: new Agent({ instructions: 'test' }) });\n        expect(warn).toHaveBeenCalledWith(\n          expect.stringContaining('allowInterruptions cannot be false'),\n        );\n      } finally {\n        await session.close();\n        warn.mockRestore();\n      }\n    },\n  );\n});\n"],"mappings":";AAGA,sBAA2B;AAC3B,iBAAqE;AACrE,sBAA6B;AAC7B,oBAAyC;AACzC,0BAA+C;AAC/C,sBAA4E;AAC5E,0BAA4B;AAC5B,iBAAoB;AACpB,iBAA4C;AAC5C,mBAAsB;AACtB,2BAA6B;AAC7B,gBAA4B;AAE5B,MAAM,2BAA2B,sBAAY;AAAA,EAC3C,SAAS;AAAA,EACT,SAAS;AAAA,EACT,SAAS;AAAA,EACT,cAAc;AACZ,UAAM,MAAQ,QAAW,EAAE,OAAO,KAAK,CAAC;AAAA,EAC1C;AAAA,EACA,MAAM,aAAa,OAAkC;AACnD,UAAM,MAAM,aAAa,KAAK;AAC9B,SAAK;AACL,SAAK,kBAAkB,KAAK,IAAI,CAAC;AAAA,EACnC;AAAA,EACA,QAAc;AACZ,UAAM,MAAM;AACZ,SAAK,mBAAmB,EAAE,kBAAkB,KAAK,SAAS,KAAK,aAAa,MAAM,CAAC;AAAA,EACrF;AAAA,EACA,cAAoB;AAClB,SAAK;AACL,SAAK,mBAAmB,EAAE,kBAAkB,KAAK,SAAS,KAAK,aAAa,KAAK,CAAC;AAAA,EACpF;AAAA,EACA,QAAc;AACZ,SAAK;AAAA,EACP;AAAA,EACA,SAAe;AAAA,EAAC;AAClB;AAEA,SAAS,QAA8B;AACrC,SAAO,IAAI,0BAAkB;AAAA,IAC3B,MAAM,YAAY;AAChB,iBAAW,MAAM;AAAA,IACnB;AAAA,EACF,CAAC;AACH;AAEA,MAAM,4BAA4B,gCAAgB;AAAA,EAChD,UAAU,gCAAY,MAAM;AAAA,EAC5B,QAAQ,gCAAY,MAAM;AAAA,EAC1B,OAAO;AAAA,EACP;AAAA,EACA;AAAA,EACA,MAAM,qBAAoC;AAAA,EAAC;AAAA,EAC3C,MAAM,cAAc,SAAqC;AACvD,SAAK,UAAU;AAAA,EACjB;AAAA,EACA,MAAM,YAAY,OAAmC;AACnD,SAAK,QAAQ;AAAA,EACf;AAAA,EACA,gBAAsB;AAAA,EAAC;AAAA,EACvB,YAAkB;AAAA,EAAC;AAAA,EACnB,MAAM,gBAAiD;AACrD,SAAK;AACL,WAAO,KAAK,WAAW,IAAI;AAAA,EAC7B;AAAA,EACA,WAAW,eAAgD;AACzD,UAAM,cAAc,IAAI,0BAA2B;AAAA,MACjD,OAAO,CAAC,eAAe;AACrB,aAAK,QAAQ;AAAA,MACf;AAAA,MACA,QAAQ,MAAM;AACZ,aAAK,QAAQ;AAAA,MACf;AAAA,IACF,CAAC;AACD,UAAM,aAAa,IAAI,0BAAuB;AAAA,MAC5C,OAAO,CAAC,eAAe;AACrB,aAAK,OAAO;AAAA,MACd;AAAA,MACA,QAAQ,MAAM;AACZ,aAAK,OAAO;AAAA,MACd;AAAA,IACF,CAAC;AACD,SAAK,MAAO,QAAQ,IAAI,2BAAW,IAAI,WAAW,IAAI,GAAG,MAAQ,GAAG,IAAI,CAAC;AACzE,SAAK,KAAM,QAAQ,sBAAsB;AACzC,WAAO;AAAA,MACL;AAAA,MACA,eAAe,IAAI,0BAAe;AAAA,QAChC,MAAM,YAAY;AAChB,qBAAW,QAAQ;AAAA,YACjB,WAAW;AAAA,YACX;AAAA,YACA;AAAA,YACA,YAAY,QAAQ,QAAQ,CAAC,SAAS,MAAM,CAAyB;AAAA,UACvE,CAAC;AACD,qBAAW,MAAM;AAAA,QACnB;AAAA,MACF,CAAC;AAAA,MACD,gBAAgB,MAAoB;AAAA,IACtC;AAAA,EACF;AAAA,EACA,YAAkB;AAxGpB;AAyGI,eAAK,UAAL,mBAAY;AACZ,eAAK,SAAL,mBAAW;AAAA,EACb;AAAA,EACA,MAAM,cAA6B;AAAA,EAAC;AAAA,EACpC,MAAM,aAA4B;AAAA,EAAC;AAAA,EACnC,MAAM,YAA2B;AAAA,EAAC;AAAA,EAClC,MAAM,WAA0B;AAAA,EAAC;AACnC;AAEA,MAAM,0BAA0B,8BAAc;AAAA,EAC5C;AAAA,EACA,YAAY,SAAkB,gBAAgB,MAAM;AAClD,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB;AAAA,MACA,2BAA2B;AAAA,MAC3B,mBAAmB;AAAA,MACnB,yBAAyB;AAAA,MACzB,aAAa;AAAA,MACb,qBAAqB;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EACA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA,EACA,UAA+B;AAC7B,WAAQ,KAAK,gBAAgB,IAAI,oBAAoB,IAAI;AAAA,EAC3D;AAAA,EACA,MAAM,QAAuB;AAAA,EAAC;AAChC;AAEA,MAAM,cAAwB;AAAA,EAC5B,MAAM,wBAAa;AAAA,EACnB,cAAc;AAAA,EACd,WAAW;AAAA,EACX,gBAAgB;AAAA,EAChB,iBAAiB;AAAA,EACjB,QAAQ,CAAC;AAAA,EACT,aAAa;AAAA,EACb,mBAAmB;AAAA,EACnB,UAAU;AAAA,EACV,uBAAuB;AAAA,EACvB,sBAAsB;AACxB;AAEA,eAAe,MAAM,SAAkB,gBAAgB,MAAM;AAC3D,QAAM,QAAQ,IAAI,kBAAkB,SAAS,aAAa;AAC1D,QAAM,UAAU,IAAI,kCAAa;AAAA,IAC/B,KAAK;AAAA,IACL,KAAK;AAAA,IACL,mBAAmB;AAAA,IACnB,cAAc,EAAE,eAAe,gBAAgB,iBAAiB,SAAS;AAAA,EAC3E,CAAC;AACD,QAAM,SAAS,IAAI,mBAAmB;AACtC,UAAQ,OAAO,QAAQ;AACvB,QAAM,QAAQ,IAAI,mBAAM,EAAE,cAAc,aAAa,CAAC;AACtD,QAAM,QAAQ,MAAM,EAAE,MAAM,CAAC;AAC7B,SAAO,EAAE,OAAO,SAAS,QAAQ,UAAU,MAAM,mBAAmB,EAAE;AACxE;AAAA,IAEA,wBAAS,+BAA+B,MAAM;AAC5C,mBAAG,KAAK,CAAC,OAAO,IAAI,CAAC,EAAE,+CAA+C,OAAO,YAAY;AACvF,UAAM,EAAE,OAAO,SAAS,OAAO,IAAI,MAAM,MAAM,OAAO;AACtD,QAAI;AACF,YAAM,SAAS,QAAQ,cAAc;AACrC,gBAAM,8BAAa;AACnB,gCAAO,OAAO,MAAM,EAAE,gBAAgB,CAAC;AACvC,YAAM,cAAc,KAAK,wBAAwB,CAAC,CAAC;AACnD,gBAAM,8BAAa;AACnB,gCAAO,OAAO,WAAW,EAAE,KAAK,CAAC,OAAO;AACxC,gCAAO,OAAO,MAAM,EAAE,KAAK,UAAU,IAAI,CAAC;AAC1C,gCAAO,QAAQ,SAAS,EAAE,KAAK,UAAU;AAAA,IAC3C,UAAE;AACA,YAAM,cAAc,UAAU;AAC9B,YAAM,QAAQ,MAAM;AAAA,IACtB;AAAA,EACF,CAAC;AAED,mBAAG,KAAK,CAAC,OAAO,IAAI,CAAC;AAAA,IACnB;AAAA,IACA,OAAO,cAAc;AACnB,YAAM,EAAE,OAAO,SAAS,QAAQ,SAAS,IAAI,MAAM,MAAM,IAAI;AAC7D,UAAI;AACF,iBAAS,gBAAgB,WAAW;AACpC,YAAI,UAAW,SAAQ,cAAc;AAAA,YAChC,OAAM,cAAc,KAAK,sBAAsB,MAAM,cAAc,WAAW,KAAK,CAAC;AACzF,kBAAM,8BAAa;AACnB,kCAAO,OAAO,MAAM,EAAE,gBAAgB,CAAC;AACvC,kCAAO,QAAQ,UAAU,EAAE,KAAK,UAAU;AAC1C,kCAAO,OAAO,MAAM,EAAE,KAAK,CAAC;AAC5B,kCAAO,OAAO,MAAM,EAAE,KAAK,CAAC;AAAA,MAC9B,UAAE;AACA,cAAM,cAAc,UAAU;AAC9B,cAAM,QAAQ,MAAM;AAAA,MACtB;AAAA,IACF;AAAA,EACF;AAEA,mBAAG,KAAK;AAAA,IACN,EAAE,SAAS,OAAO,eAAe,KAAK;AAAA,IACtC,EAAE,SAAS,MAAM,eAAe,MAAM;AAAA,EACxC,CAAC;AAAA,IACC;AAAA,IACA,OAAO,EAAE,SAAS,cAAc,MAAM;AACpC,YAAM,EAAE,OAAO,SAAS,QAAQ,SAAS,IAAI,MAAM,MAAM,SAAS,aAAa;AAC/E,UAAI;AACF,iBAAS,gBAAgB,WAAW;AACpC,gBAAQ,cAAc;AACtB,kBAAM,8BAAa;AACnB,kCAAO,MAAM,cAAc,IAAI,EAAE,KAAK,CAAC;AACvC,kCAAO,OAAO,MAAM,EAAE,KAAK,CAAC;AAC5B,iBAAS,cAAc;AACvB,kBAAM,8BAAa;AACnB,kCAAO,MAAM,cAAc,IAAI,EAAE,KAAK,CAAC;AACvC,kCAAO,OAAO,MAAM,EAAE,gBAAgB,CAAC;AAAA,MACzC,UAAE;AACA,cAAM,cAAc,UAAU;AAC9B,cAAM,QAAQ,MAAM;AAAA,MACtB;AAAA,IACF;AAAA,EACF;AAEA,mBAAG,KAAK,CAAC,OAAO,IAAI,CAAC;AAAA,IACnB;AAAA,IACA,OAAO,YAAY;AACjB,YAAM,UAAU,IAAI,kCAAa;AAAA,QAC/B,KAAK,IAAI,kBAAkB,OAAO;AAAA,QAClC,KAAK;AAAA,QACL,cAAc,EAAE,cAAc,EAAE,SAAS,MAAM,EAAE;AAAA,MACnD,CAAC;AACD,YAAM,OAAO,iBAAG,UAAM,gBAAI,GAAG,MAAM;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,EAAE,OAAO,IAAI,mBAAM,EAAE,cAAc,OAAO,CAAC,EAAE,CAAC;AAClE,kCAAO,IAAI,EAAE;AAAA,UACX,qBAAO,iBAAiB,oCAAoC;AAAA,QAC9D;AAAA,MACF,UAAE;AACA,cAAM,QAAQ,MAAM;AACpB,aAAK,YAAY;AAAA,MACnB;AAAA,IACF;AAAA,EACF;AACF,CAAC;","names":[]}