{"version":3,"sources":["../../src/voice/events.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { InterruptionDetectionError } from '../inference/interruption/errors.js';\nimport type { OverlappingSpeechEvent } from '../inference/interruption/types.js';\nimport type { LanguageCode } from '../language.js';\nimport type {\n  AgentHandoffItem,\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n  LLM,\n  RealtimeModel,\n  RealtimeModelError,\n} from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport type { AgentMetrics } from '../metrics/base.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS } from '../tts/index.js';\nimport type { TTSError } from '../tts/tts.js';\nimport type { AgentSessionUsage } from './agent_session.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport enum AgentSessionEventTypes {\n  UserInputTranscribed = 'user_input_transcribed',\n  UserTranscriptionTimeout = 'user_transcription_timeout',\n  AgentStateChanged = 'agent_state_changed',\n  UserStateChanged = 'user_state_changed',\n  ConversationItemAdded = 'conversation_item_added',\n  FunctionToolsExecuted = 'function_tools_executed',\n  MetricsCollected = 'metrics_collected',\n  SessionUsageUpdated = 'session_usage_updated',\n  DebugMessage = 'debug_message',\n  SpeechCreated = 'speech_created',\n  AgentFalseInterruption = 'agent_false_interruption',\n  OverlappingSpeech = 'overlapping_speech',\n  /** Audio EOT detector emitted a per-turn prediction. */\n  EotPrediction = 'eot_prediction',\n  Error = 'error',\n  /** Emitted after activity teardown, when the session is no longer running. */\n  Close = 'close',\n}\n\nexport type UserState = 'speaking' | 'listening' | 'away';\nexport type AgentState = 'initializing' | 'idle' | 'listening' | 'thinking' | 'speaking';\n\nexport enum CloseReason {\n  ERROR = 'error',\n  JOB_SHUTDOWN = 'job_shutdown',\n  PARTICIPANT_DISCONNECTED = 'participant_disconnected',\n  USER_INITIATED = 'user_initiated',\n}\n\nexport type ShutdownReason = CloseReason | string;\n\nexport type SpeechSource = 'say' | 'generate_reply' | 'tool_response';\n\nexport type UserStateChangedEvent = {\n  type: 'user_state_changed';\n  oldState: UserState;\n  newState: UserState;\n  createdAt: number;\n};\n\nexport const createUserStateChangedEvent = (\n  oldState: UserState,\n  newState: UserState,\n  createdAt: number = Date.now(),\n): UserStateChangedEvent => ({\n  type: 'user_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type AgentStateChangedEvent = {\n  type: 'agent_state_changed';\n  oldState: AgentState;\n  newState: AgentState;\n  createdAt: number;\n};\n\nexport const createAgentStateChangedEvent = (\n  oldState: AgentState,\n  newState: AgentState,\n  createdAt: number = Date.now(),\n): AgentStateChangedEvent => ({\n  type: 'agent_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type UserInputTranscribedEvent = {\n  type: 'user_input_transcribed';\n  transcript: string;\n  isFinal: boolean;\n  /** Provider-specific ID for the transcribed input item, when available. */\n  itemId: string | null;\n  // TODO(AJS-106): add multi participant support\n  /** Not supported yet. Always null by default. */\n  speakerId: string | null;\n  createdAt: number;\n  language: LanguageCode | null;\n};\n\nexport const createUserInputTranscribedEvent = ({\n  transcript,\n  isFinal,\n  itemId = null,\n  speakerId = null,\n  language = null,\n  createdAt = Date.now(),\n}: {\n  transcript: string;\n  isFinal: boolean;\n  itemId?: string | null;\n  speakerId?: string | null;\n  language?: LanguageCode | null;\n  createdAt?: number;\n}): UserInputTranscribedEvent => ({\n  type: 'user_input_transcribed',\n  transcript,\n  isFinal,\n  itemId,\n  speakerId,\n  language,\n  createdAt,\n});\n\nexport type UserTranscriptionTimeoutEvent = {\n  type: 'user_transcription_timeout';\n  /** Total VAD-detected speech in the turn that produced no transcript, in milliseconds. */\n  speechDuration: number;\n  /** When VAD first detected speech for this untranscribed turn, in milliseconds since epoch. */\n  vadSpeechStartedAt: number;\n  createdAt: number;\n};\n\nexport const createUserTranscriptionTimeoutEvent = ({\n  speechDuration,\n  vadSpeechStartedAt,\n  createdAt = Date.now(),\n}: {\n  speechDuration: number;\n  vadSpeechStartedAt: number;\n  createdAt?: number;\n}): UserTranscriptionTimeoutEvent => ({\n  type: 'user_transcription_timeout',\n  speechDuration,\n  vadSpeechStartedAt,\n  createdAt,\n});\n\nexport type MetricsCollectedEvent = {\n  type: 'metrics_collected';\n  metrics: AgentMetrics;\n  createdAt: number;\n};\n\nexport const createMetricsCollectedEvent = ({\n  metrics,\n  createdAt = Date.now(),\n}: {\n  metrics: AgentMetrics;\n  createdAt?: number;\n}): MetricsCollectedEvent => ({\n  type: 'metrics_collected',\n  metrics,\n  createdAt,\n});\n\nexport type SessionUsageUpdatedEvent = {\n  type: 'session_usage_updated';\n  usage: AgentSessionUsage;\n  createdAt: number;\n};\n\nexport const createSessionUsageUpdatedEvent = ({\n  usage,\n  createdAt = Date.now(),\n}: {\n  usage: AgentSessionUsage;\n  createdAt?: number;\n}): SessionUsageUpdatedEvent => ({\n  type: 'session_usage_updated',\n  usage,\n  createdAt,\n});\n\nexport type ConversationItemAddedEvent = {\n  type: 'conversation_item_added';\n  item: ChatMessage | AgentHandoffItem;\n  createdAt: number;\n};\n\nexport const createConversationItemAddedEvent = (\n  item: ChatMessage | AgentHandoffItem,\n  createdAt: number = Date.now(),\n): ConversationItemAddedEvent => ({\n  type: 'conversation_item_added',\n  item,\n  createdAt,\n});\n\nexport type FunctionToolsExecutedEvent = {\n  type: 'function_tools_executed';\n  /**\n   * Function calls and outputs are parallel arrays: the output at a given index\n   * belongs to the call at the same index, and its `callId` matches the paired\n   * function call's `callId`.\n   */\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt: number;\n};\n\nexport const createFunctionToolsExecutedEvent = ({\n  functionCalls,\n  functionCallOutputs,\n  createdAt = Date.now(),\n}: {\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt?: number;\n}): FunctionToolsExecutedEvent => {\n  return {\n    type: 'function_tools_executed',\n    functionCalls,\n    functionCallOutputs,\n    createdAt,\n  };\n};\n\nexport const zipFunctionCallsAndOutputs = (\n  event: FunctionToolsExecutedEvent,\n): Array<[FunctionCall, FunctionCallOutput]> => {\n  // Pair calls with outputs by list position.\n  return event.functionCalls.map((call, index) => [call, event.functionCallOutputs[index]!]);\n};\n\nexport type SpeechCreatedEvent = {\n  type: 'speech_created';\n  /**\n   * True if the speech was created using public methods like `say` or `generate_reply`\n   */\n  userInitiated: boolean;\n  /**\n   * Source indicating how the speech handle was created\n   */\n  source: SpeechSource;\n  /**\n   * The speech handle that was created\n   */\n  // TODO(shubhra): we need to make sure this doesn't get serialized\n  speechHandle: SpeechHandle;\n  /**\n   * The timestamp when the speech handle was created\n   */\n  createdAt: number;\n};\n\nexport const createSpeechCreatedEvent = ({\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt = Date.now(),\n}: {\n  userInitiated: boolean;\n  source: SpeechSource;\n  speechHandle: SpeechHandle;\n  createdAt?: number;\n}): SpeechCreatedEvent => ({\n  type: 'speech_created',\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt,\n});\n\n/**\n * Audio EOT prediction landed on the wire. Emitted once per turn boundary\n * decision when a `TurnDetector` is wired into the session.\n */\nexport type EotPredictionEvent = {\n  type: 'eot_prediction';\n  /** End-of-turn probability in [0, 1] returned by the detector. */\n  probability: number;\n  /** Threshold below which the detector treats the prediction as unlikely. */\n  threshold: number;\n  /** Model-side inference time, in milliseconds. */\n  inferenceDurationMs: number;\n  /** End-of-speech → prediction receive time, in milliseconds. */\n  delayMs: number;\n  createdAt: number;\n};\n\nexport const createEotPredictionEvent = ({\n  probability,\n  threshold,\n  inferenceDurationMs,\n  delayMs,\n  createdAt = Date.now(),\n}: {\n  probability: number;\n  threshold: number;\n  inferenceDurationMs: number;\n  delayMs: number;\n  createdAt?: number;\n}): EotPredictionEvent => ({\n  type: 'eot_prediction',\n  probability,\n  threshold,\n  inferenceDurationMs,\n  delayMs,\n  createdAt,\n});\n\n/**\n * Internal: a window in which the agent could backchannel (a short acknowledgment\n * such as \"mm-hmm\"), as predicted by the turn detector. Passed to `AgentActivity`\n * only — not surfaced as a public `AgentSession` event (absent from `AgentEvent`,\n * `AgentSessionEventTypes`, and the package exports).\n *\n * `AgentActivity` owns the decision of what to do with it. The end-of-turn margin\n * (`endOfTurnThreshold - endOfTurnProbability`) gives a progressive risk axis: a\n * large positive margin means the user is clearly still going, so riskier\n * backchannels (yeah/okay/right) are safe; a small margin (or a negative one,\n * where `endOfTurnProbability >= endOfTurnThreshold` and a reply is imminent)\n * calls for safe, less ambiguous ones (hmm/uh-huh) that won't collide with the reply.\n *\n * @internal\n */\nexport type _AgentBackchannelOpportunityEvent = {\n  type: 'agent_backchannel_opportunity';\n  probability: number;\n  threshold: number;\n  endOfTurnProbability: number;\n  endOfTurnThreshold: number;\n  language?: string;\n  createdAt: number;\n};\n\n/** @internal */\nexport const _createAgentBackchannelOpportunityEvent = ({\n  probability,\n  threshold,\n  endOfTurnProbability,\n  endOfTurnThreshold,\n  language,\n  createdAt = Date.now(),\n}: {\n  probability: number;\n  threshold: number;\n  endOfTurnProbability: number;\n  endOfTurnThreshold: number;\n  language?: string;\n  createdAt?: number;\n}): _AgentBackchannelOpportunityEvent => ({\n  type: 'agent_backchannel_opportunity',\n  probability,\n  threshold,\n  endOfTurnProbability,\n  endOfTurnThreshold,\n  language,\n  createdAt,\n});\n\nexport type UserTurnExceededEvent = {\n  type: 'user_turn_exceeded';\n  /** Transcript from the current uncommitted user turn only. */\n  transcript: string;\n  /** Full transcript since the start of user speaking in the accumulation window. */\n  accumulatedTranscript: string;\n  /** Total word count since the start of user speaking in the accumulation window. */\n  accumulatedWordCount: number;\n  /** Duration of the user turn accumulation window in milliseconds. */\n  duration: number;\n  createdAt: number;\n};\n\nexport const createUserTurnExceededEvent = ({\n  transcript,\n  accumulatedTranscript,\n  accumulatedWordCount,\n  duration,\n  createdAt = Date.now(),\n}: {\n  transcript: string;\n  accumulatedTranscript: string;\n  accumulatedWordCount: number;\n  duration: number;\n  createdAt?: number;\n}): UserTurnExceededEvent => ({\n  type: 'user_turn_exceeded',\n  transcript,\n  accumulatedTranscript,\n  accumulatedWordCount,\n  duration,\n  createdAt,\n});\n\nexport type ErrorEvent = {\n  type: 'error';\n  error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError;\n  source?: LLM | STT | TTS | RealtimeModel;\n  createdAt: number;\n};\n\nexport const createErrorEvent = (\n  error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError,\n  source?: LLM | STT | TTS | RealtimeModel,\n  createdAt: number = Date.now(),\n): ErrorEvent => ({\n  type: 'error',\n  error,\n  source,\n  createdAt,\n});\n\nexport type CloseEvent = {\n  type: 'close';\n  error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | null;\n  reason: ShutdownReason;\n  createdAt: number;\n};\n\nexport const createCloseEvent = (\n  reason: ShutdownReason,\n  error:\n    | RealtimeModelError\n    | STTError\n    | TTSError\n    | LLMError\n    | InterruptionDetectionError\n    | null = null,\n  createdAt: number = Date.now(),\n): CloseEvent => ({\n  type: 'close',\n  error,\n  reason,\n  createdAt,\n});\n\nexport type AgentFalseInterruptionEvent = {\n  type: 'agent_false_interruption';\n  /** Whether the false interruption was resumed automatically. */\n  resumed: boolean;\n  createdAt: number;\n};\n\nexport const createAgentFalseInterruptionEvent = ({\n  resumed,\n  createdAt = Date.now(),\n}: {\n  resumed: boolean;\n  createdAt?: number;\n}): AgentFalseInterruptionEvent => ({\n  type: 'agent_false_interruption',\n  resumed,\n  createdAt,\n});\n\nexport type AgentEvent =\n  | UserInputTranscribedEvent\n  | UserTranscriptionTimeoutEvent\n  | UserStateChangedEvent\n  | AgentStateChangedEvent\n  | MetricsCollectedEvent\n  | SessionUsageUpdatedEvent\n  | ConversationItemAddedEvent\n  | FunctionToolsExecutedEvent\n  | SpeechCreatedEvent\n  | AgentFalseInterruptionEvent\n  | OverlappingSpeechEvent\n  | ErrorEvent\n  | CloseEvent;\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAwBO,IAAK,yBAAL,kBAAKA,4BAAL;AACL,EAAAA,wBAAA,0BAAuB;AACvB,EAAAA,wBAAA,8BAA2B;AAC3B,EAAAA,wBAAA,uBAAoB;AACpB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,yBAAsB;AACtB,EAAAA,wBAAA,kBAAe;AACf,EAAAA,wBAAA,mBAAgB;AAChB,EAAAA,wBAAA,4BAAyB;AACzB,EAAAA,wBAAA,uBAAoB;AAEpB,EAAAA,wBAAA,mBAAgB;AAChB,EAAAA,wBAAA,WAAQ;AAER,EAAAA,wBAAA,WAAQ;AAjBE,SAAAA;AAAA,GAAA;AAuBL,IAAK,cAAL,kBAAKC,iBAAL;AACL,EAAAA,aAAA,WAAQ;AACR,EAAAA,aAAA,kBAAe;AACf,EAAAA,aAAA,8BAA2B;AAC3B,EAAAA,aAAA,oBAAiB;AAJP,SAAAA;AAAA,GAAA;AAkBL,MAAM,8BAA8B,CACzC,UACA,UACA,YAAoB,KAAK,IAAI,OACF;AAAA,EAC3B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,+BAA+B,CAC1C,UACA,UACA,YAAoB,KAAK,IAAI,OACD;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AAeO,MAAM,kCAAkC,CAAC;AAAA,EAC9C;AAAA,EACA;AAAA,EACA,SAAS;AAAA,EACT,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,YAAY,KAAK,IAAI;AACvB,OAOkC;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAWO,MAAM,sCAAsC,CAAC;AAAA,EAClD;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAIsC;AAAA,EACpC,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AAQO,MAAM,8BAA8B,CAAC;AAAA,EAC1C;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAG8B;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AACF;AAQO,MAAM,iCAAiC,CAAC;AAAA,EAC7C;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAGiC;AAAA,EAC/B,MAAM;AAAA,EACN;AAAA,EACA;AACF;AAQO,MAAM,mCAAmC,CAC9C,MACA,YAAoB,KAAK,IAAI,OACG;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AACF;AAcO,MAAM,mCAAmC,CAAC;AAAA,EAC/C;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,MAIkC;AAChC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,MAAM,6BAA6B,CACxC,UAC8C;AAE9C,SAAO,MAAM,cAAc,IAAI,CAAC,MAAM,UAAU,CAAC,MAAM,MAAM,oBAAoB,KAAK,CAAE,CAAC;AAC3F;AAuBO,MAAM,2BAA2B,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAK2B;AAAA,EACzB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAmBO,MAAM,2BAA2B,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAM2B;AAAA,EACzB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AA4BO,MAAM,0CAA0C,CAAC;AAAA,EACtD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAO0C;AAAA,EACxC,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAeO,MAAM,8BAA8B,CAAC;AAAA,EAC1C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAM8B;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,OACA,QACA,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,QACA,QAMW,MACX,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,oCAAoC,CAAC;AAAA,EAChD;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAGoC;AAAA,EAClC,MAAM;AAAA,EACN;AAAA,EACA;AACF;","names":["AgentSessionEventTypes","CloseReason"]}