import { type JsonObject } from '@bufbuild/protobuf'; import { AgentSession as pb } from '@livekit/protocol'; import { type AudioFrame, type RemoteParticipant, type Room } from '@livekit/rtc-node'; import { type Throws } from '@livekit/throws-transformer/throws'; import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter'; import type { Context, Span } from '@opentelemetry/api'; import type { ReadableStream } from 'node:stream/web'; import type { z } from 'zod'; import type { BaseStreamingTurnDetector } from '../inference/eot/base.js'; import { type LLMModels, type STTModelString, type TTSModelString } from '../inference/index.js'; import type { OverlappingSpeechEvent } from '../inference/interruption/types.js'; import type { FunctionCall, FunctionCallOutput } from '../llm/chat_context.js'; import { AgentHandoffItem, ChatContext, ChatMessage, type Instructions } from '../llm/chat_context.js'; import type { LLM, RealtimeModel, RealtimeModelError, ToolChoice, ToolContextEntry, ToolContextLike } from '../llm/index.js'; import { DuplexModel, ToolContext } from '../llm/index.js'; import type { LLMError } from '../llm/llm.js'; import { type ModelUsage, ModelUsageCollector } from '../metrics/model_usage.js'; import type { STT } from '../stt/index.js'; import type { STTError } from '../stt/stt.js'; import { type SpeechSteeringOptions } from '../tts/provider_format.js'; import type { TTS, TTSError } from '../tts/tts.js'; import { type ResolvedSessionConnectOptions, type SessionConnectOptions } from '../types.js'; import { Task } from '../utils.js'; import type { VAD } from '../vad.js'; import { type Agent } from './agent.js'; import { AgentActivity } from './agent_activity.js'; import type { AMD, AMDPredictionEvent } from './amd.js'; import type { _TurnDetector } from './audio_recognition.js'; import { type AgentEvent, type AgentFalseInterruptionEvent, AgentSessionEventTypes, type AgentState, type AgentStateChangedEvent, type CloseEvent, type ConversationItemAddedEvent, type EotPredictionEvent, type ErrorEvent, type FunctionToolsExecutedEvent, type MetricsCollectedEvent, type SessionUsageUpdatedEvent, type ShutdownReason, type SpeechCreatedEvent, type UserInputTranscribedEvent, type UserState, type UserStateChangedEvent, type UserTranscriptionTimeoutEvent } from './events.js'; import { AgentInput, AgentOutput } from './io.js'; import { KeytermDetector, type KeytermsOptions } from './keyterm_detection.js'; import { RecorderIO } from './recorder_io/index.js'; import { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js'; import type { UnknownUserData } from './run_context.js'; import type { SpeechHandle } from './speech_handle.js'; import { type RunOutputOptions, RunResult } from './testing/run_result.js'; import { type AsyncToolOptions, type ToolHandlingOptions } from './tool_executor.js'; import type { TextTransform } from './transcription/text_transforms.js'; import type { EndpointingOptions } from './turn_config/endpointing.js'; import type { InternalTurnHandlingOptions, TurnHandlingOptions } from './turn_config/turn_handling.js'; export interface AgentSessionUsage { /** List of usage summaries, one per model/provider combination. */ modelUsage: Array>; } /** * Granular control over which recording features are active. * * Recording keys default to `true` when omitted, so `{ logs: false }` means "record * everything except logs". Redaction defaults to the project setting; `false` is ignored when * redaction is enabled globally for the project. Pass to {@link AgentSession.start} as `record`: * * - `record: true` — all on (backward compatible) * - `record: false` — all off (backward compatible) * - `record: { audio: true, traces: false }` — granular * - `record: { redaction: true }` — enable redaction for the session */ export interface RecordingOptions { /** Record session audio. Defaults to `true`. */ audio?: boolean; /** Export OpenTelemetry trace spans. Defaults to `true`. */ traces?: boolean; /** Export OpenTelemetry logs. Defaults to `true`. */ logs?: boolean; /** Upload the conversation transcript (chat history). Defaults to `true`. */ transcript?: boolean; /** Enable redaction. `false` does not disable project redaction. */ redaction?: boolean; } /** @internal Recording options with every category resolved to a boolean. */ export type ResolvedRecordingOptions = Required; /** * Resolve a `record` argument into explicit per-category flags. A boolean turns * every category on or off; a partial object is merged onto all-on so omitted * keys default to `true`. * * @internal */ export declare function resolveRecordingOptions(record: boolean | RecordingOptions): ResolvedRecordingOptions; export interface InternalSessionOptions extends AgentSessionOptions { turnHandling: InternalTurnHandlingOptions; useTtsAlignedTranscript: boolean; maxToolSteps: number; userAwayTimeout: number | null; transcriptionTimeout: number | null; ttsReadIdleTimeout: number; forwardAudioIdleTimeout: number; ttsTextTransforms: readonly TextTransform[] | null; /** Resolved per-category recording options for this session. */ recordingOptions: ResolvedRecordingOptions; } export declare const defaultAgentSessionOptions: { readonly maxToolSteps: 3; readonly userAwayTimeout: 15; readonly transcriptionTimeout: null; readonly aecWarmupDuration: 3000; readonly ttsReadIdleTimeout: 10000; readonly forwardAudioIdleTimeout: 10000; readonly turnHandling: {}; readonly useTtsAlignedTranscript: true; readonly ttsTextTransforms: readonly ["filter_markdown", "filter_emoji"]; readonly expressive: false; }; /** @deprecated {@link VoiceOptions} has been flattened onto to {@link AgentSessionOptions} */ export type VoiceOptions = { maxToolSteps: number; /** @deprecated Use {@link AgentSessionOptions.turnHandling}.preemptiveGeneration instead. */ preemptiveGeneration?: boolean; userAwayTimeout?: number | null; /** @deprecated Use {@link AgentSessionOptions.turnHandling}.interruption.mode instead. */ allowInterruptions?: boolean; /** @deprecated Use {@link AgentSessionOptions.turnHandling}.interruption.discardAudioIfUninterruptible instead. */ discardAudioIfUninterruptible?: boolean; /** @deprecated Use {@link AgentSessionOptions.turnHandling}.interruption.minDuration instead. */ minInterruptionDuration?: number; /** @deprecated Use {@link AgentSessionOptions.turnHandling}.interruption.minWords instead. */ minInterruptionWords?: number; /** @deprecated Use {@link AgentSessionOptions.turnHandling}.endpointing.minDelay instead. */ minEndpointingDelay?: number; /** @deprecated Use {@link AgentSessionOptions.turnHandling}.endpointing.maxDelay instead. */ maxEndpointingDelay?: number; }; export type TurnDetectionMode = 'stt' | 'vad' | 'realtime_llm' | 'manual' | _TurnDetector | BaseStreamingTurnDetector; export type AgentSessionCallbacks = { [AgentSessionEventTypes.UserInputTranscribed]: (ev: UserInputTranscribedEvent) => void; [AgentSessionEventTypes.UserTranscriptionTimeout]: (ev: UserTranscriptionTimeoutEvent) => void; [AgentSessionEventTypes.AgentStateChanged]: (ev: AgentStateChangedEvent) => void; [AgentSessionEventTypes.UserStateChanged]: (ev: UserStateChangedEvent) => void; [AgentSessionEventTypes.ConversationItemAdded]: (ev: ConversationItemAddedEvent) => void; [AgentSessionEventTypes.FunctionToolsExecuted]: (ev: FunctionToolsExecutedEvent) => void; [AgentSessionEventTypes.MetricsCollected]: (ev: MetricsCollectedEvent) => void; [AgentSessionEventTypes.SessionUsageUpdated]: (ev: SessionUsageUpdatedEvent) => void; [AgentSessionEventTypes.DebugMessage]: (ev: pb.DebugMessage) => void; [AgentSessionEventTypes.SpeechCreated]: (ev: SpeechCreatedEvent) => void; [AgentSessionEventTypes.AgentFalseInterruption]: (ev: AgentFalseInterruptionEvent) => void; [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void; [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void; [AgentSessionEventTypes.OverlappingSpeech]: (ev: OverlappingSpeechEvent) => void; [AgentSessionEventTypes.EotPrediction]: (ev: EotPredictionEvent) => void; }; export type AgentSessionOptions = { stt?: STT | STTModelString; /** * Voice Activity Detection. When omitted, `AgentSession` auto-provisions a * bundled `inference.VAD({ model: 'silero' })` and marks it as the default * (so sites that check whether the user supplied a VAD treat the bundled * one as absent). Pass `null` to opt out entirely. */ vad?: VAD | null; llm?: LLM | RealtimeModel | DuplexModel | LLMModels; tts?: TTS | TTSModelString; userData?: UserData; connOptions?: SessionConnectOptions; tools?: ToolContextLike; toolHandling?: ToolHandlingOptions; /** @deprecated use turnHandling.turnDetection instead */ turnDetection?: TurnDetectionMode; /** @deprecated use top-level SessionOptions fields instead */ voiceOptions?: Partial; maxToolSteps?: number; /** * @deprecated Use `turnHandling.preemptiveGeneration` instead. * When set, migrated into `turnHandling.preemptiveGeneration.enabled`. */ preemptiveGeneration?: boolean; /** * If set, set the user state as "away" after this amount of time after user and agent are * silent. Set to `null` to disable. * @defaultValue 15.0 */ userAwayTimeout?: number | null; /** * Emit `user_transcription_timeout` when VAD detects user speech during the user's turn but no * non-empty final transcript arrives within this many milliseconds after the speech ends. This * can happen because STT failed or because audio was intentionally withheld from STT, such as * during AEC warmup or uninterruptible agent speech. A non-empty final transcript satisfies the * timeout for the current turn even if adaptive interruption detection later discards it as a * backchannel. Requires both VAD and STT. Set to `null` to disable. * @defaultValue null */ transcriptionTimeout?: number | null; /** * Duration in milliseconds for AEC (Acoustic Echo Cancellation) warmup, during which * interruptions from audio activity are suppressed. Set to `null` to disable. * Defaults to 3000, or `null` for outbound SIP calls. */ aecWarmupDuration?: number | null; /** * Maximum time in milliseconds to wait for the next frame on the TTS audio stream * inside `performTTSInference`. Applies to every read, including the first. * If exceeded, the TTS stream is forcibly closed and a stall warning is logged. * @defaultValue 10000 */ ttsReadIdleTimeout?: number; /** * Maximum time in milliseconds to wait for the next frame while forwarding TTS * audio to the audio output inside `performAudioForwarding`. Applies to every read, * including the first. If exceeded, forwarding is forcibly closed and a stall * warning is logged. * @defaultValue 10000 */ forwardAudioIdleTimeout?: number; /** * Configuration for turn handling. */ turnHandling?: Partial; /** * Keyterm biasing for the STT. Holds static `keyterms` plus `keytermDetection` * (LLM extraction). Applies to STTs that accept a term list; on others it warns * and is ignored. */ keytermsOptions?: KeytermsOptions; useTtsAlignedTranscript?: boolean; /** * Transforms to apply to TTS input text. Built-in transforms are `filter_markdown` * and `filter_emoji`; pass `null` to disable text transforms. */ ttsTextTransforms?: readonly TextTransform[] | null; /** * Let the LLM steer how the agent sounds. * * When enabled, the provider's markup guide is injected into the LLM prompt so it can * emit inline delivery tags (emotion, pacing, non-verbal sounds), which are rendered by * the TTS and stripped from the transcript. Pass an {@link ExpressiveOptions} object to * steer or override the injected instructions. Requires an * {@link inference.TTS | inference TTS} with a model that declares a markup dialect; it * stays off otherwise. * * @defaultValue false */ expressive?: boolean | ExpressiveOptions; }; /** * Configuration for the expressive pipeline, passed as `AgentSession({ expressive: ... })`. * * Controls how TTS markup instructions are injected into the LLM when expressive is * enabled. All keys are optional; common shapes: * * - `{ speechSteering: {...} }` — steer delivery and non-verbal sounds on top of the * provider-agnostic default instructions. * - `{ ttsInstructionsTemplate: '...' }` — a fully custom prompt. * - `{ ttsInstructionsAppend: '...' }` — your own rules appended to the template. * * Any explicit template overrides the default; unset parts fall back to the * provider-agnostic default. */ export interface ExpressiveOptions { speechSteering?: SpeechSteeringOptions; ttsInstructionsTemplate?: Instructions | string; ttsInstructionsAppend?: string; } /** The placeholder the expressive template substitutes the provider's markup guide into. */ export declare const TTS_INSTRUCTIONS_PLACEHOLDER = "{tts.markup.llm_instructions}"; export declare const DEFAULT_EXPRESSIVE_OPTIONS: ExpressiveOptions; /** * Resolve a user {@link ExpressiveOptions} to a concrete options object for a provider. * * Starts from `defaults`, renders `speechSteering` into per-provider delivery guidelines * appended to the template, then applies any explicit `ttsInstructionsTemplate` override * and `ttsInstructionsAppend` (last, so the user's free-form rules always win). Steering * fields the user doesn't set fall back to `defaults`' `speechSteering`, so an explicit * value always wins over a default. The returned object always has * `ttsInstructionsTemplate` and `speechSteering` (never `ttsInstructionsAppend`); * `speechSteering` passes through so injection can filter the advertised markup vocabulary * (`TTSMarkup.llmInstructions`) with it. */ export declare function resolveExpressiveOptions(expr: ExpressiveOptions, options: { providerKey: string; defaults: ExpressiveOptions; }): ExpressiveOptions; export type AgentSessionUpdateOptions = { /** Change expressive TTS delivery for subsequent replies. */ expressive?: boolean | ExpressiveOptions; /** Configuration updates for turn handling. */ turnHandling?: { /** * Strategy for deciding when the user has finished speaking. * * - `undefined`: leave the current turn detection setting unchanged. * - `null`: clear the current turn detection setting and return to automatic selection. * - `TurnDetectionMode`: set the turn detection strategy to the provided value. */ turnDetection?: TurnDetectionMode | null; /** Endpointing options to merge into the current session defaults. */ endpointing?: Partial; }; /** * @deprecated use turnHandling.turnDetection instead. * * - `undefined`: leave the current turn detection setting unchanged. * - `null`: clear the current turn detection setting and return to automatic selection. * - `TurnDetectionMode`: set the turn detection strategy to the provided value. */ turnDetection?: TurnDetectionMode | null; /** * Replace the user-defined keyterms applied to the STT. Auto-detected keyterms * are left untouched. */ keyterms?: string[]; }; type ActivityTransitionOptions = { previousActivity?: 'close' | 'pause'; newActivity?: 'start' | 'resume'; blockedTasks?: Task[]; waitOnEnter?: boolean; }; declare const AgentSession_base: new () => TypedEmitter; export declare class AgentSession extends AgentSession_base { vad?: VAD; stt?: STT; llm?: LLM | RealtimeModel; tts?: TTS; turnDetection?: TurnDetectionMode; /** @deprecated use {@link sessionOptions } instead */ readonly options: VoiceOptions; readonly sessionOptions: InternalSessionOptions; private readonly activityLock; private agent?; private activity?; private nextActivity?; private updateActivityTask?; private started; private sessionHost?; private _chatCtx; private _userData; private _toolCtx; private _userState; private _agentState; private _input; private _output; private closing; private closingController; private closingTask; private userAwayTimer; private idleHolds; private idleReleased; private _aecWarmupTimer; private readonly _aecWarmupDurationExplicit; /** * The session's expressive setting, as the user passed it. * @internal */ _expressive: boolean | ExpressiveOptions; /** * Whether the markup guide has been injected at least once this session. * * Latches on: it is what licenses the history scrub on a later expressive-off turn * (a handoff to a TTS without a markup dialect builds a fresh `AgentActivity`, so the * flag has to outlive it). Sessions that never enabled expressive keep it `false` and * are never scrubbed. * * @internal */ _expressiveEverActive: boolean; /** * Whether the "template has no markup-guide placeholder" warning has been emitted. * * Session-scoped so a misconfigured template warns once rather than once per turn, and * survives a handoff (which builds a fresh `AgentActivity`). * * @internal */ _warnedExpressiveTemplate: boolean; private _connOptions; private llmErrorCounts; private ttsErrorCounts; private sessionSpan?; private agentSpeakingSpan?; private loopStallCount; private loopStallTotal; private loopStallMax; private _interruptionDetection?; /** * True iff this session auto-provisioned the bundled silero VAD because the * caller passed no `vad=`. Set once in the constructor; immutable from then * on. Read it via `AgentActivity.usingDefaultVad` from voice-pipeline code. * * @internal */ _usingDefaultVad: boolean; /** @internal True when the current job is a text simulation. */ get _textOnly(): boolean; /** @internal */ _usageCollector: ModelUsageCollector; /** @internal */ readonly _keytermDetector: KeytermDetector; /** @internal */ _roomIO?: RoomIO; /** * Currently active AMD instance, if one was constructed against this session. * Mirrors python `AgentSession._amd`. Useful for tests, telemetry, and * higher-level helpers that need to introspect classification state. */ private _amd; /** @internal */ _aecWarmupRemaining: number; /** @internal */ _recorderIO?: RecorderIO; /** @internal */ rootSpanContext?: Context; /** @internal */ _recordedEvents: AgentEvent[]; /** @internal */ _asyncToolOptions: AsyncToolOptions; /** @internal */ _sessionToolsetsSetup: boolean; /** @internal True when any recording category is enabled. */ get _enableRecording(): boolean; /** @internal True when project or session redaction is enabled. */ _redactionEnabled: boolean; /** @internal Whether the realtime audio-redaction warning was emitted for this session. */ _warnedRealtimeAudioRedaction: boolean; /** @internal - Timestamp when the session started (milliseconds) */ _startedAt?: number; /** @internal - Whether `start()` has been called and completed. */ get _started(): boolean; /** @internal - Whether the session is closing/draining. */ get _closing(): boolean; /** * Aborted when shutdown starts, before the activity drains or Agent.onExit() runs. * Use this signal to release work that activity teardown can await. * The Close event fires after activity teardown and cannot release those waits. * The signal stays aborted after close; read it again when starting a new run. * @internal */ get _closingSignal(): AbortSignal; /** @internal - Current run state for testing */ _globalRunState?: RunResult; /** @internal */ _userSpeakingSpan?: Span; private logger; constructor(options?: AgentSessionOptions); emit(event: K, ...args: Parameters): boolean; get input(): AgentInput; get output(): AgentOutput; get userData(): UserData; get history(): ChatContext; /** The effective keyterms (user-defined + auto-detected) currently applied to the STT. */ get keyterms(): string[]; /** Connection options for STT, LLM, and TTS. */ get connOptions(): ResolvedSessionConnectOptions; get interruptionDetection(): "adaptive" | "vad" | undefined; /** * Returns usage summaries for this session, one per model/provider combination. */ get usage(): AgentSessionUsage; get useTtsAlignedTranscript(): boolean; set userData(value: UserData); private _startImpl; start({ agent, room, inputOptions, outputOptions, record, }: { agent: Agent; room?: Room; inputOptions?: Partial; outputOptions?: Partial; record?: boolean | RecordingOptions; }): Promise; updateAgent(agent: Agent): void; commitUserTurn(): void; clearUserTurn(): void; say(text: string | ReadableStream, options?: { audio?: ReadableStream; allowInterruptions?: boolean; addToChatCtx?: boolean; }): SpeechHandle; /** * Interrupt the current speech generation. * * A queued speech created with `allowInterruptions: false` keeps playing, along with the ones * behind it, unless `force` is set. * * @returns A future that completes when the interruption is fully processed. * @throws Error if the session is not running, or if the speech currently playing disallows * interruptions and `force` is false. */ interrupt(options?: { force?: boolean; }): import("../utils.js").Future; /** @internal — emit a debug/trace payload to the debugger/recorder. */ _emitDebugMessage(payload: JsonObject): void; /** * The currently bound `AMD` instance, or `null` if AMD is not in use. * Mirrors python `AgentSession.amd`. */ get amd(): AMD | null; /** @internal — used by AMD to register/unregister itself with the session. */ _setAmd(amd: AMD | null): void; /** * The currently running activity, or `undefined` when none is active. Mirrors * python `session._activity` — exposed so tightly-coupled internals (e.g. AMD) * can read activity state such as the endpointing delay. * @internal */ get _activity(): AgentActivity | undefined; /** * @internal — forwarded to {@link SessionHost} so a connected * {@link RemoteSession} peer receives an `amd_prediction` event when AMD * settles. Mirrors python `AgentSession._session_host._on_amd_prediction`. */ _onAmdPrediction(event: AMDPredictionEvent): void; /** * @internal — returns a tee'd branch of the active participant audio stream * for AMD's dedicated STT. Returns `undefined` when no `AgentActivity` is * running yet (the AMD STT pump retries until an activity is available). */ _subscribeAudioStream(): ReadableStream | undefined; pauseReplyAuthorization(): void; /** * Resume automatic replies after pauseReplyAuthorization(). * @throws Error if the session is not running. */ resumeReplyAuthorization(): Throws; updateOptions(options?: AgentSessionUpdateOptions): void; generateReply(options?: { userInput?: string | ChatMessage; chatCtx?: ChatContext; instructions?: string | Instructions; toolChoice?: ToolChoice; allowInterruptions?: boolean; /** The input modality used for generating the reply. Defaults to `"text"`. */ inputModality?: 'audio' | 'text'; }): SpeechHandle; /** * Run a test with user input and return a result for assertions. * * This method is primarily used for testing agent behavior without * requiring a real room connection. * * @example * ```typescript * const result = await session.run({ userInput: 'Hello' }); * result.expect.nextEvent().isMessage({ role: 'assistant' }); * result.expect.noMoreEvents(); * ``` * * @param options - Run options including user input and optional output type. * When `outputType` is set and the turn ends without structured output, the * run re-prompts the model up to `outputOptions.maxRetries` times (default 2) * before rejecting with `UnexpectedModelBehavior`. Pass `outputOptions: null` * to disable retries entirely. * @returns A RunResult that resolves when the agent finishes responding */ run({ userInput, inputModality, outputType, outputOptions, }: { userInput: string; inputModality?: 'audio' | 'text'; outputType?: z.ZodType; outputOptions?: RunOutputOptions | null; }): RunResult; /** @internal */ _updateActivity(agent: Agent, options?: ActivityTransitionOptions): Promise; get chatCtx(): ChatContext; get agentState(): AgentState; get userState(): UserState; get currentAgent(): Agent; get toolCtx(): ToolContext; get tools(): readonly ToolContextEntry[]; waitForIdle(): Promise; waitForIdleAndHold(fn: (activity: AgentActivity) => Promise | T): Promise; /** * Wait until any foreground idle-hold (`waitForIdleAndHold`) is released. * Returns `true` if it actually waited for a release — callers use that to * re-verify idleness, since work may have resumed during the hold. * @internal */ _waitForIdleHoldReleased(): Promise; close(): Promise; shutdown(options?: { drain?: boolean; reason?: ShutdownReason; }): void; /** @internal */ _closeSoon({ reason, drain, error, }: { reason: ShutdownReason; drain?: boolean; error?: RealtimeModelError | STTError | TTSError | LLMError | null; }): void; /** @internal */ _onError(error: RealtimeModelError | STTError | TTSError | LLMError): void; /** @internal */ _conversationItemAdded(item: ChatMessage | AgentHandoffItem): void; /** @internal */ _toolItemsAdded(items: (FunctionCall | FunctionCallOutput)[]): void; /** @internal */ _updateAgentState(state: AgentState, options?: { startTime?: number; otelContext?: Context; }): void; /** @internal */ _updateUserState(state: UserState, options?: { lastSpeakingTime?: number; otelContext?: Context; }): void; private onAudioInputChanged; private onAudioEnabledChanged; private onAudioOutputChanged; private onTextOutputChanged; private _setUserAwayTimer; private _cancelUserAwayTimer; /** @internal */ _onAecWarmupExpired(): void; /** @internal */ _onRoomIOParticipantLinked(participant: RemoteParticipant): void; /** @internal */ _recordLoopStall(durationInS: number, timestampMs: number, cause: string): void; private _onUserInputTranscribed; private closeImpl; private closeImplInner; } export {}; //# sourceMappingURL=agent_session.d.ts.map