import { AsyncIterableQueue } from '../utils.js';
import type { TokenData } from './tokenizer.js';
import { SentenceStream, WordStream } from './tokenizer.js';
type TokenizeFunc = (x: string) => string[] | [string, number, number][];
/** Return true if `text` contains an incomplete or unclosed XML tag. */
export declare function hasUnclosedXmlTags(text: string): boolean;
/**
 * Wrap a tokenizer so XML tags don't interfere with sentence splitting.
 *
 * Strips tag markers before tokenization (content inside wrapping tags is kept so the
 * tokenizer can account for its length), remaps offsets back to the original text, and
 * merges sentences with unclosed or tag-only content.
 */
export declare function xmlWrapTokenizer(tokenizeFnc: TokenizeFunc): TokenizeFunc;
export interface BufferedTokenStreamOptions {
    /**
     * Hard cap on emitted token length; a token is flushed before appending a piece that
     * would exceed it.
     */
    maxTokenLength?: number;
    /**
     * Minimum length for the *first* token of each segment, when it should differ from
     * `minTokenLength`.
     *
     * Batching trades time-to-first-result for larger chunks. Setting this lets the opening
     * token go out as soon as it is ready while later tokens still accumulate to
     * `minTokenLength` — so a consumer that batches pays nothing at the head of a segment.
     * Reset by `flush()`, since each segment gets its own first token.
     */
    firstTokenLength?: number;
    /**
     * Treat XML markup as atomic — never split a tag across tokens and merge
     * tag-only/unclosed spans forward. Only enable when the input actually carries markup
     * (e.g. expressive TTS): a stray "<" in plain text can otherwise hold back streaming
     * until flush.
     */
    xmlAware?: boolean;
}
export declare class BufferedTokenStream implements AsyncIterableIterator<TokenData> {
    #private;
    protected queue: AsyncIterableQueue<TokenData>;
    protected closed: boolean;
    constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number, options?: BufferedTokenStreamOptions);
    /** Push a string of text into the token stream */
    pushText(text: string): void;
    /** Flush the stream, causing it to process all pending text */
    flush(): void;
    /** Mark the input as ended and forbid additional pushes */
    endInput(): void;
    next(): Promise<IteratorResult<TokenData>>;
    /** Close both the input and output of the token stream */
    close(): void;
    [Symbol.asyncIterator](): BufferedTokenStream;
}
export declare class BufferedSentenceStream extends SentenceStream {
    #private;
    constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number, options?: BufferedTokenStreamOptions);
    pushText(text: string): void;
    flush(): void;
    close(): void;
    endInput(): void;
    next(): Promise<IteratorResult<TokenData>>;
}
export declare class BufferedWordStream extends WordStream {
    #private;
    constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number);
    pushText(text: string): void;
    flush(): void;
    endInput(): void;
    close(): void;
    next(): Promise<IteratorResult<TokenData>>;
}
export {};
//# sourceMappingURL=token_stream.d.ts.map