{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { BufferedSentenceStream, BufferedWordStream, xmlWrapTokenizer } from '../token_stream.js';\nimport * as tokenizer from '../tokenizer.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n language: string;\n /**\n * Minimum length for a span to be treated as its own sentence; shorter spans are merged\n * forward into the next one.\n */\n minSentenceLength: number;\n /** Minimum buffered text before the stream emits. */\n streamContextLength: number;\n /** Keep original whitespace/formatting in emitted tokens. */\n retainFormat: boolean;\n /**\n * Hard cap on emitted token length; a token is flushed before appending a sentence that\n * would exceed it. Unlimited when omitted.\n */\n maxTokenLength?: number;\n /**\n * Minimum length a token must reach before it is emitted. Sentences are batched together\n * until the running token reaches this length, so raising it (e.g. toward\n * `maxTokenLength`) yields larger, fewer chunks. Defaults to `minSentenceLength`\n * (per-sentence emission).\n */\n minTokenLength?: number;\n /**\n * Minimum length for the *first* token of each segment, when it should differ from\n * `minTokenLength`. Lets a batching consumer still emit its opening chunk as soon as one\n * sentence is ready, so batching costs nothing at the head of a segment.\n */\n firstTokenLength?: number;\n /**\n * Treat XML markup as atomic — never split a tag across tokens and keep tags attached to\n * the following sentence. Only enable when the input actually carries markup (e.g.\n * expressive TTS): a stray \"<\" in plain text can otherwise hold back streaming until\n * flush.\n */\n xmlAware?: boolean;\n}\n\nconst defaultTokenizerOptions: TokenizerOptions = {\n language: 'en-US',\n minSentenceLength: 20,\n streamContextLength: 10,\n retainFormat: false,\n};\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n #config: TokenizerOptions;\n\n constructor(options?: Partial) {\n super();\n this.#config = {\n ...defaultTokenizerOptions,\n ...options,\n };\n }\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n tokenize(text: string, language?: string): string[] {\n const split = (input: string) =>\n splitSentences(input, this.#config.minSentenceLength, this.#config.retainFormat);\n const tokenizeFnc = this.#config.xmlAware ? xmlWrapTokenizer(split) : split;\n return tokenizeFnc(text).map((tok) => (Array.isArray(tok) ? tok[0] : tok));\n }\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n stream(language?: string): tokenizer.SentenceStream {\n return new BufferedSentenceStream(\n (text: string) =>\n splitSentences(text, this.#config.minSentenceLength, this.#config.retainFormat),\n this.#config.minTokenLength ?? this.#config.minSentenceLength,\n this.#config.streamContextLength,\n {\n maxTokenLength: this.#config.maxTokenLength,\n firstTokenLength: this.#config.firstTokenLength,\n xmlAware: this.#config.xmlAware,\n },\n );\n }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n #ignorePunctuation: boolean;\n\n constructor(ignorePunctuation = true) {\n super();\n this.#ignorePunctuation = ignorePunctuation;\n }\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n tokenize(text: string, language?: string): string[] {\n return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n }\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n stream(language?: string): tokenizer.WordStream {\n return new BufferedWordStream(\n (text: string) => splitWords(text, this.#ignorePunctuation),\n 1,\n 1,\n );\n }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n return hyphenator.hyphenateWord(word);\n};\n\nexport { splitWords };\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,0BAA6E;AAC7E,gBAA2B;AAC3B,wBAA2B;AAC3B,uBAAgC;AAChC,sBAA+B;AAC/B,kBAA2B;AAwC3B,MAAM,0BAA4C;AAAA,EAChD,UAAU;AAAA,EACV,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,cAAc;AAChB;AAEO,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,SAAqC;AAC/C,UAAM;AACN,SAAK,UAAU;AAAA,MACb,GAAG;AAAA,MACH,GAAG;AAAA,IACL;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,UAAM,QAAQ,CAAC,cACb,gCAAe,OAAO,KAAK,QAAQ,mBAAmB,KAAK,QAAQ,YAAY;AACjF,UAAM,cAAc,KAAK,QAAQ,eAAW,sCAAiB,KAAK,IAAI;AACtE,WAAO,YAAY,IAAI,EAAE,IAAI,CAAC,QAAS,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,IAAI,GAAI;AAAA,EAC3E;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,aACC,gCAAe,MAAM,KAAK,QAAQ,mBAAmB,KAAK,QAAQ,YAAY;AAAA,MAChF,KAAK,QAAQ,kBAAkB,KAAK,QAAQ;AAAA,MAC5C,KAAK,QAAQ;AAAA,MACb;AAAA,QACE,gBAAgB,KAAK,QAAQ;AAAA,QAC7B,kBAAkB,KAAK,QAAQ;AAAA,QAC/B,UAAU,KAAK,QAAQ;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,wBAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,aAAiB,wBAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,6BAAW,cAAc,IAAI;AACtC;AAIO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,aAAO,kCAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}