{"version":3,"sources":["../../src/tokenize/tokenizer.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AsyncIterableQueue } from '../utils.js';\n\n// prettier-ignore\nexport const PUNCTUATIONS = [\n  '!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=',\n  '>', '?', '@', '[', '\\\\', ']', '^', '_', '`', '{', '|', '}', '~', '±', '—', '‘', '’', '“', '”',\n  '…',\n]\n\nexport interface TokenData {\n  segmentId: string;\n  token: string;\n}\n\nexport abstract class SentenceTokenizer {\n  abstract tokenize(text: string, language?: string): string[];\n\n  /**\n   * Returns a {@link SentenceStream} that can be used to push strings and receive smaller segments.\n   */\n  abstract stream(): SentenceStream;\n}\n\nexport abstract class SentenceStream {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<string | typeof SentenceStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<TokenData>();\n  #closed = false;\n\n  get closed(): boolean {\n    return this.#closed;\n  }\n\n  /** Push a string of text to the tokenizer */\n  pushText(text: string) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the tokenizer, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SentenceStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the tokenizer stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.#closed = true;\n  }\n\n  [Symbol.asyncIterator](): SentenceStream {\n    return this;\n  }\n}\n\nexport abstract class WordTokenizer {\n  abstract tokenize(text: string, language?: string): string[];\n\n  /**\n   * Returns a {@link WordStream} that can be used to push words and receive smaller segments.\n   */\n  abstract stream(): WordStream;\n}\n\nexport abstract class WordStream {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<string | typeof WordStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<TokenData>();\n  #closed = false;\n\n  get closed(): boolean {\n    return this.#closed;\n  }\n\n  /** Push a string of text to the tokenizer */\n  pushText(text: string) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the tokenizer, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(WordStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the tokenizer stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.#closed = true;\n  }\n\n  [Symbol.asyncIterator](): WordStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,mBAAmC;AAG5B,MAAM,eAAe;AAAA,EAC1B;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAC1F;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAM;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAC3F;AACF;AAOO,MAAe,kBAAkB;AAOxC;AAEO,MAAe,eAAe;AAAA,EACnC,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,gCAAkE;AAAA,EAC9E,QAAQ,IAAI,gCAA8B;AAAA,EACpD,UAAU;AAAA,EAEV,IAAI,SAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,eAAe,cAAc;AAAA,EAC9C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAoB;AACvC,WAAO;AAAA,EACT;AACF;AAEO,MAAe,cAAc;AAOpC;AAEO,MAAe,WAAW;AAAA,EAC/B,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,gCAA8D;AAAA,EAC1E,QAAQ,IAAI,gCAA8B;AAAA,EACpD,UAAU;AAAA,EAEV,IAAI,SAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,WAAW,cAAc;AAAA,EAC1C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAgB;AACnC,WAAO;AAAA,EACT;AACF;","names":[]}