{"version":3,"sources":["../../src/ipc/supervised_proc.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChildProcess } from 'node:child_process';\nimport { once } from 'node:events';\nimport pidusage from 'pidusage';\nimport type { RunningJobInfo } from '../job.js';\nimport { log, loggerOptions } from '../log.js';\nimport { Future } from '../utils.js';\nimport type { IPCMessage } from './message.js';\n\nconst MEMORY_MONITOR_INTERVAL = 5000;\nconst MEMORY_WARN_COOLDOWN = 120000;\nconst MEMORY_WARN_RESET_DELTA_MB = 50;\n\nexport interface ProcOpts {\n  /** Timeout for process initialization in milliseconds. */\n  initializeTimeout: number;\n  /** Timeout for process shutdown in milliseconds. */\n  closeTimeout: number;\n  /** Memory usage warning threshold in megabytes. */\n  memoryWarnMB: number;\n  /** Memory usage limit in megabytes. */\n  memoryLimitMB: number;\n  /** Interval for health check pings in milliseconds. */\n  pingInterval: number;\n  /** Timeout waiting for pong response in milliseconds. */\n  pingTimeout: number;\n  /** Threshold for warning about unresponsive processes in milliseconds. */\n  highPingThreshold: number;\n}\n\nexport abstract class SupervisedProc {\n  #opts: ProcOpts;\n  #started = false;\n  #closing = false;\n  #startedAt?: number;\n  #runningJob?: RunningJobInfo = undefined;\n  proc?: ChildProcess;\n  #pingInterval?: ReturnType<typeof setInterval>;\n  #memoryMonitorInterval?: ReturnType<typeof setInterval>;\n  #pongTimeout?: ReturnType<typeof setTimeout>;\n  private memoryBaselineMB?: number;\n  #lastMemoryWarnAt = 0;\n  #lastMemoryWarnMB = 0;\n  protected init = new Future();\n  #join = new Future();\n  #shutdownRequestAck = new Future();\n  #shuttingDown = new Future();\n  #closePromise?: Promise<void>;\n  #logger = log().child({ runningJob: this.#runningJob });\n\n  constructor(opts: ProcOpts) {\n    this.#opts = opts;\n  }\n\n  abstract createProcess(): ChildProcess;\n  abstract mainTask(child: ChildProcess): Promise<void>;\n  protected abstract get processKind(): string;\n\n  get started(): boolean {\n    return this.#started;\n  }\n\n  get isAlive(): boolean {\n    return this.#started && !this.#closing && !!this.proc?.connected;\n  }\n\n  get runningJob(): RunningJobInfo | undefined {\n    return this.#runningJob;\n  }\n\n  async start() {\n    if (this.#started) {\n      throw new Error('runner already started');\n    } else if (this.#closing) {\n      throw new Error('runner is closed');\n    }\n\n    this.proc = this.createProcess();\n\n    this.#started = true;\n    this.#startedAt = performance.now();\n    this.run().catch((error) => {\n      this.#logger.child({ error }).warn('supervised process run failed');\n      // Note: we intentionally do NOT kill the child process here. Killing it\n      // would race with initialize()'s `once(proc, 'message')`, causing\n      // initialize() to hang forever and deadlocking the caller (proc_pool).\n      // The child process is cleaned up when the pool shuts down.\n      this.#join.resolve();\n    });\n  }\n\n  async run() {\n    await this.init.await;\n\n    this.#pingInterval = setInterval(() => {\n      if (this.proc?.connected) {\n        this.proc.send({ case: 'pingRequest', value: { timestamp: Date.now() } });\n      }\n    }, this.#opts.pingInterval);\n\n    this.#pongTimeout = setTimeout(() => {\n      this.#logger.warn('job is unresponsive');\n      clearTimeout(this.#pongTimeout);\n      clearInterval(this.#pingInterval);\n      this.proc!.kill('SIGKILL');\n      this.#join.resolve();\n    }, this.#opts.pingTimeout);\n\n    const checkMemoryUsage = async (): Promise<void> => {\n      const memoryMB = await this.getChildMemoryUsageMB();\n      if (memoryMB === 0) {\n        return;\n      }\n\n      this.memoryBaselineMB ??= memoryMB;\n\n      if (this.#opts.memoryLimitMB > 0 && memoryMB > this.#opts.memoryLimitMB) {\n        this.#logger\n          .child(this.memoryLoggingFields(memoryMB))\n          .error(`${this.processKind} process exceeded memory limit, killing it`);\n        this.#closing = true;\n        this.clearTimers();\n        this.proc?.kill('SIGKILL');\n      } else if (this.#opts.memoryWarnMB > 0 && memoryMB > this.#opts.memoryWarnMB) {\n        if (this.shouldEmitMemoryWarning(memoryMB)) {\n          const advisory = this.#opts.memoryLimitMB <= 0;\n          this.#logger\n            .child(this.memoryLoggingFields(memoryMB))\n            .warn(\n              `${this.processKind} process memory usage is above the warning threshold${\n                advisory ? ' (advisory only, the process will not be terminated)' : ''\n              }`,\n            );\n        }\n      }\n    };\n\n    this.#memoryMonitorInterval = setInterval(() => {\n      void checkMemoryUsage().catch((error) => {\n        this.#logger.child({ error }).warn('failed to check supervised process memory usage');\n      });\n    }, MEMORY_MONITOR_INTERVAL);\n\n    const listener = (msg: IPCMessage) => {\n      switch (msg.case) {\n        case 'pongResponse': {\n          const delay = Date.now() - msg.value.timestamp;\n          if (delay > this.#opts.highPingThreshold) {\n            this.#logger.child({ delay }).warn('job executor is unresponsive');\n          }\n          this.#pongTimeout?.refresh();\n          break;\n        }\n        case 'exiting': {\n          this.#logger.child({ reason: msg.value.reason }).debug('job exiting');\n          break;\n        }\n        case 'shutdownRequestAck': {\n          if (!this.#shutdownRequestAck.done) this.#shutdownRequestAck.resolve();\n          break;\n        }\n        case 'shuttingDown': {\n          if (!this.#shuttingDown.done) this.#shuttingDown.resolve();\n          break;\n        }\n        case 'done': {\n          if (!this.#shutdownRequestAck.done) this.#shutdownRequestAck.resolve();\n          if (!this.#shuttingDown.done) this.#shuttingDown.resolve();\n          this.#closing = true;\n          this.proc!.off('message', listener);\n          break;\n        }\n      }\n    };\n    this.proc!.on('message', listener);\n    this.proc!.on('error', (error) => {\n      if (this.#closing) return;\n      this.#logger\n        .child({ error })\n        .warn('job process exited unexpectedly; this likely means the error above caused a crash');\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.proc!.on('exit', () => {\n      this.clearTimers();\n      this.#join.resolve();\n    });\n\n    this.mainTask(this.proc!);\n\n    await this.#join.await;\n  }\n\n  async join() {\n    if (!this.#started) {\n      throw new Error('runner not started');\n    }\n\n    await this.#join.await;\n  }\n\n  async initialize(options: { sessionEndTimeout?: number } = {}) {\n    if (!this.proc?.connected) {\n      const err = new Error('process not connected');\n      this.init.reject(err);\n      throw err;\n    }\n    this.proc.send({\n      case: 'initializeRequest',\n      value: {\n        loggerOptions: loggerOptions(),\n        pingInterval: this.#opts.pingInterval,\n        pingTimeout: this.#opts.pingTimeout,\n        highPingThreshold: this.#opts.highPingThreshold,\n        ...options,\n      },\n    });\n\n    // Race the first message against child exit and the timeout: a child that\n    // crashes before sending initializeResponse would otherwise leave the\n    // `once('message')` pending forever and hang the caller.\n    const abort = new AbortController();\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    const timeout = new Promise<never>((_, reject) => {\n      timer = setTimeout(\n        () => reject(new Error('runner initialization timed out')),\n        this.#opts.initializeTimeout,\n      );\n    });\n    const onMessage = once(this.proc!, 'message', { signal: abort.signal }).then(\n      ([msg]: IPCMessage[]) => {\n        if (msg!.case !== 'initializeResponse') {\n          throw new Error('first message must be InitializeResponse');\n        }\n      },\n    );\n    const onExit = once(this.proc!, 'exit', { signal: abort.signal }).then(([code, signal]) => {\n      throw new Error(`process exited before initializing (code ${code}, signal ${signal})`);\n    });\n\n    try {\n      await Promise.race([onMessage, onExit, timeout]);\n      this.init.resolve();\n    } catch (err) {\n      this.init.reject(err as Error);\n      // On timeout (or a bad first message) the child is still alive — kill it\n      // so a failed initialize doesn't leak the process.\n      if (this.proc && this.proc.exitCode === null && !this.proc.killed) {\n        this.proc.kill('SIGKILL');\n      }\n      throw err;\n    } finally {\n      clearTimeout(timer);\n      abort.abort();\n      // Swallow the AbortError from whichever listener lost the race.\n      onMessage.catch(() => {});\n      onExit.catch(() => {});\n    }\n  }\n\n  close(): Promise<void> {\n    if (!this.#started) {\n      return Promise.resolve();\n    }\n    this.#closePromise ??= this.closeOnce();\n    return this.#closePromise;\n  }\n\n  private async closeOnce(): Promise<void> {\n    this.#closing = true;\n    clearInterval(this.#memoryMonitorInterval);\n\n    const waitForShutdownProgress = async (\n      stage: Future,\n      timeout?: { ms: number; message: string },\n    ): Promise<boolean> => {\n      if (stage.done) return true;\n      if (this.#join.done) return false;\n\n      const stageOrExit =\n        stage === this.#join\n          ? stage.await.then(() => 'stage' as const)\n          : Promise.race([\n              stage.await.then(() => 'stage' as const),\n              this.#join.await.then(() => 'exit' as const),\n            ]);\n      if (!timeout) return (await stageOrExit) === 'stage';\n\n      let timer: ReturnType<typeof setTimeout> | undefined;\n      const result = await Promise.race([\n        stageOrExit,\n        new Promise<'timeout'>((resolve) => {\n          timer = setTimeout(() => resolve('timeout'), timeout.ms);\n        }),\n      ]);\n      clearTimeout(timer);\n\n      if (result === 'timeout') {\n        this.#logger.child({ timeout: timeout.ms }).error(timeout.message);\n        this.proc?.kill('SIGKILL');\n        await this.#join.await;\n      }\n\n      return result === 'stage';\n    };\n\n    try {\n      if (this.#join.done) return;\n\n      if (!this.proc?.connected) {\n        await waitForShutdownProgress(this.#join, {\n          ms: this.#opts.closeTimeout,\n          message: 'job shutdown is taking too much time',\n        });\n        return;\n      }\n\n      this.proc.send({\n        case: 'shutdownRequest',\n        value: { reason: 'shutdown request' },\n      } satisfies IPCMessage);\n\n      if (\n        !(await waitForShutdownProgress(this.#shutdownRequestAck, {\n          ms: this.#opts.closeTimeout,\n          message: 'job did not acknowledge shutdown in time',\n        }))\n      ) {\n        return;\n      }\n\n      if (!(await waitForShutdownProgress(this.#shuttingDown))) return;\n\n      await waitForShutdownProgress(this.#join, {\n        ms: this.#opts.closeTimeout,\n        message: 'job shutdown is taking too much time',\n      });\n    } finally {\n      this.clearTimers();\n    }\n  }\n\n  async launchJob(info: RunningJobInfo) {\n    if (this.#runningJob) {\n      throw new Error('executor already has a running job');\n    }\n    if (!this.proc?.connected) {\n      throw new Error('process not connected');\n    }\n    this.#runningJob = info;\n    this.proc.send({ case: 'startJobRequest', value: { runningJob: info } });\n  }\n\n  private async getChildMemoryUsageMB(): Promise<number> {\n    const pid = this.proc?.pid;\n    if (!pid) {\n      return 0;\n    }\n    try {\n      const stats = await pidusage(pid);\n      return stats.memory / (1024 * 1024);\n    } catch (err) {\n      const code = (err as NodeJS.ErrnoException).code;\n      if (code === 'ENOENT' || code === 'ESRCH') {\n        return 0;\n      }\n      throw err;\n    }\n  }\n\n  private get uptime(): number {\n    if (this.#startedAt === undefined) {\n      return 0;\n    }\n    return performance.now() - this.#startedAt;\n  }\n\n  private shouldEmitMemoryWarning(memoryMB: number, now: number = performance.now()): boolean {\n    const cooledDown = now - this.#lastMemoryWarnAt >= MEMORY_WARN_COOLDOWN;\n    const grew = memoryMB - this.#lastMemoryWarnMB >= MEMORY_WARN_RESET_DELTA_MB;\n    if (cooledDown || grew) {\n      this.#lastMemoryWarnAt = now;\n      this.#lastMemoryWarnMB = memoryMB;\n      return true;\n    }\n    return false;\n  }\n\n  private memoryLoggingFields(memoryMB: number): Record<string, unknown> {\n    const fields: Record<string, unknown> = {\n      pid: this.proc?.pid,\n      memoryUsageMB: Math.round(memoryMB * 10) / 10,\n      memoryWarnMB: this.#opts.memoryWarnMB,\n      memoryLimitMB: this.#opts.memoryLimitMB,\n      uptime: this.uptime,\n      hasRunningJob: this.runningJob !== undefined,\n    };\n\n    if (this.memoryBaselineMB !== undefined) {\n      fields.baselineMemoryMB = Math.round(this.memoryBaselineMB * 10) / 10;\n      fields.growthMemoryMB = Math.round((memoryMB - this.memoryBaselineMB) * 10) / 10;\n    }\n\n    return fields;\n  }\n\n  private clearTimers() {\n    clearTimeout(this.#pongTimeout);\n    clearInterval(this.#pingInterval);\n    clearInterval(this.#memoryMonitorInterval);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,yBAAqB;AACrB,sBAAqB;AAErB,iBAAmC;AACnC,mBAAuB;AAGvB,MAAM,0BAA0B;AAChC,MAAM,uBAAuB;AAC7B,MAAM,6BAA6B;AAmB5B,MAAe,eAAe;AAAA,EACnC;AAAA,EACA,WAAW;AAAA,EACX,WAAW;AAAA,EACX;AAAA,EACA,cAA+B;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACQ;AAAA,EACR,oBAAoB;AAAA,EACpB,oBAAoB;AAAA,EACV,OAAO,IAAI,oBAAO;AAAA,EAC5B,QAAQ,IAAI,oBAAO;AAAA,EACnB,sBAAsB,IAAI,oBAAO;AAAA,EACjC,gBAAgB,IAAI,oBAAO;AAAA,EAC3B;AAAA,EACA,cAAU,gBAAI,EAAE,MAAM,EAAE,YAAY,KAAK,YAAY,CAAC;AAAA,EAEtD,YAAY,MAAgB;AAC1B,SAAK,QAAQ;AAAA,EACf;AAAA,EAMA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AAhEzB;AAiEI,WAAO,KAAK,YAAY,CAAC,KAAK,YAAY,CAAC,GAAC,UAAK,SAAL,mBAAW;AAAA,EACzD;AAAA,EAEA,IAAI,aAAyC;AAC3C,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAQ;AACZ,QAAI,KAAK,UAAU;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C,WAAW,KAAK,UAAU;AACxB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,OAAO,KAAK,cAAc;AAE/B,SAAK,WAAW;AAChB,SAAK,aAAa,YAAY,IAAI;AAClC,SAAK,IAAI,EAAE,MAAM,CAAC,UAAU;AAC1B,WAAK,QAAQ,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,+BAA+B;AAKlE,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,MAAM;AACV,UAAM,KAAK,KAAK;AAEhB,SAAK,gBAAgB,YAAY,MAAM;AAhG3C;AAiGM,WAAI,UAAK,SAAL,mBAAW,WAAW;AACxB,aAAK,KAAK,KAAK,EAAE,MAAM,eAAe,OAAO,EAAE,WAAW,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,MAC1E;AAAA,IACF,GAAG,KAAK,MAAM,YAAY;AAE1B,SAAK,eAAe,WAAW,MAAM;AACnC,WAAK,QAAQ,KAAK,qBAAqB;AACvC,mBAAa,KAAK,YAAY;AAC9B,oBAAc,KAAK,aAAa;AAChC,WAAK,KAAM,KAAK,SAAS;AACzB,WAAK,MAAM,QAAQ;AAAA,IACrB,GAAG,KAAK,MAAM,WAAW;AAEzB,UAAM,mBAAmB,YAA2B;AA9GxD;AA+GM,YAAM,WAAW,MAAM,KAAK,sBAAsB;AAClD,UAAI,aAAa,GAAG;AAClB;AAAA,MACF;AAEA,WAAK,qBAAqB;AAE1B,UAAI,KAAK,MAAM,gBAAgB,KAAK,WAAW,KAAK,MAAM,eAAe;AACvE,aAAK,QACF,MAAM,KAAK,oBAAoB,QAAQ,CAAC,EACxC,MAAM,GAAG,KAAK,WAAW,4CAA4C;AACxE,aAAK,WAAW;AAChB,aAAK,YAAY;AACjB,mBAAK,SAAL,mBAAW,KAAK;AAAA,MAClB,WAAW,KAAK,MAAM,eAAe,KAAK,WAAW,KAAK,MAAM,cAAc;AAC5E,YAAI,KAAK,wBAAwB,QAAQ,GAAG;AAC1C,gBAAM,WAAW,KAAK,MAAM,iBAAiB;AAC7C,eAAK,QACF,MAAM,KAAK,oBAAoB,QAAQ,CAAC,EACxC;AAAA,YACC,GAAG,KAAK,WAAW,uDACjB,WAAW,yDAAyD,EACtE;AAAA,UACF;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAEA,SAAK,yBAAyB,YAAY,MAAM;AAC9C,WAAK,iBAAiB,EAAE,MAAM,CAAC,UAAU;AACvC,aAAK,QAAQ,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,iDAAiD;AAAA,MACtF,CAAC;AAAA,IACH,GAAG,uBAAuB;AAE1B,UAAM,WAAW,CAAC,QAAoB;AAjJ1C;AAkJM,cAAQ,IAAI,MAAM;AAAA,QAChB,KAAK,gBAAgB;AACnB,gBAAM,QAAQ,KAAK,IAAI,IAAI,IAAI,MAAM;AACrC,cAAI,QAAQ,KAAK,MAAM,mBAAmB;AACxC,iBAAK,QAAQ,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,8BAA8B;AAAA,UACnE;AACA,qBAAK,iBAAL,mBAAmB;AACnB;AAAA,QACF;AAAA,QACA,KAAK,WAAW;AACd,eAAK,QAAQ,MAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,CAAC,EAAE,MAAM,aAAa;AACpE;AAAA,QACF;AAAA,QACA,KAAK,sBAAsB;AACzB,cAAI,CAAC,KAAK,oBAAoB,KAAM,MAAK,oBAAoB,QAAQ;AACrE;AAAA,QACF;AAAA,QACA,KAAK,gBAAgB;AACnB,cAAI,CAAC,KAAK,cAAc,KAAM,MAAK,cAAc,QAAQ;AACzD;AAAA,QACF;AAAA,QACA,KAAK,QAAQ;AACX,cAAI,CAAC,KAAK,oBAAoB,KAAM,MAAK,oBAAoB,QAAQ;AACrE,cAAI,CAAC,KAAK,cAAc,KAAM,MAAK,cAAc,QAAQ;AACzD,eAAK,WAAW;AAChB,eAAK,KAAM,IAAI,WAAW,QAAQ;AAClC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,SAAK,KAAM,GAAG,WAAW,QAAQ;AACjC,SAAK,KAAM,GAAG,SAAS,CAAC,UAAU;AAChC,UAAI,KAAK,SAAU;AACnB,WAAK,QACF,MAAM,EAAE,MAAM,CAAC,EACf,KAAK,mFAAmF;AAC3F,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,KAAM,GAAG,QAAQ,MAAM;AAC1B,WAAK,YAAY;AACjB,WAAK,MAAM,QAAQ;AAAA,IACrB,CAAC;AAED,SAAK,SAAS,KAAK,IAAK;AAExB,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,UAAM,KAAK,MAAM;AAAA,EACnB;AAAA,EAEA,MAAM,WAAW,UAA0C,CAAC,GAAG;AA5MjE;AA6MI,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,YAAM,MAAM,IAAI,MAAM,uBAAuB;AAC7C,WAAK,KAAK,OAAO,GAAG;AACpB,YAAM;AAAA,IACR;AACA,SAAK,KAAK,KAAK;AAAA,MACb,MAAM;AAAA,MACN,OAAO;AAAA,QACL,mBAAe,0BAAc;AAAA,QAC7B,cAAc,KAAK,MAAM;AAAA,QACzB,aAAa,KAAK,MAAM;AAAA,QACxB,mBAAmB,KAAK,MAAM;AAAA,QAC9B,GAAG;AAAA,MACL;AAAA,IACF,CAAC;AAKD,UAAM,QAAQ,IAAI,gBAAgB;AAClC,QAAI;AACJ,UAAM,UAAU,IAAI,QAAe,CAAC,GAAG,WAAW;AAChD,cAAQ;AAAA,QACN,MAAM,OAAO,IAAI,MAAM,iCAAiC,CAAC;AAAA,QACzD,KAAK,MAAM;AAAA,MACb;AAAA,IACF,CAAC;AACD,UAAM,gBAAY,yBAAK,KAAK,MAAO,WAAW,EAAE,QAAQ,MAAM,OAAO,CAAC,EAAE;AAAA,MACtE,CAAC,CAAC,GAAG,MAAoB;AACvB,YAAI,IAAK,SAAS,sBAAsB;AACtC,gBAAM,IAAI,MAAM,0CAA0C;AAAA,QAC5D;AAAA,MACF;AAAA,IACF;AACA,UAAM,aAAS,yBAAK,KAAK,MAAO,QAAQ,EAAE,QAAQ,MAAM,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,MAAM,MAAM;AACzF,YAAM,IAAI,MAAM,4CAA4C,IAAI,YAAY,MAAM,GAAG;AAAA,IACvF,CAAC;AAED,QAAI;AACF,YAAM,QAAQ,KAAK,CAAC,WAAW,QAAQ,OAAO,CAAC;AAC/C,WAAK,KAAK,QAAQ;AAAA,IACpB,SAAS,KAAK;AACZ,WAAK,KAAK,OAAO,GAAY;AAG7B,UAAI,KAAK,QAAQ,KAAK,KAAK,aAAa,QAAQ,CAAC,KAAK,KAAK,QAAQ;AACjE,aAAK,KAAK,KAAK,SAAS;AAAA,MAC1B;AACA,YAAM;AAAA,IACR,UAAE;AACA,mBAAa,KAAK;AAClB,YAAM,MAAM;AAEZ,gBAAU,MAAM,MAAM;AAAA,MAAC,CAAC;AACxB,aAAO,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,QAAuB;AACrB,QAAI,CAAC,KAAK,UAAU;AAClB,aAAO,QAAQ,QAAQ;AAAA,IACzB;AACA,SAAK,kBAAkB,KAAK,UAAU;AACtC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAc,YAA2B;AA/Q3C;AAgRI,SAAK,WAAW;AAChB,kBAAc,KAAK,sBAAsB;AAEzC,UAAM,0BAA0B,OAC9B,OACA,YACqB;AAtR3B,UAAAA;AAuRM,UAAI,MAAM,KAAM,QAAO;AACvB,UAAI,KAAK,MAAM,KAAM,QAAO;AAE5B,YAAM,cACJ,UAAU,KAAK,QACX,MAAM,MAAM,KAAK,MAAM,OAAgB,IACvC,QAAQ,KAAK;AAAA,QACX,MAAM,MAAM,KAAK,MAAM,OAAgB;AAAA,QACvC,KAAK,MAAM,MAAM,KAAK,MAAM,MAAe;AAAA,MAC7C,CAAC;AACP,UAAI,CAAC,QAAS,QAAQ,MAAM,gBAAiB;AAE7C,UAAI;AACJ,YAAM,SAAS,MAAM,QAAQ,KAAK;AAAA,QAChC;AAAA,QACA,IAAI,QAAmB,CAAC,YAAY;AAClC,kBAAQ,WAAW,MAAM,QAAQ,SAAS,GAAG,QAAQ,EAAE;AAAA,QACzD,CAAC;AAAA,MACH,CAAC;AACD,mBAAa,KAAK;AAElB,UAAI,WAAW,WAAW;AACxB,aAAK,QAAQ,MAAM,EAAE,SAAS,QAAQ,GAAG,CAAC,EAAE,MAAM,QAAQ,OAAO;AACjE,SAAAA,MAAA,KAAK,SAAL,gBAAAA,IAAW,KAAK;AAChB,cAAM,KAAK,MAAM;AAAA,MACnB;AAEA,aAAO,WAAW;AAAA,IACpB;AAEA,QAAI;AACF,UAAI,KAAK,MAAM,KAAM;AAErB,UAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,cAAM,wBAAwB,KAAK,OAAO;AAAA,UACxC,IAAI,KAAK,MAAM;AAAA,UACf,SAAS;AAAA,QACX,CAAC;AACD;AAAA,MACF;AAEA,WAAK,KAAK,KAAK;AAAA,QACb,MAAM;AAAA,QACN,OAAO,EAAE,QAAQ,mBAAmB;AAAA,MACtC,CAAsB;AAEtB,UACE,CAAE,MAAM,wBAAwB,KAAK,qBAAqB;AAAA,QACxD,IAAI,KAAK,MAAM;AAAA,QACf,SAAS;AAAA,MACX,CAAC,GACD;AACA;AAAA,MACF;AAEA,UAAI,CAAE,MAAM,wBAAwB,KAAK,aAAa,EAAI;AAE1D,YAAM,wBAAwB,KAAK,OAAO;AAAA,QACxC,IAAI,KAAK,MAAM;AAAA,QACf,SAAS;AAAA,MACX,CAAC;AAAA,IACH,UAAE;AACA,WAAK,YAAY;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,MAAM,UAAU,MAAsB;AAzVxC;AA0VI,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AACA,QAAI,GAAC,UAAK,SAAL,mBAAW,YAAW;AACzB,YAAM,IAAI,MAAM,uBAAuB;AAAA,IACzC;AACA,SAAK,cAAc;AACnB,SAAK,KAAK,KAAK,EAAE,MAAM,mBAAmB,OAAO,EAAE,YAAY,KAAK,EAAE,CAAC;AAAA,EACzE;AAAA,EAEA,MAAc,wBAAyC;AApWzD;AAqWI,UAAM,OAAM,UAAK,SAAL,mBAAW;AACvB,QAAI,CAAC,KAAK;AACR,aAAO;AAAA,IACT;AACA,QAAI;AACF,YAAM,QAAQ,UAAM,gBAAAC,SAAS,GAAG;AAChC,aAAO,MAAM,UAAU,OAAO;AAAA,IAChC,SAAS,KAAK;AACZ,YAAM,OAAQ,IAA8B;AAC5C,UAAI,SAAS,YAAY,SAAS,SAAS;AACzC,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,IAAY,SAAiB;AAC3B,QAAI,KAAK,eAAe,QAAW;AACjC,aAAO;AAAA,IACT;AACA,WAAO,YAAY,IAAI,IAAI,KAAK;AAAA,EAClC;AAAA,EAEQ,wBAAwB,UAAkB,MAAc,YAAY,IAAI,GAAY;AAC1F,UAAM,aAAa,MAAM,KAAK,qBAAqB;AACnD,UAAM,OAAO,WAAW,KAAK,qBAAqB;AAClD,QAAI,cAAc,MAAM;AACtB,WAAK,oBAAoB;AACzB,WAAK,oBAAoB;AACzB,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAAoB,UAA2C;AAvYzE;AAwYI,UAAM,SAAkC;AAAA,MACtC,MAAK,UAAK,SAAL,mBAAW;AAAA,MAChB,eAAe,KAAK,MAAM,WAAW,EAAE,IAAI;AAAA,MAC3C,cAAc,KAAK,MAAM;AAAA,MACzB,eAAe,KAAK,MAAM;AAAA,MAC1B,QAAQ,KAAK;AAAA,MACb,eAAe,KAAK,eAAe;AAAA,IACrC;AAEA,QAAI,KAAK,qBAAqB,QAAW;AACvC,aAAO,mBAAmB,KAAK,MAAM,KAAK,mBAAmB,EAAE,IAAI;AACnE,aAAO,iBAAiB,KAAK,OAAO,WAAW,KAAK,oBAAoB,EAAE,IAAI;AAAA,IAChF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,cAAc;AACpB,iBAAa,KAAK,YAAY;AAC9B,kBAAc,KAAK,aAAa;AAChC,kBAAc,KAAK,sBAAsB;AAAA,EAC3C;AACF;","names":["_a","pidusage"]}