openpilot_comma/tinygrad_repo/examples/tinychat/tinychat-browser/index.js

window.TINYCHAT_ROOT = "/tinychat-browser/";
const queryParams = new URLSearchParams(window.location.search);
const normalizedParams = Object.fromEntries([...queryParams].map(([key, value]) => [key.toUpperCase(), value.toUpperCase()]));
window.BACKEND = (normalizedParams["BACKEND"] === "WASM") ? "WASM" : "WebGPU";
const isMobileAgent = /Mobi|Android|iPhone|iPad|iPod/i.test(navigator.userAgent);
const hasTouchScreen = 'ontouchstart' in window || navigator.maxTouchPoints > 0;
window.isMobile = isMobileAgent || hasTouchScreen;
if (window.isMobile) document.documentElement.classList.add('mobile'); // prevent annoying auto-zoom when entering prompt on mobile
// MODEL_BASE_URL is where the weights are hosted, WEBGPU_EXPORT is the JS-wrapped WebGPU code exported from tinygrad
window.PC_MODEL_BASE_URL = ".";
window.PC_WEBGPU_EXPORT = './net.js'
window.PC_MAX_CONTEXT = 1024;
window.MOBILE_MODEL_BASE_URL = ".";
window.MOBILE_WEBGPU_EXPORT = './net.js'
window.MOBILE_MAX_CONTEXT = 1024;

const tiktokenReady = (async () => {
  const { init, get_encoding, Tiktoken, load } = await import('./tiktoken.js');
  window.Tiktoken = Tiktoken;
  window.tiktokenInit = init;
  window.tiktokenGetEncoding = get_encoding;
  window.tiktokenLoad = load;
})();

async function getDevice() {
  let adapter;
  try {
    adapter = await navigator.gpu.requestAdapter();
    if (!adapter) {
      this.loadingMessage = "Loading WASM (WebGPU not enabled):";
      throw new Error("No WebGPU adapter found");
    }
  } catch(error) {
      this.loadingMessage = "Loading WASM (WebGPU not enabled):";
    throw error;
  }
  const requiredLimits = {};
  const maxBufferSize = 322122544;
  requiredLimits.maxStorageBufferBindingSize = maxBufferSize;
  requiredLimits.maxBufferSize = maxBufferSize;
  requiredLimits.maxComputeInvocationsPerWorkgroup = 512; // may need to vary based on what the WEBGPU backend produces
            
  try {
    return await adapter.requestDevice({ requiredLimits });
  } catch(error) {
    this.loadingMessage = "Loading WASM (WebGPU error):";
    throw error;
  }
};

// copied from examples/webgpu/stable_diffusion/index.html 
function initDb() {
  return new Promise((resolve, reject) => {
    let db;
    const request = indexedDB.open('tinydb', 1);
    request.onerror = (event) => {
      console.error('Database error:', event.target.error);
      resolve(null);
    };

    request.onsuccess = (event) => {
      db = event.target.result;
      console.log("Db initialized.");
      resolve(db);
    };

    request.onupgradeneeded = (event) => {
      db = event.target.result;
      if (!db.objectStoreNames.contains('tensors')) {
        db.createObjectStore('tensors', { keyPath: 'id' });
      }
    };
  });
}

// copied from examples/webgpu/stable_diffusion/index.html 
function readTensorFromDb(db, id) {
  return new Promise((resolve, reject) => {
    if (db == null) {
      resolve(null);
    }
            
      const transaction = db.transaction(['tensors'], 'readonly');
      const store = transaction.objectStore('tensors');
      const request = store.get(id);

      transaction.onabort = (event) => {
        console.log("Transaction error while reading tensor: " + event.target.error);
        resolve(null);
      };

      request.onsuccess = (event) => {
        const result = event.target.result;
        if (result) {
          resolve(result);
        } else {
          resolve(null);
        }
      };

      request.onerror = (event) => {
        console.error('Tensor retrieve failed: ', event.target.error);
        resolve(null);
      };
  });
}

function getAllKeysFromDb(db) {
  return new Promise((resolve, reject) => {
    if (db == null) {resolve([]);}
    const transaction = db.transaction(['tensors'], 'readonly');
    const store = transaction.objectStore('tensors');
    const request = store.getAllKeys();
    transaction.onabort = (event) => {
      console.log("Transaction error while reading IndexedDB keys: " + event.target.error);
      resolve([]);
    };
    request.onsuccess = function (event) {resolve(event.target.result);};
    request.onerror = (event) => {
      console.error('Retrieval of IndexedDB keys failed: ', event.target.error);
      resolve([]);
    };
  });
}

// modified from examples/webgpu/stable_diffusion/index.html 
function saveTensorToDb(db, id, tensor) {
  return readTensorFromDb(db, id).then((result) => {
    if (!result) {
      new Promise((resolve, reject) => {
        if (db == null) {
          resolve(null);
        }

        const transaction = db.transaction(['tensors'], 'readwrite');
        const store = transaction.objectStore('tensors');
        const request = store.put({ id: id, content: tensor });

        transaction.onabort = (event) => {
          console.log("Transaction error while saving tensor: " + event.target.error);
          resolve(null);
        };

        request.onsuccess = () => {
          console.log('Tensor saved successfully.');
          resolve();
        };

        request.onerror = (event) => {
          console.error('Tensor save failed:', event.target.error);
          resolve(null);
        };
      });
    } else {
      return null;
    }
  }).catch(()=> null);
}

function deleteTensorFromDb(db, id) {
  return new Promise((resolve, reject) => {
    if (db == null) {
      console.error("Database is not initialized.");
      resolve(null);
      return;
    }

    const transaction = db.transaction(['tensors'], 'readwrite');
    const store = transaction.objectStore('tensors');
    const request = store.delete(id);

    transaction.oncomplete = () => {
      console.log(`Tensor with ID '${id}' deleted successfully.`);
      resolve();
    };

    transaction.onerror = (event) => {
      console.error("Transaction error while deleting tensor:", event.target.error);
      resolve(null);
    };

    request.onerror = (event) => {
      console.error('Tensor deletion failed:', event.target.error);
      resolve(null);
    };

    request.onsuccess = () => {
      console.log(`Delete request for tensor with ID '${id}' succeeded.`);
    };
  });
}

function makeProgress(total) {
  let acc = 0;
  const ret = function progress(amount, message) {
    if (amount >= 0) { // allow updating message only
      acc += amount;
      const percentage = total ? Math.trunc((acc / total) * 100) : 0;
      document.querySelector('.progress').style.width = `${percentage}%`;
      document.getElementById('progress-percentage').textContent = `${percentage}%`;
    }
    if (message) {
      this.loadingMessage = message;
      document.getElementById('loading-message').textContent = this.loadingMessage;
    }
  }.bind(this);
  ret.total = total;
  return ret;
}

function sendMessageToWorker(worker, message) {
  return new Promise((resolve, reject) => {
    const onMessage = (event) => {
      resolve(event.data);
      worker.removeEventListener('message', onMessage);
      worker.removeEventListener('error', onError);
    };

    const onError = (error) => {
      reject(error);
      worker.removeEventListener('message', onMessage);
      worker.removeEventListener('error', onError);
    };

    worker.addEventListener('message', onMessage);
    worker.addEventListener('error', onError);

    if (message.header === "token") worker.postMessage(message.data);
    else if (message.header === "load_state_dict") {
      if (message.data === "done") worker.postMessage(message.data);
      else worker.postMessage(message.data, message.data.map(file => file.bytes.buffer));
    }
    else if (message.header === "init") worker.postMessage("init");
  });
}

async function load_state_dict (data, device, progress) {
  let state_dict = data.metadata.state_dict;
  let completed = 0;

  // modified from examples/webgpu/stable_diffusion/index.html getProgressDlForPart
  const loadPart = async (part) => {
      const response = await fetch(part);
      const res = new Response(new ReadableStream({
          async start(controller) {
              const reader = response.body.getReader();
              for (;;) {
                  const { done, value } = await reader.read();
                  if (done) break;
                  progress(value.byteLength);
                  controller.enqueue(value);
              }
              controller.close();
          },
      }));
        
      return res.arrayBuffer();
  };

  let db = await initDb();

  const getPart = async(filename, hash) => {
    let part = await readTensorFromDb(db, hash);

    if (part) {
      console.log(`Cache hit: ${filename}, hash: ${hash}`);
      progress(part.content.byteLength);
      return Promise.resolve(part.content);
    } else {
      console.log(`Cache miss: ${filename}, hash: ${hash}`);
      return loadPart(`${window.MODEL_BASE_URL}/${filename}`);
    }
  }

  const correctHashes = data.metadata.files.map(file => file.hash)
  // delete unused cached buffers to free disk space -- if we update weights, user will otherwise have obsolete cached buffers
  const dbKeys = await getAllKeysFromDb(db);
  const correctHashesSet = new Set(correctHashes);
  const notInCorrectHashes = dbKeys.filter(key => !correctHashesSet.has(key));
  // await these right before starting to save new stuff
  const deletionPromises = notInCorrectHashes.map(async (hash) => deleteTensorFromDb(db, hash));

  // instantiates empty weight buffers on WebGPU, attaches buffers to state_dict
  let model;
  if (window.BACKEND === "WebGPU") {
    //model = await transformer().setup(device, state_dict, progress);
    model = await transformer.setupNet(device, state_dict);
    progress(0.15 * progress.total);

  }
  else if (window.BACKEND === "WASM") {
    progress(0.02 * progress.total);
    model = new Worker(`./worker.js?version=${Date.now()}`);
    await sendMessageToWorker(model, {header: "init"});
    progress(0.02 * progress.total);
    progress(0.11 * progress.total);
  }

  const downloaded = [];
  const triggerChainDownload = async (toDownload) => {
    const numDownloaders = window.isMobile ? 4 : toDownload.length; // TODO: dynamically base this on DL file size? current assumption is 16 MiB chunks

    const chainDownload = async() => {
      const file = toDownload.shift();
      loadPart(`${window.MODEL_BASE_URL}/${file.name}`) // triggers download
      .then(async (arraybuf) => { 
        downloaded.push({ ...file, bytes: new Uint8Array(arraybuf)});
        // pause downloads if further processing is a bottleneck
        while (toDownload.length && downloaded.length >= numDownloaders) await new Promise(resolve => setTimeout(resolve, 5));
        if (toDownload.length && downloaded.length < numDownloaders) chainDownload(); // start next download
      })
    }
    for (let i=0; i<numDownloaders; i++) if (toDownload.length) chainDownload();
  }

  const loadFileToStateDict = async(file) => {
    if (window.BACKEND === "WebGPU") {
      for (const part of file.parts) {
        if (part.empty) continue;
        part.bytes = (part.size === file.bytes.length) ? file.bytes : file.bytes.slice(part.file_start_pos, part.file_start_pos + part.size);
        device.queue.writeBuffer(state_dict[part.key].bytes, part.target_start_pos, part.bytes); // improves stability over mappedAtCreation writing
        part.bytes = null;
      }
    }
    else if (window.BACKEND === "WASM") {
      await sendMessageToWorker(model, {header: "load_state_dict", data: [file]});
    }
    file.bytes = null;
  }

  if (window.BACKEND === "WebGPU") { // contiguous loading not needed for WebGPU stability
    const files = data.tensor_file_groups.flatMap(obj => obj.files);
    data.tensor_file_groups = [{contiguous: false, files: files}];
  }

  for (const group of data.tensor_file_groups) {
    const contiguous = group.contiguous;
    const files = group.files;
    const tensor_file_indices = files.map(file => file.index);
    const contiguousFiles = [];
    const fileHashes = new Set(files.map(file => file.hash));
    const cachedFileHashes = new Set(dbKeys.filter(key => fileHashes.has(key)));
    const cachedFiles = files.filter(file => cachedFileHashes.has(file.hash));
    const toDownload = files.filter(file => !cachedFileHashes.has(file.hash));
    triggerChainDownload(toDownload);

    const loadDelay = 5;
    await Promise.all(deletionPromises);

    while (completed < files.length) {
      const start = performance.now();
      // prioritize files from downloaded queue, so we can continue downloading more files
      if (downloaded.length) {
        const file = downloaded.shift();
        await saveTensorToDb(db, file.hash, file.bytes); // for wasm, must await to prevent race between indexedDB and transfer to worker
        if (!contiguous) await loadFileToStateDict(file);
        else contiguousFiles.push(file);
        completed += 1;
      }
      else if (!downloaded.length && cachedFiles.length) {
        const file = cachedFiles.shift();
        file.bytes = await getPart(file.name, file.hash); // reads data from IndexedDB
        if (!contiguous) await loadFileToStateDict(file);
        else contiguousFiles.push(file);
        completed += 1;
      }
      const end = performance.now();
      const elapsed = end - start;
      if (elapsed < loadDelay) await new Promise(resolve => setTimeout(resolve, loadDelay - elapsed));
    }
    if (contiguous) {
      const orderMap = tensor_file_indices.reduce((acc, id, index) => {acc[id] = index; return acc;}, {});
      contiguousFiles.sort((a, b) => orderMap[a.index] - orderMap[b.index]); // glue files together in the right order
      await sendMessageToWorker(model, {header: "load_state_dict", data: contiguousFiles});
    }
    completed = 0;
  }

  // initialize empty kv_caches, which were part of exported model's state_dict, but which we didn't want to package/download
  if (window.BACKEND === "WASM") {
    for (const [k, v] of Object.entries(state_dict).filter(([_, v]) => v.empty === true)) {
      v.parts[0].file_start_pos = 0;
      const file = { parts: v.parts, size: v.size, bytes: new Uint8Array(v.size).fill(0) };
      await loadFileToStateDict(file);
    }
  }

  return model;
};

document.addEventListener("alpine:init", () => {
  Alpine.data("state", () => ({
    // loadingMessage updates the user on page load progress, including weights download and decompression
    // if loadingMessage is not '', then prompt box will be hidden: this is default behavior on page load
    placeholderText: "Generating...",
    loadingMessage: `Loading ${window.BACKEND} model:`,
    // model
    nets: {},
    tokenizer: null,
    max_context: 1024,
    lastSeenToks: [],

    progress: null,

    async init() {
      var device = null;
      var webgpuErrorMessage = null;
      if (window.BACKEND === "WebGPU") {
        try {
          device = await getDevice.call(this);
          console.log("WebGPU device initialized");
        } catch (error) {
          window.BACKEND = "WASM";
          console.log(`error: ${error}\nFailed to launch WebGPU. Loading WASM model instead...`); // return;
          webgpuErrorMessage = this.loadingMessage;
        }
      }

      window.MODEL_BASE_URL = (window.BACKEND === "WebGPU" && !window.isMobile) ? window.PC_MODEL_BASE_URL : window.MOBILE_MODEL_BASE_URL;
      this.max_context = (window.BACKEND === "WebGPU" && !window.isMobile) ? window.PC_MAX_CONTEXT : window.MOBILE_MAX_CONTEXT;

      const kernelsReady = (async () => {
        if (window.BACKEND === "WASM") {var exports = await import(`./net_clang.js?version=${Date.now()}`);}
        else if (window.BACKEND === "WebGPU" && !window.isMobile) {var exports = await import(`${PC_WEBGPU_EXPORT}?version=${Date.now()}`);}
        else if (window.BACKEND === "WebGPU" && window.isMobile) {var exports = await import(`${MOBILE_WEBGPU_EXPORT}?version=${Date.now()}`);}
        self.transformer = exports.default;
      })();

      const response = await fetch(`${window.MODEL_BASE_URL}/net_metadata.json`);
      // TODO: cache metadata (and everything else, including tokenizer)
      // TODO: use service worker to reload page when offline
      const data = await response.json();
      data.metadata.files = data.metadata.files.map((file, index) => ({...file, index}));
      const state_dict = data.metadata.state_dict;

      /*
      - allocating memory to WASM on mobile has longstanding issues: https://github.com/WebAssembly/design/issues/1397

      - the below pattern, while yielding a succesfully-functioning model when it doesn't crash, causes regular crashes on iOS Safari (iphone 15 iOS 18.3):
        - call WASM malloc (to fit all tensors, or one per tensor) for all tensors up front, then load tensor byte chunks into the buffers in random order

      - the below pattern has been stable on iOS Safari (iphone 15 iOS 18.3):
        - call only one WASM malloc at a time before filling the allocated bytes, as small as possible (malloc up to 256 MiB has been tested)
        - fill the malloc'd memory in linear order from start to end (what has been tested is calling wasm.HEAPU8.set on 16 MiB chunks from start to end)
        - use ALLOW_MEMORY_GROWTH=1 in wasm compilation, minimize initial memory

      - additional considerations affecting loading design, for WASM:
        - it seems that copying bytes into wasm memory cannot be zero-copy without sharedarraybuffer, which isn't currently used due to increased hosting complexity
        - non-zero copies create memory pressure, which is not reliably capped because of lack of control over garbage collection
        - to minimize peak memory pressure if GC is delayed, we process (i.e. download + copy into WASM) large tensors (> 16 MiB) one at a time, in descending size order
      */
      data.tensor_file_groups = []; // see above: for WASM, limit processing of multi-file Tensors to one at a time, in descending order based on Tensor size
      const unsplit_tensors = [];
      const sortedEntries = Object.entries(state_dict).sort(([, objA], [, objB]) => objB.size - objA.size);

      let totalSize = 0;
      const seen = new Set();
      for (const [k,v] of sortedEntries) {
        const files_in_tensor = [];
        for (const part of v.parts) {
          part.key = k;
          if (part.empty) state_dict[k].empty = true; // assumes no other parts of this weight exist and are non-empty
          else {
            const file = data.metadata.files[part.file];
            if (!seen.has(file.index)) {
              seen.add(file.index);
              files_in_tensor.push(file);
            }
            totalSize += part.size;
            part.dtype = v.dtype;
            if (!data.metadata.files[part.file].parts) data.metadata.files[part.file].parts = [];
            data.metadata.files[part.file].size ??= 0;
            data.metadata.files[part.file].size += part.size;
            data.metadata.files[part.file].parts.push(part);
          }
        }
        if (files_in_tensor.length > 1) data.tensor_file_groups.push({contiguous: true, files: files_in_tensor}); // [tensorN_file0, tensorN_file1, ...]
        else if (files_in_tensor.length > 0) unsplit_tensors.push(files_in_tensor[0]);
      }
      data.tensor_file_groups.push({contiguous: false, files: unsplit_tensors});

      data.totalSize = totalSize;
      totalSize = totalSize / 0.8; // give space in progress bar for initializing model bufs, and tokenizer
      this.progress = makeProgress.call(this, totalSize); // creates closure with totalSize

      try {
        this.progress(0.01 * totalSize, "Loading tokenizer:");
        const wasmResponse = await fetch(`${window.MODEL_BASE_URL}/tiktoken_bg.wasm`);
        this.progress(0.01 * totalSize);
        const wasmBytes = await wasmResponse.arrayBuffer();
        await tiktokenReady;
        await window.tiktokenInit((imports) => WebAssembly.instantiate(wasmBytes, imports));
        this.progress(0.01 * totalSize);

        this.tokenizer = await createTokenizer(`${window.MODEL_BASE_URL}/llama3-2.tiktoken`);
        const tokenizer_works = (new TextDecoder().decode(this.tokenizer.decode(this.tokenizer.encode("hello world"))) === "hello world");
        console.log("tokenizer works:", tokenizer_works)
        this.progress(0.01 * totalSize);
      } catch (error) {this.progress(-1, `Error launching tokenizer: ${error}`); console.log(error); return;}

      try {
        const loadModelMessage = (webgpuErrorMessage) ? webgpuErrorMessage : `Loading ${window.BACKEND} model:`
        this.progress(0, loadModelMessage);
        await kernelsReady;
        const model = await load_state_dict(data, device, this.progress);

        if (window.BACKEND === "WebGPU") {
          this.nets = {"transformer": model};
        }
        else if (window.BACKEND === "WASM") {
          const msg = await sendMessageToWorker(model, {header: "load_state_dict", data: "done"});
          this.nets = {"transformer": async (tok, start_pos) => sendMessageToWorker(model, {header: "token", data: [tok, start_pos]})};
        }
        this.progress(0.01 * totalSize, `Launching ${window.BACKEND} model:`);
        this.loadingMessage = ""; // Triggers removal of loading bar, display of prompt box
      } catch (error) {this.progress(-1, `Error launching model: ${error}`); console.log(error); return;}
    },

    // current state
    cstate: {
      time: null,
      messages: [],
    },

    // historical state
    histories: JSON.parse(localStorage.getItem("histories")) || [],

    home: 0,
    generating: false,
    maxContextReached: false,
    cancelGeneration: false,
    endpoint: `${window.location.origin}/v1`,

    // performance tracking
    time_till_first: 0,
    tokens_per_second: 0,
    total_tokens: 0,
    max_context: 0,

    removeHistory(cstate) {
      const index = this.histories.findIndex((state) => {
        return state.time === cstate.time;
      });
      if (index !== -1) {
        this.histories.splice(index, 1);
        localStorage.setItem("histories", JSON.stringify(this.histories));
      }
    },

    async handleSend() {
      const el = document.getElementById("input-form");
      const value = el.value.trim();
      if (!value) return;

      if (this.generating) return;
      this.maxContextReached = false;
      this.placeholderText = "Generating...";
      this.generating = true;
      this.cancelGeneration = false;
      if (this.home === 0) this.home = 1;

      // ensure that going back in history will go back to home
      window.history.pushState({}, "", window.TINYCHAT_ROOT || "/");

      // add message to list
      this.cstate.messages.push({ role: "user", content: value });

      // clear textarea
      el.value = "";
      el.style.height = "auto";
      el.style.height = el.scrollHeight + "px";

      // reset performance tracking
      const prefill_start = Date.now();
      let start_time = 0;
      let tokens = 0;
      this.tokens_per_second = 0;

      let gottenFirstChunk = false;
      try {
        for await (
          const chunk of this.openaiChatCompletion(this.cstate.messages)
        ) {
          if (!gottenFirstChunk) {
            this.cstate.messages.push({ role: "assistant", content: "" });
            gottenFirstChunk = true;
          }

          // add chunk to the last message
          // TODO: handle localStorage overflow
          //   possible example: this.cstate.messages[...] was undefined when trying to prompt within an old cstate (chat session)
          this.cstate.messages[this.cstate.messages.length - 1].content += chunk;

          // calculate performance tracking
          tokens += 1;
          this.total_tokens += 1;
          if (start_time === 0) {
            start_time = Date.now();
            this.time_till_first = start_time - prefill_start;
          } else {
            const diff = Date.now() - start_time;
            if (diff > 0) {
              this.tokens_per_second = tokens / (diff / 1000);
            }
          }
          this.checkMaxContext(this.total_tokens);
          if (this.cancelGeneration) break;
        }
      } finally {
        // update the state in histories or add it if it doesn't exist
        const index = this.histories.findIndex((cstate) => {
          return cstate.time === this.cstate.time;
        });
        this.cstate.time = Date.now();
        if (index !== -1) {
          // update the time
          this.histories[index] = this.cstate;
        } else {
          this.histories.push(this.cstate);
        }
        // update in local storage
        localStorage.setItem("histories", JSON.stringify(this.histories));

        if (!this.maxContextReached) this.generating = false;
        if (this.cancelGeneration && !this.maxContextReached) this.cstate = { time: null, messages: [] };
      }
    },

    async handleEnter(event) {
      // if shift is not pressed
      if (!event.shiftKey) {
        event.preventDefault();
        await this.handleSend();
      }
    },

    updateTotalTokens(messages) {
      try {
        let toks = [this.tokenizer.bos_id];
        messages.forEach((message) => {
          if (!message.role || !message.content) {
            throw new Error("Each message must have a 'role' and 'content' property.");
          }
          toks = toks.concat(this.tokenizer.encodeMessage(message.role, message.content));

          if (messages.length > 0 && messages[messages.length - 1].role === "user") {
            toks = toks.concat(this.tokenizer.encodeRole("assistant"));
          }
          this.total_tokens = toks.length;
        });
      } catch (error) {
        console.error("Error updating total tokens:", error);
      }
    },

    checkMaxContext(num_tokens) {
      if (num_tokens >= this.max_context) {
        this.cancelGeneration = true;
        this.maxContextReached = true;
        this.placeholderText = `Max context reached: ${this.max_context} tokens`;
      }
    },

    async *openaiChatCompletion(messages) {
      let tokens = [this.tokenizer.bos_id];
      for (const message of messages) {
        tokens = tokens.concat(this.tokenizer.encodeMessage(message.role, message.content));
      }
      tokens = tokens.concat(this.tokenizer.encodeRole("assistant"));
      this.checkMaxContext(tokens.length); // don't waste time prefilling if we know we're over the token limit
      let startPos = 0
      const prefillToks = tokens.slice(0, -1);

      // Skip the largest possible sequence of tokens already represented at the beginning of the model's kv caches
      for (let i=0; i <= prefillToks.length; i++) {
        startPos = i;
        if (i == prefillToks.length) break;
        if (i == this.lastSeenToks.length) break;
        if (prefillToks[i] !== this.lastSeenToks[i]) break;
      }
      //this.lastSeenToks = prefillToks;
      //prefillToks = prefillToks.slice(startPos);
      const unprocessedPrefillToks = prefillToks.slice(startPos);
      this.lastSeenToks = prefillToks.slice(0, startPos);

      this.progress = makeProgress(unprocessedPrefillToks.length);
      this.loadingMessage = (window.BACKEND === "WebGPU") ? "Reading input:" : "Loading (enable WebGPU for speed):";
      this.progress(0, this.loadingMessage);
      for (const tok of unprocessedPrefillToks) {
        if (this.cancelGeneration) {this.loadingMessage=""; return;}
        if (window.BACKEND === "WebGPU") {await this.nets["transformer"](new Int32Array([tok]), new Int32Array([startPos]));}
        else {await this.nets["transformer"](tok, startPos);}
        this.lastSeenToks.push(tok)
        startPos += 1;
        this.progress(1);
      }
      this.loadingMessage = ""; // hides progress bar

      let lastTok = tokens[tokens.length - 1];
      while (true) {
        if (window.BACKEND === "WebGPU") {var tok = await this.nets["transformer"](new Int32Array([lastTok]), new Int32Array([startPos])); tok = tok[0][0];}
        else {var tok = await this.nets["transformer"](lastTok, startPos);}
        this.lastSeenToks.push(lastTok); // lets us skip prefilling with these tokens at the next prompt in this chain
        startPos += 1;
        lastTok = tok;
        if (this.tokenizer.stop_tokens.has(lastTok)) break;
        yield new TextDecoder().decode(this.tokenizer.decode([lastTok]));
      }
    },
  }));
});

const { markedHighlight } = globalThis.markedHighlight;
marked.use(markedHighlight({
  langPrefix: "hljs language-",
  highlight(code, lang, _info) {
    const language = hljs.getLanguage(lang) ? lang : "plaintext";
    return hljs.highlight(code, { language }).value;
  },
}));

// **** eventsource-parser ****
class EventSourceParserStream extends TransformStream {
  constructor() {
    let parser;

    super({
      start(controller) {
        parser = createParser((event) => {
          if (event.type === "event") {
            controller.enqueue(event);
          }
        });
      },

      transform(chunk) {
        parser.feed(chunk);
      },
    });
  }
}

function createParser(onParse) {
  let isFirstChunk;
  let buffer;
  let startingPosition;
  let startingFieldLength;
  let eventId;
  let eventName;
  let data;
  reset();
  return {
    feed,
    reset,
  };
  function reset() {
    isFirstChunk = true;
    buffer = "";
    startingPosition = 0;
    startingFieldLength = -1;
    eventId = void 0;
    eventName = void 0;
    data = "";
  }
  function feed(chunk) {
    buffer = buffer ? buffer + chunk : chunk;
    if (isFirstChunk && hasBom(buffer)) {
      buffer = buffer.slice(BOM.length);
    }
    isFirstChunk = false;
    const length = buffer.length;
    let position = 0;
    let discardTrailingNewline = false;
    while (position < length) {
      if (discardTrailingNewline) {
        if (buffer[position] === "\n") {
          ++position;
        }
        discardTrailingNewline = false;
      }
      let lineLength = -1;
      let fieldLength = startingFieldLength;
      let character;
      for (
        let index = startingPosition;
        lineLength < 0 && index < length;
        ++index
      ) {
        character = buffer[index];
        if (character === ":" && fieldLength < 0) {
          fieldLength = index - position;
        } else if (character === "\r") {
          discardTrailingNewline = true;
          lineLength = index - position;
        } else if (character === "\n") {
          lineLength = index - position;
        }
      }
      if (lineLength < 0) {
        startingPosition = length - position;
        startingFieldLength = fieldLength;
        break;
      } else {
        startingPosition = 0;
        startingFieldLength = -1;
      }
      parseEventStreamLine(buffer, position, fieldLength, lineLength);
      position += lineLength + 1;
    }
    if (position === length) {
      buffer = "";
    } else if (position > 0) {
      buffer = buffer.slice(position);
    }
  }
  function parseEventStreamLine(lineBuffer, index, fieldLength, lineLength) {
    if (lineLength === 0) {
      if (data.length > 0) {
        onParse({
          type: "event",
          id: eventId,
          event: eventName || void 0,
          data: data.slice(0, -1),
          // remove trailing newline
        });

        data = "";
        eventId = void 0;
      }
      eventName = void 0;
      return;
    }
    const noValue = fieldLength < 0;
    const field = lineBuffer.slice(
      index,
      index + (noValue ? lineLength : fieldLength),
    );
    let step = 0;
    if (noValue) {
      step = lineLength;
    } else if (lineBuffer[index + fieldLength + 1] === " ") {
      step = fieldLength + 2;
    } else {
      step = fieldLength + 1;
    }
    const position = index + step;
    const valueLength = lineLength - step;
    const value = lineBuffer.slice(position, position + valueLength).toString();
    if (field === "data") {
      data += value ? "".concat(value, "\n") : "\n";
    } else if (field === "event") {
      eventName = value;
    } else if (field === "id" && !value.includes("\0")) {
      eventId = value;
    } else if (field === "retry") {
      const retry = parseInt(value, 10);
      if (!Number.isNaN(retry)) {
        onParse({
          type: "reconnect-interval",
          value: retry,
        });
      }
    }
  }
}
const BOM = [239, 187, 191];
function hasBom(buffer) {
  return BOM.every((charCode, index) => buffer.charCodeAt(index) === charCode);
}

const PAT_STR = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+";

async function createTokenizer(bpeUrl) {
  const num_base_tokens = 128000;
  const special_tokens = {
    "<|begin_of_text|>": 128000,
    "<|end_of_text|>": 128001,
    "<|start_header_id|>": 128006,
    "<|end_header_id|>": 128007,
    "<|eot_id|>": 128009
  };
  const model = await window.tiktokenLoad({
        "load_tiktoken_bpe": bpeUrl,
        "special_tokens": special_tokens,
        "pat_str": PAT_STR
    });
  const tokenizer = new window.Tiktoken(model.bpe_ranks, model.special_tokens, model.pat_str)

  return {
    get bos_id() {
      return special_tokens["<|begin_of_text|>"];
    },

    get stop_tokens() {
      return new Set([
        special_tokens["<|end_of_text|>"],
        special_tokens["<|eot_id|>"],
      ]);
    },

    decode(toks) {
      const filtered = toks.filter((t) => t < num_base_tokens);
      return tokenizer.decode(filtered);
    },

    encode(text, allow_special = false) {
      const allowedSpecial = allow_special ? "all" : new Set();
      const disallowedSpecial = new Set();
      return tokenizer.encode(text, allowedSpecial, disallowedSpecial);
    },

    encodeRole(role) {
      const tokens = [];
      tokens.push(special_tokens["<|start_header_id|>"]);
      tokens.push(...this.encode(role));
      tokens.push(special_tokens["<|end_header_id|>"]);
      tokens.push(...this.encode("\n\n"));
      return tokens;
    },

    encodeMessage(role, content) {
      const roleTokens = this.encodeRole(role);
      const contentTokens = this.encode(content.trim());
      return [...roleTokens, ...contentTokens, special_tokens["<|eot_id|>"]];
    },
  };
}