diff --git a/.changeset/pink-apes-decide.md b/.changeset/pink-apes-decide.md new file mode 100644 index 00000000..61886584 --- /dev/null +++ b/.changeset/pink-apes-decide.md @@ -0,0 +1,7 @@ +--- +"tiktoken": patch +"js-tiktoken": patch +"@dqbd/tiktoken": patch +--- + +Add gpt-5 models diff --git a/.github/workflows/build_js.yml b/.github/workflows/build_js.yml index 18235719..1e26e9c1 100644 --- a/.github/workflows/build_js.yml +++ b/.github/workflows/build_js.yml @@ -1,6 +1,6 @@ name: Build and Test JS/WASM -on: [push, pull_request, workflow_dispatch] +on: [pull_request, workflow_dispatch] concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} diff --git a/js/src/core.ts b/js/src/core.ts index e1aa1415..d09d6cde 100644 --- a/js/src/core.ts +++ b/js/src/core.ts @@ -318,7 +318,14 @@ export function getEncodingNameForModel(model: TiktokenModel) { case "gpt-4.1-nano": case "gpt-4.1-nano-2025-04-14": case "gpt-4.5-preview": - case "gpt-4.5-preview-2025-02-27": { + case "gpt-4.5-preview-2025-02-27": + case "gpt-5": + case "gpt-5-2025-08-07": + case "gpt-5-nano": + case "gpt-5-nano-2025-08-07": + case "gpt-5-mini": + case "gpt-5-mini-2025-08-07": + case "gpt-5-chat-latest": { return "o200k_base"; } default: diff --git a/tiktoken/model_to_encoding.json b/tiktoken/model_to_encoding.json index 1af5e9dd..7180527d 100644 --- a/tiktoken/model_to_encoding.json +++ b/tiktoken/model_to_encoding.json @@ -97,5 +97,12 @@ "gpt-4.1-nano": "o200k_base", "gpt-4.1-nano-2025-04-14": "o200k_base", "gpt-4.5-preview": "o200k_base", - "gpt-4.5-preview-2025-02-27": "o200k_base" + "gpt-4.5-preview-2025-02-27": "o200k_base", + "gpt-5": "o200k_base", + "gpt-5-2025-08-07": "o200k_base", + "gpt-5-nano": "o200k_base", + "gpt-5-nano-2025-08-07": "o200k_base", + "gpt-5-mini": "o200k_base", + "gpt-5-mini-2025-08-07": "o200k_base", + "gpt-5-chat-latest": "o200k_base" } diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index 6d978b60..dde55bfb 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -22,4 +22,10 @@ gloo-utils = { version = "0.1", features = ["serde"] } [features] default = ["inline"] -inline = [] \ No newline at end of file +inline = [] + +[package.metadata.wasm-pack.profile.profiling] +wasm-opt = ['-O', ''] + +[package.metadata.wasm-pack.profile.release] +wasm-opt = ['-O', '--enable-bulk-memory'] diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index 29a71952..295ac133 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -462,7 +462,14 @@ export type TiktokenModel = | "gpt-4.1-nano" | "gpt-4.1-nano-2025-04-14" | "gpt-4.5-preview" - | "gpt-4.5-preview-2025-02-27"; + | "gpt-4.5-preview-2025-02-27" + | "gpt-5" + | "gpt-5-2025-08-07" + | "gpt-5-nano" + | "gpt-5-nano-2025-08-07" + | "gpt-5-mini" + | "gpt-5-mini-2025-08-07" + | "gpt-5-chat-latest"; /** * @param {TiktokenModel} encoding @@ -578,6 +585,13 @@ pub fn encoding_for_model( "gpt-4.1-nano-2025-04-14" => Ok("o200k_base"), "gpt-4.5-preview" => Ok("o200k_base"), "gpt-4.5-preview-2025-02-27" => Ok("o200k_base"), + "gpt-5" => Ok("o200k_base"), + "gpt-5-2025-08-07" => Ok("o200k_base"), + "gpt-5-nano" => Ok("o200k_base"), + "gpt-5-nano-2025-08-07" => Ok("o200k_base"), + "gpt-5-mini" => Ok("o200k_base"), + "gpt-5-mini-2025-08-07" => Ok("o200k_base"), + "gpt-5-chat-latest" => Ok("o200k_base"), model => Err(JsError::new( format!("Invalid model: {}", model.to_string()).as_str(), )),