From 5775a4fc518dd8f0e4a06b00fd27120838c30598 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Sat, 9 Aug 2025 02:36:14 +0200 Subject: [PATCH 1/4] feat: add missing gpt-5 models --- .changeset/pink-apes-decide.md | 7 +++++++ js/src/core.ts | 9 ++++++++- tiktoken/model_to_encoding.json | 9 ++++++++- wasm/src/lib.rs | 16 +++++++++++++++- 4 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 .changeset/pink-apes-decide.md diff --git a/.changeset/pink-apes-decide.md b/.changeset/pink-apes-decide.md new file mode 100644 index 00000000..61886584 --- /dev/null +++ b/.changeset/pink-apes-decide.md @@ -0,0 +1,7 @@ +--- +"tiktoken": patch +"js-tiktoken": patch +"@dqbd/tiktoken": patch +--- + +Add gpt-5 models diff --git a/js/src/core.ts b/js/src/core.ts index e1aa1415..d09d6cde 100644 --- a/js/src/core.ts +++ b/js/src/core.ts @@ -318,7 +318,14 @@ export function getEncodingNameForModel(model: TiktokenModel) { case "gpt-4.1-nano": case "gpt-4.1-nano-2025-04-14": case "gpt-4.5-preview": - case "gpt-4.5-preview-2025-02-27": { + case "gpt-4.5-preview-2025-02-27": + case "gpt-5": + case "gpt-5-2025-08-07": + case "gpt-5-nano": + case "gpt-5-nano-2025-08-07": + case "gpt-5-mini": + case "gpt-5-mini-2025-08-07": + case "gpt-5-chat-latest": { return "o200k_base"; } default: diff --git a/tiktoken/model_to_encoding.json b/tiktoken/model_to_encoding.json index 1af5e9dd..7180527d 100644 --- a/tiktoken/model_to_encoding.json +++ b/tiktoken/model_to_encoding.json @@ -97,5 +97,12 @@ "gpt-4.1-nano": "o200k_base", "gpt-4.1-nano-2025-04-14": "o200k_base", "gpt-4.5-preview": "o200k_base", - "gpt-4.5-preview-2025-02-27": "o200k_base" + "gpt-4.5-preview-2025-02-27": "o200k_base", + "gpt-5": "o200k_base", + "gpt-5-2025-08-07": "o200k_base", + "gpt-5-nano": "o200k_base", + "gpt-5-nano-2025-08-07": "o200k_base", + "gpt-5-mini": "o200k_base", + "gpt-5-mini-2025-08-07": "o200k_base", + "gpt-5-chat-latest": "o200k_base" } diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index 29a71952..295ac133 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -462,7 +462,14 @@ export type TiktokenModel = | "gpt-4.1-nano" | "gpt-4.1-nano-2025-04-14" | "gpt-4.5-preview" - | "gpt-4.5-preview-2025-02-27"; + | "gpt-4.5-preview-2025-02-27" + | "gpt-5" + | "gpt-5-2025-08-07" + | "gpt-5-nano" + | "gpt-5-nano-2025-08-07" + | "gpt-5-mini" + | "gpt-5-mini-2025-08-07" + | "gpt-5-chat-latest"; /** * @param {TiktokenModel} encoding @@ -578,6 +585,13 @@ pub fn encoding_for_model( "gpt-4.1-nano-2025-04-14" => Ok("o200k_base"), "gpt-4.5-preview" => Ok("o200k_base"), "gpt-4.5-preview-2025-02-27" => Ok("o200k_base"), + "gpt-5" => Ok("o200k_base"), + "gpt-5-2025-08-07" => Ok("o200k_base"), + "gpt-5-nano" => Ok("o200k_base"), + "gpt-5-nano-2025-08-07" => Ok("o200k_base"), + "gpt-5-mini" => Ok("o200k_base"), + "gpt-5-mini-2025-08-07" => Ok("o200k_base"), + "gpt-5-chat-latest" => Ok("o200k_base"), model => Err(JsError::new( format!("Invalid model: {}", model.to_string()).as_str(), )), From 68057bcde882a7775c93ef179fba98a5bb109d07 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Sat, 9 Aug 2025 02:39:46 +0200 Subject: [PATCH 2/4] Fix CI --- .github/workflows/build_js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_js.yml b/.github/workflows/build_js.yml index 18235719..1e26e9c1 100644 --- a/.github/workflows/build_js.yml +++ b/.github/workflows/build_js.yml @@ -1,6 +1,6 @@ name: Build and Test JS/WASM -on: [push, pull_request, workflow_dispatch] +on: [pull_request, workflow_dispatch] concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} From eb2aa9d28b41b635a5f8c10777bd828c1d885cce Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Sat, 9 Aug 2025 02:41:09 +0200 Subject: [PATCH 3/4] Add missing flag --- wasm/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wasm/package.json b/wasm/package.json index 5389836b..8b95ac40 100644 --- a/wasm/package.json +++ b/wasm/package.json @@ -9,8 +9,8 @@ "build:wasm": "run-s wasm:*", "build:postprocess": "tsx scripts/postprocess.ts", "build:alias": "tsx scripts/alias.ts", - "wasm:bundler": "wasm-pack build --target bundler --weak-refs --release --out-dir dist && rm -rf dist/.gitignore dist/README.md dist/package.json", - "wasm:lite": "wasm-pack build --target bundler --weak-refs --release --out-dir dist/lite --no-default-features && rm -rf dist/lite/.gitignore dist/lite/README.md dist/lite/package.json", + "wasm:bundler": "wasm-pack build --enable-bulk-memory --target bundler --weak-refs --release --out-dir dist && rm -rf dist/.gitignore dist/README.md dist/package.json", + "wasm:lite": "wasm-pack build --enable-bulk-memory --target bundler --weak-refs --release --out-dir dist/lite --no-default-features && rm -rf dist/lite/.gitignore dist/lite/README.md dist/lite/package.json", "test": "yarn vitest run" }, "repository": { From e9865428b866fa9457e6629917836c1c1ba0f2dc Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Sat, 9 Aug 2025 02:43:32 +0200 Subject: [PATCH 4/4] Maybe? --- wasm/Cargo.toml | 8 +++++++- wasm/package.json | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index 6d978b60..dde55bfb 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -22,4 +22,10 @@ gloo-utils = { version = "0.1", features = ["serde"] } [features] default = ["inline"] -inline = [] \ No newline at end of file +inline = [] + +[package.metadata.wasm-pack.profile.profiling] +wasm-opt = ['-O', ''] + +[package.metadata.wasm-pack.profile.release] +wasm-opt = ['-O', '--enable-bulk-memory'] diff --git a/wasm/package.json b/wasm/package.json index 8b95ac40..5389836b 100644 --- a/wasm/package.json +++ b/wasm/package.json @@ -9,8 +9,8 @@ "build:wasm": "run-s wasm:*", "build:postprocess": "tsx scripts/postprocess.ts", "build:alias": "tsx scripts/alias.ts", - "wasm:bundler": "wasm-pack build --enable-bulk-memory --target bundler --weak-refs --release --out-dir dist && rm -rf dist/.gitignore dist/README.md dist/package.json", - "wasm:lite": "wasm-pack build --enable-bulk-memory --target bundler --weak-refs --release --out-dir dist/lite --no-default-features && rm -rf dist/lite/.gitignore dist/lite/README.md dist/lite/package.json", + "wasm:bundler": "wasm-pack build --target bundler --weak-refs --release --out-dir dist && rm -rf dist/.gitignore dist/README.md dist/package.json", + "wasm:lite": "wasm-pack build --target bundler --weak-refs --release --out-dir dist/lite --no-default-features && rm -rf dist/lite/.gitignore dist/lite/README.md dist/lite/package.json", "test": "yarn vitest run" }, "repository": {