From fa52d4b604e848966deeab504a37a603798c4bd8 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Tue, 22 Apr 2025 10:56:00 +0200 Subject: [PATCH 1/2] feat: add o3, o4, gpt-4.1 and missing specialised / pro models --- js/src/core.ts | 29 ++++++++++++++-- tiktoken/model_to_encoding.json | 30 +++++++++++++++-- wasm/src/lib.rs | 60 ++++++++++++++++++++++++++++++--- 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/js/src/core.ts b/js/src/core.ts index 90bde7a3..e1aa1415 100644 --- a/js/src/core.ts +++ b/js/src/core.ts @@ -282,18 +282,43 @@ export function getEncodingNameForModel(model: TiktokenModel) { case "gpt-4o-2024-11-20": case "gpt-4o-mini-2024-07-18": case "gpt-4o-mini": + case "gpt-4o-search-preview": + case "gpt-4o-search-preview-2025-03-11": + case "gpt-4o-mini-search-preview": + case "gpt-4o-mini-search-preview-2025-03-11": + case "gpt-4o-audio-preview": + case "gpt-4o-audio-preview-2024-12-17": + case "gpt-4o-audio-preview-2024-10-01": + case "gpt-4o-mini-audio-preview": + case "gpt-4o-mini-audio-preview-2024-12-17": case "o1": case "o1-2024-12-17": case "o1-mini": + case "o1-mini-2024-09-12": case "o1-preview": case "o1-preview-2024-09-12": - case "o1-mini-2024-09-12": + case "o1-pro": + case "o1-pro-2025-03-19": + case "o3": + case "o3-2025-04-16": case "o3-mini": case "o3-mini-2025-01-31": + case "o4-mini": + case "o4-mini-2025-04-16": case "chatgpt-4o-latest": case "gpt-4o-realtime": case "gpt-4o-realtime-preview-2024-10-01": - { + case "gpt-4o-realtime-preview-2024-12-17": + case "gpt-4o-mini-realtime-preview": + case "gpt-4o-mini-realtime-preview-2024-12-17": + case "gpt-4.1": + case "gpt-4.1-2025-04-14": + case "gpt-4.1-mini": + case "gpt-4.1-mini-2025-04-14": + case "gpt-4.1-nano": + case "gpt-4.1-nano-2025-04-14": + case "gpt-4.5-preview": + case "gpt-4.5-preview-2025-02-27": { return "o200k_base"; } default: diff --git a/tiktoken/model_to_encoding.json b/tiktoken/model_to_encoding.json index 6b55f317..1af5e9dd 100644 --- a/tiktoken/model_to_encoding.json +++ b/tiktoken/model_to_encoding.json @@ -61,15 +61,41 @@ "gpt-4o-2024-11-20": "o200k_base", "gpt-4o-mini-2024-07-18": "o200k_base", "gpt-4o-mini": "o200k_base", + "gpt-4o-search-preview": "o200k_base", + "gpt-4o-search-preview-2025-03-11": "o200k_base", + "gpt-4o-mini-search-preview": "o200k_base", + "gpt-4o-mini-search-preview-2025-03-11": "o200k_base", + "gpt-4o-audio-preview": "o200k_base", + "gpt-4o-audio-preview-2024-12-17": "o200k_base", + "gpt-4o-audio-preview-2024-10-01": "o200k_base", + "gpt-4o-mini-audio-preview": "o200k_base", + "gpt-4o-mini-audio-preview-2024-12-17": "o200k_base", "o1": "o200k_base", "o1-2024-12-17": "o200k_base", "o1-mini": "o200k_base", + "o1-mini-2024-09-12": "o200k_base", "o1-preview": "o200k_base", "o1-preview-2024-09-12": "o200k_base", - "o1-mini-2024-09-12": "o200k_base", + "o1-pro": "o200k_base", + "o1-pro-2025-03-19": "o200k_base", + "o3": "o200k_base", + "o3-2025-04-16": "o200k_base", "o3-mini": "o200k_base", "o3-mini-2025-01-31": "o200k_base", + "o4-mini": "o200k_base", + "o4-mini-2025-04-16": "o200k_base", "chatgpt-4o-latest": "o200k_base", "gpt-4o-realtime": "o200k_base", - "gpt-4o-realtime-preview-2024-10-01": "o200k_base" + "gpt-4o-realtime-preview-2024-10-01": "o200k_base", + "gpt-4o-realtime-preview-2024-12-17": "o200k_base", + "gpt-4o-mini-realtime-preview": "o200k_base", + "gpt-4o-mini-realtime-preview-2024-12-17": "o200k_base", + "gpt-4.1": "o200k_base", + "gpt-4.1-2025-04-14": "o200k_base", + "gpt-4.1-mini": "o200k_base", + "gpt-4.1-mini-2025-04-14": "o200k_base", + "gpt-4.1-nano": "o200k_base", + "gpt-4.1-nano-2025-04-14": "o200k_base", + "gpt-4.5-preview": "o200k_base", + "gpt-4.5-preview-2025-02-27": "o200k_base" } diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index 65b29a00..29a71952 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -426,17 +426,43 @@ export type TiktokenModel = | "gpt-4o-2024-11-20" | "gpt-4o-mini-2024-07-18" | "gpt-4o-mini" + | "gpt-4o-search-preview" + | "gpt-4o-search-preview-2025-03-11" + | "gpt-4o-mini-search-preview" + | "gpt-4o-mini-search-preview-2025-03-11" + | "gpt-4o-audio-preview" + | "gpt-4o-audio-preview-2024-12-17" + | "gpt-4o-audio-preview-2024-10-01" + | "gpt-4o-mini-audio-preview" + | "gpt-4o-mini-audio-preview-2024-12-17" | "o1" | "o1-2024-12-17" | "o1-mini" + | "o1-mini-2024-09-12" | "o1-preview" | "o1-preview-2024-09-12" - | "o1-mini-2024-09-12" + | "o1-pro" + | "o1-pro-2025-03-19" + | "o3" + | "o3-2025-04-16" | "o3-mini" | "o3-mini-2025-01-31" + | "o4-mini" + | "o4-mini-2025-04-16" | "chatgpt-4o-latest" | "gpt-4o-realtime" | "gpt-4o-realtime-preview-2024-10-01" + | "gpt-4o-realtime-preview-2024-12-17" + | "gpt-4o-mini-realtime-preview" + | "gpt-4o-mini-realtime-preview-2024-12-17" + | "gpt-4.1" + | "gpt-4.1-2025-04-14" + | "gpt-4.1-mini" + | "gpt-4.1-mini-2025-04-14" + | "gpt-4.1-nano" + | "gpt-4.1-nano-2025-04-14" + | "gpt-4.5-preview" + | "gpt-4.5-preview-2025-02-27"; /** * @param {TiktokenModel} encoding @@ -515,17 +541,43 @@ pub fn encoding_for_model( "gpt-4o-2024-11-20" => Ok("o200k_base"), "gpt-4o-mini-2024-07-18" => Ok("o200k_base"), "gpt-4o-mini" => Ok("o200k_base"), + "gpt-4o-search-preview" => Ok("o200k_base"), + "gpt-4o-search-preview-2025-03-11" => Ok("o200k_base"), + "gpt-4o-mini-search-preview" => Ok("o200k_base"), + "gpt-4o-mini-search-preview-2025-03-11" => Ok("o200k_base"), + "gpt-4o-audio-preview" => Ok("o200k_base"), + "gpt-4o-audio-preview-2024-12-17" => Ok("o200k_base"), + "gpt-4o-audio-preview-2024-10-01" => Ok("o200k_base"), + "gpt-4o-mini-audio-preview" => Ok("o200k_base"), + "gpt-4o-mini-audio-preview-2024-12-17" => Ok("o200k_base"), "o1" => Ok("o200k_base"), "o1-2024-12-17" => Ok("o200k_base"), "o1-mini" => Ok("o200k_base"), + "o1-mini-2024-09-12" => Ok("o200k_base"), "o1-preview" => Ok("o200k_base"), "o1-preview-2024-09-12" => Ok("o200k_base"), - "o1-mini-2024-09-12" => Ok("o200k_base"), + "o1-pro" => Ok("o200k_base"), + "o1-pro-2025-03-19" => Ok("o200k_base"), + "o3" => Ok("o200k_base"), + "o3-2025-04-16" => Ok("o200k_base"), + "o3-mini" => Ok("o200k_base"), + "o3-mini-2025-01-31" => Ok("o200k_base"), + "o4-mini" => Ok("o200k_base"), + "o4-mini-2025-04-16" => Ok("o200k_base"), "chatgpt-4o-latest" => Ok("o200k_base"), "gpt-4o-realtime" => Ok("o200k_base"), "gpt-4o-realtime-preview-2024-10-01" => Ok("o200k_base"), - "o3-mini" => Ok("o200k_base"), - "o3-mini-2025-01-31" => Ok("o200k_base"), + "gpt-4o-realtime-preview-2024-12-17" => Ok("o200k_base"), + "gpt-4o-mini-realtime-preview" => Ok("o200k_base"), + "gpt-4o-mini-realtime-preview-2024-12-17" => Ok("o200k_base"), + "gpt-4.1" => Ok("o200k_base"), + "gpt-4.1-2025-04-14" => Ok("o200k_base"), + "gpt-4.1-mini" => Ok("o200k_base"), + "gpt-4.1-mini-2025-04-14" => Ok("o200k_base"), + "gpt-4.1-nano" => Ok("o200k_base"), + "gpt-4.1-nano-2025-04-14" => Ok("o200k_base"), + "gpt-4.5-preview" => Ok("o200k_base"), + "gpt-4.5-preview-2025-02-27" => Ok("o200k_base"), model => Err(JsError::new( format!("Invalid model: {}", model.to_string()).as_str(), )), From 35f522f13f6159229a5a1f9f089942b7f812d999 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Tue, 22 Apr 2025 10:56:25 +0200 Subject: [PATCH 2/2] Add changeset --- .changeset/cuddly-comics-type.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/cuddly-comics-type.md diff --git a/.changeset/cuddly-comics-type.md b/.changeset/cuddly-comics-type.md new file mode 100644 index 00000000..d545f51d --- /dev/null +++ b/.changeset/cuddly-comics-type.md @@ -0,0 +1,7 @@ +--- +"tiktoken": patch +"js-tiktoken": patch +"@dqbd/tiktoken": patch +--- + +add o3, o4, gpt-4.1 and missing specialised / pro models