Skip to content

Commit 417ca9d

Browse files
kochj23claude
andcommitted
fix: Fall back to flat prompt when model's Jinja chat template fails
Models like Mistral 7B use Jinja features not yet supported by swift-jinja. chatCompletion() now catches TemplateException and retries with a flat User:/Assistant: prompt format so generation succeeds regardless of the model's tokenizer template complexity. Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
1 parent addade7 commit 417ca9d

1 file changed

Lines changed: 33 additions & 2 deletions

File tree

MLX Code/Services/MLXService.swift

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,8 +208,22 @@ actor MLXService {
208208
]
209209
}
210210

211-
let userInput = UserInput(prompt: .messages(messageDicts))
212-
let lmInput = try await container.prepare(input: userInput)
211+
// Try the model's native chat template first. Some models (e.g. Mistral 7B) use
212+
// Jinja features not yet supported by swift-jinja — fall back to flat prompt format.
213+
let lmInput: LMInput
214+
do {
215+
let userInput = UserInput(prompt: .messages(messageDicts))
216+
lmInput = try await container.prepare(input: userInput)
217+
} catch {
218+
await SecureLogger.shared.warning(
219+
"Chat template failed (\(error.localizedDescription)), falling back to flat prompt",
220+
category: "MLXService"
221+
)
222+
let flatPrompt = formatMessagesAsPrompt(messages)
223+
let fallbackInput = UserInput(prompt: .text(flatPrompt))
224+
lmInput = try await container.prepare(input: fallbackInput)
225+
}
226+
213227
let stream = try await container.generate(input: lmInput, parameters: params)
214228

215229
var fullResponse = ""
@@ -279,6 +293,23 @@ actor MLXService {
279293

280294
// MARK: - Private Helpers
281295

296+
/// Formats chat messages as a flat prompt string — used as fallback when
297+
/// the model's Jinja chat template is not supported by swift-jinja.
298+
private func formatMessagesAsPrompt(_ messages: [Message]) -> String {
299+
var prompt = ""
300+
for message in messages {
301+
let prefix: String
302+
switch message.role {
303+
case .system: prefix = "System: "
304+
case .user: prefix = "User: "
305+
case .assistant: prefix = "Assistant: "
306+
}
307+
prompt += prefix + message.content + "\n\n"
308+
}
309+
prompt += "Assistant: "
310+
return prompt
311+
}
312+
282313
private func parseModelDirectory(_ url: URL) async throws -> MLXModel? {
283314
let fileManager = FileManager.default
284315

0 commit comments

Comments
 (0)