From 2476660001cfc046adbcb53f9f1529c8445452b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20L=C3=B6fgren?= Date: Sat, 14 Feb 2026 13:06:58 +0100 Subject: [PATCH] Fix separator handling in tokenizer --- third_party/z65/z65.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/third_party/z65/z65.c b/third_party/z65/z65.c index 7ba10294..7b67cc32 100644 --- a/third_party/z65/z65.c +++ b/third_party/z65/z65.c @@ -594,7 +594,7 @@ static uint8_t encode_a2(char c) { for (uint8_t i = 0; zalph[2][i]; i++) { if (zalph[2][i] == c) - return i; + return i+6; } return 0; } @@ -666,8 +666,9 @@ static uint8_t tokenize(char *in, uint16_t parse_buf, uint16_t text_buf) max_tok = MAX_TOKENS; while (in[i] && count < max_tok) { - while (is_sep(in[i])) + while (in[i] == ' ') i++; + if (!in[i]) break; @@ -676,13 +677,21 @@ static uint8_t tokenize(char *in, uint16_t parse_buf, uint16_t text_buf) uint8_t dict_len = 0; uint8_t word_len = 0; - while (in[i] && !is_sep(in[i])) { - if (dict_len < 7) - word[dict_len++] = in[i]; - word_len++; - i++; - } - + if(is_sep(in[i])) { + word[0] = in[i]; + word_len = 1; + i++; + } else { + while (in[i] && in[i] != ' ') { + if(is_sep(in[i])) + break; + + if (dict_len < 7) + word[dict_len++] = in[i]; + word_len++; + i++; + } + } uint16_t dict = dict_lookup(word); uint16_t entry = parse_buf + 2 + (count << 2);