From ad5899883f13b581ad5dc51f648be480857e6ab6 Mon Sep 17 00:00:00 2001 From: Nick Dimou Date: Fri, 1 Aug 2025 14:39:40 +0300 Subject: [PATCH] Fix issue #20 ttok is showing a UserWarning coming from the click library reporting that --tokens is used more than once: $ ttok --version /Users/brian/.pyenv/versions/3.13.3/envs/llm-dev-3.13/lib/python3.13/site-packages/click/core.py:1193: UserWarning: The parameter --tokens is used more than once. Remove its duplicate as parameters should be unique. parser = self.make_parser(ctx) /Users/brian/.pyenv/versions/3.13.3/envs/llm-dev-3.13/lib/python3.13/site-packages/click/core.py:1186: UserWarning: The parameter --tokens is used more than once. Remove its duplicate as parameters should be unique. self.parse_args(ctx, args) ttok, version 0.3 # What I think is the root cause Click is generating this warning because the parameter `--tokens` has been registered for two **different** actions (`encode_tokens` and `as_tokens`): ``` @click.option("encode_tokens", "--encode", "--tokens", is_flag=True, help="Output token integers") @click.option("as_tokens", "--tokens", is_flag=True, help="Output full tokens") ``` Based on the examples from the help `--tokens` behavior aligns with `as_tokens`(_Output full tokens_) and not with that of `encode_tokens` (_Output token integers_) so we should remove `--tokens` from `click.option("encode_tokens"` Here's the help: >To view token integers: > > cat input.txt | ttok --encode > >[...] >To see the details of the tokens: > > ttok "hello world" --tokens > > Outputs: > > [b'hello', b' world'] > --- ttok/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ttok/cli.py b/ttok/cli.py index dfd8828..7ecb802 100644 --- a/ttok/cli.py +++ b/ttok/cli.py @@ -13,7 +13,7 @@ ) @click.option("-m", "--model", default="gpt-3.5-turbo", help="Which model to use") @click.option( - "encode_tokens", "--encode", "--tokens", is_flag=True, help="Output token integers" + "encode_tokens", "--encode", is_flag=True, help="Output token integers" ) @click.option( "decode_tokens", "--decode", is_flag=True, help="Convert token integers to text"