diff --git a/welt_training/experiments/easy-tasks/README.md b/welt_training/experiments/easy-tasks/README.md index 340b1e1..2393eae 100644 --- a/welt_training/experiments/easy-tasks/README.md +++ b/welt_training/experiments/easy-tasks/README.md @@ -32,3 +32,10 @@ docker run -it --rm --gpus all \ export WANDB_PROJECT="ocr" welt-train welt_training/experiments/easy-tasks/ocr.yaml ``` + +### Letter Count + +```bash +export WANDB_PROJECT="letter-count" +welt-train welt_training/experiments/easy-tasks/letter-count.yaml +``` diff --git a/welt_training/experiments/easy-tasks/letter-count.yaml b/welt_training/experiments/easy-tasks/letter-count.yaml new file mode 100644 index 0000000..7d2d344 --- /dev/null +++ b/welt_training/experiments/easy-tasks/letter-count.yaml @@ -0,0 +1,66 @@ +# Model Setup +image_encoder_model_name_or_path: null +bytes_encoder_model_name_or_path: prajjwal1/bert-tiny +latent_transformer_model_name_or_path: sbintuitions/tiny-lm +bytes_decoder_model_name_or_path: sign/utf8-lm-tiny +load_pretrained: true + +# Dataset setup +dataset_name: magus4450/english-words-small-letter-count +dataset_text_template: + - "\x0E{text}\x0F " + - "{count}" +max_sequence_length: 128 +max_word_length: 16 + +max_eval_samples: 32 + +# Data Loader +dataloader_num_workers: 8 +dataloader_prefetch_factor: 4 +dataloader_pin_memory: true +dataloader_persistent_workers: true + +# Training setup +remove_unused_columns: false # Necessary +per_device_train_batch_size: 32 +per_device_eval_batch_size: 32 +auto_find_batch_size: true +output_dir: ./output/letter-count +overwrite_output_dir: true +do_train: true + +# Optimizer setup +optim: dion +weight_decay: 0.001 +max_steps: 10000 +warmup_ratio: 0.05 +learning_rate: 6.0e-4 + +# Evaluation +do_eval: true +eval_on_start: true +eval_strategy: steps +eval_steps: 100 +metric_for_best_model: chrf # Using generation-based metric +eval_metrics: [sacrebleu, chrf, wer] # Generation-based evaluation metrics +predict_with_generate: true +generation_max_length: 50 # Max tokens/words to generate during evaluation +log_samples: 5 # Number of sample predictions to log + +# Logging +logging_steps: 10 +logging_strategy: steps +include_tokens_per_second: true +include_num_input_tokens_seen: true +report_to: wandb + +# FLOPS profiling +profile_flops: true +flops_profile_steps: 500 +flops_warmup_steps: 10 +flops_active_steps: 10 + +# Dtype +bf16: true +dtype: bfloat16