[tests] failed test_character_level_conditioning[unpacked / no_image_encoder #28
Description
Summary
Failed on commit: 5c2e86b
System MacOS.
____________________________________________________________________ test_character_level_conditioning[unpacked / no_image_encoder] _____________________________________________________________________
model_configuration = (ImageLatentTransformerForCausalLM(
(image_encoder): None
(bytes_encoder): BertForMaskedLM(
(bert): BertModel(...48,
"processor_class": "TextImageProcessor"
}
, functools.partial(<function collate_fn at 0x12fc33a60>, pad_value=0))
@parameterization
def test_character_level_conditioning(model_configuration):
"""Test 1: Character-level conditioning (a b vs a a, b a vs b b)"""
model, processor, collator = model_configuration
print("\n=== Test 1: Character-level conditioning ===")
test_texts_char = ["a b", "b a", "a a", "b b"]
losses, predictions = predict_dataset(test_texts_char, model, processor, collator)
# Check conditioning: trained sequences should have lower loss
assert losses['a b'] < losses['a a'], \
f"'a b' should have lower loss than 'a a': {losses['a b']:.4f} vs {losses['a a']:.4f}"
> assert losses['b a'] < losses['b b'], \
f"'b a' should have lower loss than 'b b': {losses['b a']:.4f} vs {losses['b b']:.4f}"
E AssertionError: 'b a' should have lower loss than 'b b': 0.4799 vs 0.4407
E assert 0.4799067974090576 < 0.4406731923421224
tests/test_model_overfitting.py:132: AssertionError
----------------------------------------------------------------------------------------- Captured stdout setup -----------------------------------------------------------------------------------------
[Configuration: Model without image encoder]
----------------------------------------------------------------------------------------- Captured stdout call ------------------------------------------------------------------------------------------
=== Test 1: Character-level conditioning ===
Loss for 'a b': 0.3614
Loss for 'b a': 0.4799
Loss for 'a a': 0.9456
Loss for 'b b': 0.4407
FAILED tests/test_model_overfitting.py::test_character_level_conditioning[unpacked / no_image_encoder] - AssertionError: 'b a' should have lower loss than 'b b': 0.4799 vs 0.4407Reproduce
pytest