From 3c57953e8d43377dec453e907d346a71377d2e4b Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Sat, 27 Sep 2025 01:19:12 -0400 Subject: [PATCH 1/6] RE fixes for modern compilers --- modules/re/regexpr.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/re/regexpr.h b/modules/re/regexpr.h index 2aee62d..feac7dc 100644 --- a/modules/re/regexpr.h +++ b/modules/re/regexpr.h @@ -137,11 +137,11 @@ void re_compile_fastmap(regexp_t compiled); extern int re_syntax; extern unsigned char re_syntax_table[256]; void re_compile_initialize(); -int re_set_syntax(); -char *re_compile_pattern(); -int re_match(); -int re_search(); -void re_compile_fastmap(); +int re_set_syntax(int); +char *re_compile_pattern(unsigned char*, int, regexp_t); +int re_match(regexp_t, unsigned char*, int, int, regexp_registers_t); +int re_search(regexp_t, unsigned char*, int, int, int, regexp_registers_t); +void re_compile_fastmap(regexp_t); #endif /* HAVE_PROTOTYPES */ From acf55d20a9ba35d71390b4730db776717c1d1671 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Sat, 27 Sep 2025 01:19:41 -0400 Subject: [PATCH 2/6] Add debugging to GC collection --- tinypy/tp_gc.c | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tinypy/tp_gc.c b/tinypy/tp_gc.c index b10157a..b143ff7 100644 --- a/tinypy/tp_gc.c +++ b/tinypy/tp_gc.c @@ -21,6 +21,11 @@ #define TP_GC_TRACE 0 #define TP_GC_ASSERT_LISTS_ARE_DISJOINT 0 /* Assert no white object is on the black list. Very slow. */ +#define TP_GC_DEBUG_MARKING 1 + +/* Maximum number of objects to collect per partial GC run. */ +#define TP_GC_STEP_MAX 8 + /* tp_grey: ensure an object to the grey list, if the object is already * marked grey, then do nothing. */ void tp_grey(TP, tp_obj v) { @@ -197,6 +202,12 @@ void tp_collect(TP) { } void tp_mark(TP, int max) { + #if TP_GC_DEBUG_MARKING + printf( + "GC: Coloring objects, %d out of %d to mark\n", + max, tp->grey->len + ); + #endif while (tp->grey->len && max > 0) { tp_obj v; /* pick a grey object */ @@ -216,6 +227,9 @@ void tp_mark(TP, int max) { tp_follow(tp,v); if(max > 0) max--; } + #if TP_GC_DEBUG_MARKING + printf("GC complete, %d items remaining\n", tp->grey->len); + #endif } void tp_gc_dump(TP, tpd_list * l, int name, int mark) { @@ -242,10 +256,16 @@ void tp_gc_dump(TP, tpd_list * l, int name, int mark) { void tp_gc_run(TP, int full) { if (full || tp->gcmax == 0 || (tp->steps % tp->gcmax == 0)) { + #if TP_GC_DEBUG_MARKING + printf("Running full GC\n"); + #endif tp_mark(tp, -1); } else { - /* mark 2 items from the grey list every step */ - tp_mark(tp, 8); + /* mark a fixed number of items from the grey list every step */ + #if TP_GC_DEBUG_MARKING + printf("Running partial GC of up to %d items\n", TP_GC_STEP_MAX); + #endif + tp_mark(tp, TP_GC_STEP_MAX); } /* grey list is empty, we can run a collection */ @@ -286,4 +306,3 @@ void tp_gc_deinit(TP) { /**/ - From 51d70ffa2609442958faa0bce4e81078bffb24ef Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Sat, 27 Sep 2025 01:20:18 -0400 Subject: [PATCH 3/6] Add basic GC test --- tests/test_gc.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tests/test_gc.py diff --git a/tests/test_gc.py b/tests/test_gc.py new file mode 100644 index 0000000..2c5a4b6 --- /dev/null +++ b/tests/test_gc.py @@ -0,0 +1,39 @@ +from tinypy.runtime.testing import UnitTest + +class Create: + def __init__(self, **args): + self.__dict__.update(args) + + def _create_with_a(a): + return Create(a=a) + create_with_a = staticmethod(_create_with_a) + + def _create_with_b(b): + return Create(b=b) + create_with_b = staticmethod(_create_with_b) + + +class GcTest(UnitTest): + objA = None + objB = None + scrap_objects = [] + + def test_create(self): + self.objA = Create.create_with_a(3) + assert self.objA.a == 3 + self.objB = Create.create_with_b(3) + assert self.objB.b == 3 + + def test_delete(self): + assert self.objA is not None + assert self.objB is not None + del self.objA + for i in range(1000): + self.scrap_objects.append(Create(a=i)) + del self.objB + #assert not hasattr(self, "objA") + #assert not hasattr(self, "objB") + +t = GcTest() + +t.run() From beb65a82e1669b79f53eb235b445570e1815d631 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Sat, 27 Sep 2025 03:04:56 -0400 Subject: [PATCH 4/6] Define DEBUG_PRINTF macro in debug mode --- Makefile | 8 ++++---- tinypy/tp_internal.h | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index b82dc91..6b71f05 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ TESTS_PY_FILES=$(wildcard tests/*.py) # rule to make objects for static linkage .objs/%.o : %.c @mkdir -p $(dir $@) - $(CC) $(CFLAGS) -g -O0 -I . -c -o $@ $< + $(CC) $(CFLAGS) -DNDEBUG -g -O0 -I . -c -o $@ $< .dbgobjs/%.o : %.c @mkdir -p $(dir $@) @@ -53,7 +53,7 @@ TESTS_PY_FILES=$(wildcard tests/*.py) # rule to make objects for dynamic linkage .dynobjs/%.o : %.c @mkdir -p $(dir $@) - $(CC) $(CFLAGS) -fPIC -g -O0 -I . -c -o $@ $< + $(CC) $(CFLAGS) -DNDEBUG -fPIC -g -O0 -I . -c -o $@ $< all: tpy tpvm @@ -97,7 +97,7 @@ GENERATED_SOURCE_FILES+=tinypy/tp_opcodes.h # tpvm only takes compiled byte codes (.tpc files) tpvm : $(VMLIB_FILES:%.c=.objs/tinypy/%.o) .objs/tinypy/vmmain.o modules/modules.a - $(CC) $(CFLAGS) -g -O0 -o $@ $^ -lm + $(CC) $(CFLAGS) -DNDEBUG -g -O0 -o $@ $^ -lm # # tpvm only takes compiled byte codes (.tpc files) tpvm-dbg : $(VMLIB_FILES:%.c=.dbgobjs/tinypy/%.o) .dbgobjs/tinypy/vmmain.o modules/modules.a @@ -105,7 +105,7 @@ tpvm-dbg : $(VMLIB_FILES:%.c=.dbgobjs/tinypy/%.o) .dbgobjs/tinypy/vmmain.o modul # tpy takes .py files tpy : $(TPLIB_FILES:%.c=.objs/tinypy/%.o) .objs/tinypy/tpmain.o modules/modules.a - $(CC) $(CFLAGS) -o $@ $^ -lm + $(CC) $(CFLAGS) -DNDEBUG -o $@ $^ -lm # tpy takes .py files tpy-dbg : $(TPLIB_FILES:%.c=.dbgobjs/tinypy/%.o) .dbgobjs/tinypy/tpmain.o modules/modules.a diff --git a/tinypy/tp_internal.h b/tinypy/tp_internal.h index 82e09d1..f4889b5 100644 --- a/tinypy/tp_internal.h +++ b/tinypy/tp_internal.h @@ -25,3 +25,11 @@ tp_inline static tpd_frame * tp_get_cur_frame(TP) { return tp_get_frame(tp, tp-> /* Detect unintended size changes. Update as needed. */ STATIC_ASSERT(sizeof(tpd_code) == 4, "size of tpd_code must be 4"); + +#ifndef DEBUG_PRINTF +#ifdef NDEBUG +#define DEBUG_PRINTF(...) {0;} +#else +#define DEBUG_PRINTF(...) fprintf(stderr, __VA_ARGS__) +#endif +#endif From f5d69fda6223fd946cf53d1871e37fcdafd11e03 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Sat, 27 Sep 2025 03:05:10 -0400 Subject: [PATCH 5/6] Garbage collection: Fix full GC runs - Switch over to `DEBUG_PRINTF()` for debugging output - Add MIN() and MAX() macros borrowed from glibc - Rename tp_mark 'max' parameter to 'gc_max' - Fix loop logic to perform full GCs as expected --- tinypy/tp_gc.c | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/tinypy/tp_gc.c b/tinypy/tp_gc.c index b143ff7..075d91d 100644 --- a/tinypy/tp_gc.c +++ b/tinypy/tp_gc.c @@ -21,11 +21,15 @@ #define TP_GC_TRACE 0 #define TP_GC_ASSERT_LISTS_ARE_DISJOINT 0 /* Assert no white object is on the black list. Very slow. */ -#define TP_GC_DEBUG_MARKING 1 - /* Maximum number of objects to collect per partial GC run. */ #define TP_GC_STEP_MAX 8 +/* Macros for min/max, borrowed from glibc sys/param.h */ +#ifndef MAX +#define MIN(a,b) (((a)<(b))?(a):(b)) +#define MAX(a,b) (((a)>(b))?(a):(b)) +#endif + /* tp_grey: ensure an object to the grey list, if the object is already * marked grey, then do nothing. */ void tp_grey(TP, tp_obj v) { @@ -201,14 +205,14 @@ void tp_collect(TP) { tp->black->len = 0; } -void tp_mark(TP, int max) { - #if TP_GC_DEBUG_MARKING - printf( - "GC: Coloring objects, %d out of %d to mark\n", - max, tp->grey->len - ); - #endif - while (tp->grey->len && max > 0) { +void tp_mark(TP, int gc_max) { + int target_len = gc_max > 0 ? MAX(0, tp->grey->len - gc_max) : 0; + DEBUG_PRINTF( + "GC: Coloring objects, %d out of %d to mark\n", + tp->grey->len - target_len, tp->grey->len + ); + + while (tp->grey->len > target_len) { tp_obj v; /* pick a grey object */ v = tpd_list_pop(tp, tp->grey, tp->grey->len-1, "_tp_gcinc"); @@ -225,11 +229,8 @@ void tp_mark(TP, int max) { /* put children to grey. */ tp_follow(tp,v); - if(max > 0) max--; } - #if TP_GC_DEBUG_MARKING - printf("GC complete, %d items remaining\n", tp->grey->len); - #endif + DEBUG_PRINTF("GC complete, %d items remaining\n", tp->grey->len); } void tp_gc_dump(TP, tpd_list * l, int name, int mark) { @@ -256,15 +257,11 @@ void tp_gc_dump(TP, tpd_list * l, int name, int mark) { void tp_gc_run(TP, int full) { if (full || tp->gcmax == 0 || (tp->steps % tp->gcmax == 0)) { - #if TP_GC_DEBUG_MARKING - printf("Running full GC\n"); - #endif + DEBUG_PRINTF("Running full GC\n"); tp_mark(tp, -1); } else { /* mark a fixed number of items from the grey list every step */ - #if TP_GC_DEBUG_MARKING - printf("Running partial GC of up to %d items\n", TP_GC_STEP_MAX); - #endif + DEBUG_PRINTF("Running partial GC of up to %d items\n", TP_GC_STEP_MAX); tp_mark(tp, TP_GC_STEP_MAX); } From eeeacf645207c940ea19982304c62fec324096a1 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Sat, 27 Sep 2025 03:07:23 -0400 Subject: [PATCH 6/6] Update GC test, still doesn't really test GC --- tests/test_gc.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_gc.py b/tests/test_gc.py index 2c5a4b6..ab889f0 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -27,12 +27,14 @@ def test_create(self): def test_delete(self): assert self.objA is not None assert self.objB is not None + for i in range(20): + new_obj = Create(a=i) + self.scrap_objects.append(new_obj) del self.objA - for i in range(1000): - self.scrap_objects.append(Create(a=i)) del self.objB - #assert not hasattr(self, "objA") - #assert not hasattr(self, "objB") + #assert self.objA is None + #assert self.objB is None + t = GcTest()