From 1adbe03ddc525a76fef9ceebaa435119ae219776 Mon Sep 17 00:00:00 2001 From: =Ondrej Slamecka Date: Mon, 20 Feb 2023 21:33:12 +0000 Subject: [PATCH 1/2] Move mychr to capnpy.util --- capnpy/segment/base.py | 24 +----------------------- capnpy/util.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/capnpy/segment/base.py b/capnpy/segment/base.py index 78f32e34..f5d80ac6 100644 --- a/capnpy/segment/base.py +++ b/capnpy/segment/base.py @@ -1,27 +1,5 @@ import struct -from six import int2byte -from pypytools import IS_PYPY - - -if IS_PYPY: - # workaround for a limitation of the PyPy JIT: struct.unpack is optimized - # only if the format string is a tracing-time constant; this is because of - # this line in rlib/rstruct/formatiterator.py: - # @jit.look_inside_iff(lambda self, fmt: jit.isconstant(fmt)) - # def interpret(self, fmt): - # ... - # - # The problem is that if you use struct.unpack(chr(113), '...'), chr(113) - # is not a tracing-time constant (it becomes constant later, during - # optimizeopt). The work around is to use mychr, which pyjitpl.py is smart - # enough to detect as a tracing-time constant. - _CHR = tuple(map(int2byte, range(256))) - def mychr(i): - return _CHR[i] - -else: - mychr = int2byte - +from capnpy.util import mychr def unpack_uint32(buf, offset): if offset < 0 or offset + 4 > len(buf): diff --git a/capnpy/util.py b/capnpy/util.py index 9969389b..c242323f 100644 --- a/capnpy/util.py +++ b/capnpy/util.py @@ -1,5 +1,6 @@ import sys import py +from pypytools import IS_PYPY import six import capnpy @@ -11,6 +12,25 @@ Py_TPFLAGS_HEAPTYPE = (1<<9) # from object.h +if IS_PYPY: + # workaround for a limitation of the PyPy JIT: struct.unpack is optimized + # only if the format string is a tracing-time constant; this is because of + # this line in rlib/rstruct/formatiterator.py: + # @jit.look_inside_iff(lambda self, fmt: jit.isconstant(fmt)) + # def interpret(self, fmt): + # ... + # + # The problem is that if you use struct.unpack(chr(113), '...'), chr(113) + # is not a tracing-time constant (it becomes constant later, during + # optimizeopt). The work around is to use mychr, which pyjitpl.py is smart + # enough to detect as a tracing-time constant. + _CHR = tuple(map(six.int2byte, range(256))) + def mychr(i): + return _CHR[i] + +else: + mychr = six.int2byte + def magic_setattr(cls, attr, value): if cls.__flags__ & Py_TPFLAGS_HEAPTYPE: # normal case From 3c97679936d634a0d06d3c3ad38c5c763ea6b17a Mon Sep 17 00:00:00 2001 From: =Ondrej Slamecka Date: Mon, 20 Feb 2023 21:35:59 +0000 Subject: [PATCH 2/2] Implement default values for floats Before this patch the compiler generates a^b which is not valid Python for floats. A motivation for implementing this (apart from satisfying the capnp specs) are systems for optional values where extremes mark None/Nothing/nullopt/... --- capnpy/compiler/field.py | 15 +++++++++++++++ capnpy/compiler/request.py | 1 + capnpy/testing/compiler/test_field.py | 23 +++++++++++++++++++++++ capnpy/util.py | 5 +++++ 4 files changed, 44 insertions(+) diff --git a/capnpy/compiler/field.py b/capnpy/compiler/field.py index 993e67ac..ddb84ffd 100644 --- a/capnpy/compiler/field.py +++ b/capnpy/compiler/field.py @@ -34,6 +34,8 @@ def _emit(self, m, ns, name): # if self.slot.type.is_void(): self._emit_void(m, ns, name) + elif self.slot.type.is_float32() or self.slot.type.is_float64(): + self._emit_float(m, ns, name) elif self.slot.type.is_primitive(): self._emit_primitive(m, ns, name) elif self.slot.type.is_bool(): @@ -78,6 +80,19 @@ def _emit_primitive(self, m, ns, name): return value """) + def _emit_float(self, m, ns, name): + ns.typename = '_Types.%s' % self.slot.type.which() + ns.default_ = self.slot.defaultValue.as_pyobj() + ns.ifmt = "ord(%r)" % self.slot.get_fmt() + m.def_property(ns, name, """ + {ensure_union} + value = self._read_primitive({offset}, {ifmt}) + + if {default_} != 0: + value = _fxor(value, {default_}, {ifmt}) + return value + """) + def _emit_bool(self, m, ns, name): byteoffset, bitoffset = divmod(self.slot.offset, 8) ns.offset = byteoffset diff --git a/capnpy/compiler/request.py b/capnpy/compiler/request.py index 57a0e910..f5a07356 100644 --- a/capnpy/compiler/request.py +++ b/capnpy/compiler/request.py @@ -74,6 +74,7 @@ def emit(self, m): m.w("from capnpy.util import extend_module_maybe as _extend_module_maybe") m.w("from capnpy.util import check_version as _check_version") m.w("from capnpy.util import encode_maybe as _encode_maybe") + m.w("from capnpy.util import fxor as _fxor") # if m.pyx: m.w("from capnpy cimport _hash") diff --git a/capnpy/testing/compiler/test_field.py b/capnpy/testing/compiler/test_field.py index b0773249..61837158 100644 --- a/capnpy/testing/compiler/test_field.py +++ b/capnpy/testing/compiler/test_field.py @@ -50,6 +50,29 @@ def test_primitive_default(self): assert p.x == 0 assert p.y is False + def test_float_default(self): + schema = """ + @0xbf5147cbbecf40c1; + struct Foo { + x @0 :Float32 = 0.5; + y @1 :Float64 = 8.0; + } + """ + mod = self.compile(schema) + # + buf = b('\x00\x00\x00\x00\x00\x00\x00\x00' + '\x00\x00\x00\x00\x00\x00\x00\x00') + p = mod.Foo.from_buffer(buf, 0, 2, 0) + assert p.x == 0.5 + assert p.y == 8.0 + # + buf = b('\x00\x00\x80\x00\x00\x00\x00\x00' + '\x00\x00\x00\x00\x00\x00\x10\x00') + p = mod.Foo.from_buffer(buf, 0, 2, 0) + assert p.x == 1.0 + assert p.y == 16.0 + + def test_void(self): schema = """ @0xbf5147cbbecf40c1; diff --git a/capnpy/util.py b/capnpy/util.py index c242323f..6c9f5c8a 100644 --- a/capnpy/util.py +++ b/capnpy/util.py @@ -2,6 +2,7 @@ import py from pypytools import IS_PYPY import six +import struct import capnpy try: @@ -131,3 +132,7 @@ def data_repr(s): except ImportError: float32_repr = float64_repr = repr +# https://stackoverflow.com/a/51939549 +def fxor(a, b, fmt): + raw = [x ^ y for (x, y) in zip(struct.pack(mychr(fmt), a), struct.pack(mychr(fmt), b))] + return struct.unpack(mychr(fmt), bytes(raw))[0]