diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 47b2dfd993..72213e9a52 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -577,12 +577,6 @@ jobs: - name: Run make install for python run: sudo make -C lib/py install - # - name: Run make install-exec-hook for python - # run: sudo make -C lib/py install-exec-hook - - - name: Run make for python libs - run: make -C lib/py - - name: Run make check for python libs run: make -C lib/py check @@ -842,7 +836,7 @@ jobs: # kotlin cross test are failing -> see THRIFT-5879 server_lang: ['java', 'go', 'cpp', 'py', 'rb'] # we always use comma join as many client langs as possible, to reduce the number of jobs - client_lang: ['java,kotlin', 'go,cpp', 'py', 'rb'] + client_lang: ['java,kotlin', 'go,cpp,py', 'rb'] fail-fast: false steps: - uses: actions/checkout@v6 diff --git a/LANGUAGES.md b/LANGUAGES.md index 8bf9a333d5..a92d53043b 100644 --- a/LANGUAGES.md +++ b/LANGUAGES.md @@ -301,7 +301,7 @@ Thrift's core protocol is TBinary, supported by all languages except for JavaScr 0.2.0 YesYes 2.7.12, 3.5.22.7.15, 3.6.8 - +Yes YesYesYes YesYesYes YesYesYesYes diff --git a/lib/cpp/src/thrift/protocol/TProtocol.h b/lib/cpp/src/thrift/protocol/TProtocol.h index 63b959163d..565a8ecba8 100644 --- a/lib/cpp/src/thrift/protocol/TProtocol.h +++ b/lib/cpp/src/thrift/protocol/TProtocol.h @@ -784,6 +784,10 @@ uint32_t skip(Protocol_& prot, TType type) { result += prot.readListEnd(); return result; } + case T_UUID: { + TUuid uuid; + return prot.readUUID(uuid); + } default: break; } diff --git a/lib/py/setup.py b/lib/py/setup.py index 2dd2a77aa3..456fd6d342 100644 --- a/lib/py/setup.py +++ b/lib/py/setup.py @@ -85,6 +85,14 @@ def run_setup(with_binary): 'src/ext/binary.cpp', 'src/ext/compact.cpp', ], + depends=[ + 'src/ext/binary.h', + 'src/ext/compact.h', + 'src/ext/endian.h', + 'src/ext/protocol.h', + 'src/ext/protocol.tcc', + 'src/ext/types.h', + ], include_dirs=include_dirs, ) ], @@ -138,6 +146,8 @@ def run_setup(with_binary): try: with_binary = True run_setup(with_binary) + sys.exit(0) + except BuildFailed: print() print('*' * 80) @@ -146,4 +156,16 @@ def run_setup(with_binary): print('*' * 80) print() +# Retry but without the binary +try: run_setup(False) + sys.exit(0) + +except BuildFailed: + print() + print('*' * 80) + print("An error occurred while trying to compile without the C extension enabled") + print("Build failed") + print('*' * 80) + print() + sys.exit(1) diff --git a/lib/py/src/Thrift.py b/lib/py/src/Thrift.py index 81fe8cf33f..7b28c908c7 100644 --- a/lib/py/src/Thrift.py +++ b/lib/py/src/Thrift.py @@ -34,8 +34,7 @@ class TType(object): MAP = 13 SET = 14 LIST = 15 - UTF8 = 16 - UTF16 = 17 + UUID = 16 _VALUES_TO_NAMES = ( 'STOP', @@ -54,8 +53,7 @@ class TType(object): 'MAP', 'SET', 'LIST', - 'UTF8', - 'UTF16', + 'UUID', ) diff --git a/lib/py/src/ext/binary.h b/lib/py/src/ext/binary.h index 960b0d003a..dd7750b49a 100644 --- a/lib/py/src/ext/binary.h +++ b/lib/py/src/ext/binary.h @@ -88,6 +88,10 @@ class BinaryProtocol : public ProtocolBase { return encodeValue(value, parsedspec.type, parsedspec.typeargs); } + void writeUuid(char* value) { + writeBuffer(value, 16); + } + void writeFieldStop() { writeByte(static_cast(T_STOP)); } bool readBool(bool& val) { @@ -159,6 +163,13 @@ class BinaryProtocol : public ProtocolBase { return len; } + int32_t readUuid(char** buf) { + if (!readBytes(buf, 16)) { + return -1; + } + return 16; + } + int32_t readListBegin(TType& etype) { int32_t len; uint8_t b = 0; @@ -206,6 +217,7 @@ class BinaryProtocol : public ProtocolBase { } SKIPBYTES(len); } + bool skipUuid() { SKIPBYTES(16); } #undef SKIPBYTES private: diff --git a/lib/py/src/ext/compact.cpp b/lib/py/src/ext/compact.cpp index ae89f2a655..8d13d3d196 100644 --- a/lib/py/src/ext/compact.cpp +++ b/lib/py/src/ext/compact.cpp @@ -24,23 +24,26 @@ namespace apache { namespace thrift { namespace py { +/** Mapping of Compact type to Thrift Type according. + * This list must match the TType enum in TEnum.h */ const uint8_t CompactProtocol::TTypeToCType[] = { - CT_STOP, // T_STOP - 0, // unused - CT_BOOLEAN_TRUE, // T_BOOL - CT_BYTE, // T_BYTE - CT_DOUBLE, // T_DOUBLE - 0, // unused - CT_I16, // T_I16 - 0, // unused - CT_I32, // T_I32 - 0, // unused - CT_I64, // T_I64 - CT_BINARY, // T_STRING - CT_STRUCT, // T_STRUCT - CT_MAP, // T_MAP - CT_SET, // T_SET - CT_LIST, // T_LIST +/* 0 */ CT_STOP, // T_STOP +/* 1 */ 0, // unused +/* 2 */ CT_BOOLEAN_TRUE, // T_BOOL +/* 3 */ CT_BYTE, // T_BYTE +/* 4 */ CT_DOUBLE, // T_DOUBLE +/* 5 */ 0, // unused +/* 6 */ CT_I16, // T_I16 +/* 7 */ 0, // unused +/* 8 */ CT_I32, // T_I32 +/* 9 */ 0, // unused +/* 10 */ CT_I64, // T_I64 +/* 11 */ CT_BINARY, // T_STRING +/* 12 */ CT_STRUCT, // T_STRUCT +/* 13 */ CT_MAP, // T_MAP +/* 14 */ CT_SET, // T_SET +/* 15 */ CT_LIST, // T_LIST +/* 16 */ CT_UUID, // T_UUID }; bool CompactProtocol::readFieldBegin(TType& type, int16_t& tag) { @@ -98,6 +101,8 @@ TType CompactProtocol::getTType(uint8_t type) { return T_MAP; case CT_STRUCT: return T_STRUCT; + case CT_UUID: + return T_UUID; default: PyErr_Format(PyExc_TypeError, "don't know what type: %d", type); return static_cast(-1); diff --git a/lib/py/src/ext/compact.h b/lib/py/src/ext/compact.h index 8f72b09259..0d8946b344 100644 --- a/lib/py/src/ext/compact.h +++ b/lib/py/src/ext/compact.h @@ -104,6 +104,10 @@ class CompactProtocol : public ProtocolBase { void writeFieldStop() { writeByte(0); } + void writeUuid(char* value) { + writeBuffer(value, 16); + } + bool readBool(bool& val) { if (readBool_.exists) { readBool_.exists = false; @@ -231,6 +235,13 @@ class CompactProtocol : public ProtocolBase { } bool readFieldBegin(TType& type, int16_t& tag); + bool readUuid(char** buf) { + if (!readBytes(buf, 16)) { + return false; + } + return true; + } + bool skipBool() { bool val; return readBool(val); @@ -263,6 +274,9 @@ class CompactProtocol : public ProtocolBase { } SKIPBYTES(len); } + bool skipUuid() { + SKIPBYTES(16); + } #undef SKIPBYTES private: @@ -279,7 +293,8 @@ class CompactProtocol : public ProtocolBase { CT_LIST = 0x09, CT_SET = 0x0A, CT_MAP = 0x0B, - CT_STRUCT = 0x0C + CT_STRUCT = 0x0C, + CT_UUID = 0x0D, }; static const uint8_t TTypeToCType[]; @@ -288,7 +303,7 @@ class CompactProtocol : public ProtocolBase { int toCompactType(TType type) { int i = static_cast(type); - return i < 16 ? TTypeToCType[i] : -1; + return i <= 16 ? TTypeToCType[i] : -1; } uint32_t toZigZag(int32_t val) { return (val >> 31) ^ (val << 1); } diff --git a/lib/py/src/ext/module.cpp b/lib/py/src/ext/module.cpp index a1b0e5633e..e2b540e6f6 100644 --- a/lib/py/src/ext/module.cpp +++ b/lib/py/src/ext/module.cpp @@ -38,6 +38,8 @@ PyObject* INTERN_STRING(TFrozenDict); PyObject* INTERN_STRING(cstringio_buf); PyObject* INTERN_STRING(cstringio_refill); +PyObject* INTERN_STRING(UUID); +PyObject* INTERN_STRING(bytes); static PyObject* INTERN_STRING(string_length_limit); static PyObject* INTERN_STRING(container_length_limit); static PyObject* INTERN_STRING(trans); @@ -186,6 +188,8 @@ void initfastbinary() { INIT_INTERN_STRING(string_length_limit); INIT_INTERN_STRING(container_length_limit); INIT_INTERN_STRING(trans); + INIT_INTERN_STRING(UUID); + INIT_INTERN_STRING(bytes); #undef INIT_INTERN_STRING PyObject* module = diff --git a/lib/py/src/ext/protocol.tcc b/lib/py/src/ext/protocol.tcc index aad5a3c88e..b517c381ee 100644 --- a/lib/py/src/ext/protocol.tcc +++ b/lib/py/src/ext/protocol.tcc @@ -542,10 +542,27 @@ bool ProtocolBase::encodeValue(PyObject* value, TType type, PyObject* type return true; } + case T_UUID: { + ScopedPyObject instval(PyObject_GetAttr(value, INTERN_STRING(bytes))); + if (!instval) { + return false; + } + + Py_ssize_t size; + char* buffer; + if (PyBytes_AsStringAndSize(instval.get(), &buffer, &size) < 0) { + return false; + } + if (size != 16) { + PyErr_SetString(PyExc_TypeError, "uuid.bytes must be exactly 16 bytes long"); + return false; + } + impl()->writeUuid(buffer); + return true; + } + case T_STOP: case T_VOID: - case T_UTF16: - case T_UTF8: case T_U64: default: PyErr_Format(PyExc_TypeError, "Unexpected TType for encodeValue: %d", type); @@ -625,11 +642,12 @@ bool ProtocolBase::skip(TType type) { } return true; } + case T_UUID: { + return impl()->skipUuid(); + } case T_STOP: case T_VOID: - case T_UTF16: - case T_UTF8: case T_U64: default: PyErr_Format(PyExc_TypeError, "Unexpected TType for skip: %d", type); @@ -816,10 +834,36 @@ PyObject* ProtocolBase::decodeValue(TType type, PyObject* typeargs) { return readStruct(Py_None, parsedargs.klass, parsedargs.spec); } + case T_UUID: { + char* buf = nullptr; + if(!impl()->readUuid(&buf)) { + return nullptr; + } + + if(!UuidModule) { + UuidModule = PyImport_ImportModule("uuid"); + if (!UuidModule) + return nullptr; + } + + ScopedPyObject cls(PyObject_GetAttr(UuidModule, INTERN_STRING(UUID))); + if (!cls) { + return nullptr; + } + + ScopedPyObject pyBytes(PyBytes_FromStringAndSize(buf, 16)); + if (!pyBytes) { + return nullptr; + } + + ScopedPyObject args(PyTuple_New(0)); + ScopedPyObject kwargs(Py_BuildValue("{O:O}", INTERN_STRING(bytes), pyBytes.get())); + ScopedPyObject ret(PyObject_Call(cls.get(), args.get(), kwargs.get())); + return ret.release(); + } + case T_STOP: case T_VOID: - case T_UTF16: - case T_UTF8: case T_U64: default: PyErr_Format(PyExc_TypeError, "Unexpected TType for decodeValue: %d", type); diff --git a/lib/py/src/ext/types.cpp b/lib/py/src/ext/types.cpp index 0c20e56224..a7ccd0a8c6 100644 --- a/lib/py/src/ext/types.cpp +++ b/lib/py/src/ext/types.cpp @@ -26,6 +26,7 @@ namespace thrift { namespace py { PyObject* ThriftModule = nullptr; +PyObject* UuidModule = nullptr; #if PY_MAJOR_VERSION < 3 char refill_signature[] = {'s', '#', 'i'}; diff --git a/lib/py/src/ext/types.h b/lib/py/src/ext/types.h index 9b45dd065f..2848b28f0b 100644 --- a/lib/py/src/ext/types.h +++ b/lib/py/src/ext/types.h @@ -48,6 +48,8 @@ extern "C" { extern PyObject* INTERN_STRING(TFrozenDict); extern PyObject* INTERN_STRING(cstringio_buf); extern PyObject* INTERN_STRING(cstringio_refill); +extern PyObject* INTERN_STRING(UUID); +extern PyObject* INTERN_STRING(bytes); } namespace apache { @@ -55,6 +57,7 @@ namespace thrift { namespace py { extern PyObject* ThriftModule; +extern PyObject* UuidModule; // Stolen out of TProtocol.h. // It would be a huge pain to have both get this from one place. @@ -76,8 +79,7 @@ enum TType { T_MAP = 13, T_SET = 14, T_LIST = 15, - T_UTF8 = 16, - T_UTF16 = 17 + T_UUID = 16, }; // replace with unique_ptr when we're OK with C++11 diff --git a/lib/py/src/protocol/TBinaryProtocol.py b/lib/py/src/protocol/TBinaryProtocol.py index af64ec1035..b73e3c9394 100644 --- a/lib/py/src/protocol/TBinaryProtocol.py +++ b/lib/py/src/protocol/TBinaryProtocol.py @@ -18,6 +18,7 @@ # from struct import pack, unpack +import uuid from .TProtocol import TType, TProtocolBase, TProtocolException, TProtocolFactory @@ -131,6 +132,9 @@ def writeBinary(self, str): self.writeI32(len(str)) self.trans.write(str) + def writeUuid(self, uuid): + self.trans.write(uuid.bytes) + def readMessageBegin(self): sz = self.readI32() if sz < 0: @@ -235,6 +239,11 @@ def readBinary(self): s = self.trans.readAll(size) return s + def readUuid(self): + buff = self.trans.readAll(16) + val = uuid.UUID(bytes=buff) + return val + class TBinaryProtocolFactory(TProtocolFactory): def __init__(self, strictRead=False, strictWrite=True, **kwargs): diff --git a/lib/py/src/protocol/TCompactProtocol.py b/lib/py/src/protocol/TCompactProtocol.py index a3527cd47a..b58095a000 100644 --- a/lib/py/src/protocol/TCompactProtocol.py +++ b/lib/py/src/protocol/TCompactProtocol.py @@ -19,6 +19,7 @@ from .TProtocol import TType, TProtocolBase, TProtocolException, TProtocolFactory, checkIntegerLimits from struct import pack, unpack +import uuid __all__ = ['TCompactProtocol', 'TCompactProtocolFactory'] @@ -80,6 +81,7 @@ def readVarint(trans): shift += 7 +# As per TCompactProtocol.tcc class CompactType(object): STOP = 0x00 TRUE = 0x01 @@ -94,6 +96,7 @@ class CompactType(object): SET = 0x0A MAP = 0x0B STRUCT = 0x0C + UUID = 0x0D CTYPES = { @@ -109,6 +112,7 @@ class CompactType(object): TType.LIST: CompactType.LIST, TType.SET: CompactType.SET, TType.MAP: CompactType.MAP, + TType.UUID: CompactType.UUID, } TTYPES = {} @@ -276,6 +280,10 @@ def writeI64(self, i64): def writeDouble(self, dub): self.trans.write(pack(' 1: + logging.info('testUuid(%s)' % thing) + return thing + def testStruct(self, thing): if self.options.verbose > 1: logging.info('testStruct({%s, %s, %s, %s})' % (thing.string_thing, thing.byte_thing, thing.i32_thing, thing.i64_thing)) diff --git a/test/py/TestTypes.py b/test/py/TestTypes.py index bb1bc35e87..0df87aa6fd 100644 --- a/test/py/TestTypes.py +++ b/test/py/TestTypes.py @@ -20,6 +20,7 @@ from ThriftTest import ThriftTest from ThriftTest.ThriftTest import Client from ThriftTest.ttypes import Xtruct +from uuid import UUID import unittest @@ -66,3 +67,6 @@ def test_list(self): def test_set(self): self.assertEqual(Client.testSet.__annotations__, {'return': set[int], 'thing': set[int]}) + + def test_uuid(self): + self.assertEqual(Client.testUuid.__annotations__, {'return': UUID, 'thing': UUID}) diff --git a/test/tests.json b/test/tests.json index bcc87d3217..0f7dbe75e7 100644 --- a/test/tests.json +++ b/test/tests.json @@ -99,7 +99,7 @@ ] }, "client": { - "timeout": 6, + "timeout": 15, "command": [ "testclient" ] @@ -292,7 +292,7 @@ ] }, "client": { - "timeout": 10, + "timeout": 15, "command": [ "TestClient.py", "--verbose", @@ -342,7 +342,7 @@ ] }, "client": { - "timeout": 10, + "timeout": 15, "command": [ "python3", "TestClient.py", @@ -393,7 +393,7 @@ ] }, "client": { - "timeout": 8, + "timeout": 15, "command": [ "TestClient" ],