From 9ad9ba16d505a5eb7d7d34a6b702b06b63b0af5c Mon Sep 17 00:00:00 2001 From: Nikita Siniachenko Date: Wed, 5 Nov 2025 01:03:30 +0300 Subject: [PATCH 1/7] almost implemented zstd --- runtime-light/stdlib/stdlib.cmake | 3 +- runtime-light/stdlib/zstd/zstd-functions.cpp | 138 +++++++++++++++++++ runtime-light/stdlib/zstd/zstd-functions.h | 37 +++++ 3 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 runtime-light/stdlib/zstd/zstd-functions.cpp create mode 100644 runtime-light/stdlib/zstd/zstd-functions.h diff --git a/runtime-light/stdlib/stdlib.cmake b/runtime-light/stdlib/stdlib.cmake index a176dac8f5..4ab1cfd301 100644 --- a/runtime-light/stdlib/stdlib.cmake +++ b/runtime-light/stdlib/stdlib.cmake @@ -38,4 +38,5 @@ prepend( time/time-functions.cpp time/time-state.cpp time/timelib-functions.cpp - zlib/zlib-functions.cpp) + zlib/zlib-functions.cpp + zstd/zstd-functions.cpp) diff --git a/runtime-light/stdlib/zstd/zstd-functions.cpp b/runtime-light/stdlib/zstd/zstd-functions.cpp new file mode 100644 index 0000000000..b3d3998b838 --- /dev/null +++ b/runtime-light/stdlib/zstd/zstd-functions.cpp @@ -0,0 +1,138 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#include "runtime-light/stdlib/zstd/zstd-functions.h" + +#include "common/smart_ptrs/unique_ptr_with_delete_function.h" +#include "runtime-common/core/runtime-core.h" +#include "runtime-common/stdlib/string/string-context.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +#define ZSTD_STATIC_LINKING_ONLY + +#include "zstd/zstd.h" + +namespace { + +static_assert(2 * ZSTD_BLOCKSIZE_MAX < StringLibContext::STATIC_BUFFER_LENGTH, "double block size is expected to be less then buffer size"); + +ZSTD_customMem make_custom_alloc() noexcept { + auto alloc = [](void*, size_t size) { return RuntimeAllocator::get().alloc_script_memory(size); }; + auto dealloc = [](void*, void* ptr) { return RuntimeAllocator::get().free_script_memory(ptr, size); }; + return ZSTD_customMem{alloc, dealloc}; +} + +} // namespace + +namespace zstd_impl_ { + +template +void free_ctx_wrapper(T* ptr) { + Deleter(ptr); +} + +using ZSTD_CCtxPtr = vk::unique_ptr_with_delete_function>; +using ZSTD_DCtxPtr = vk::unique_ptr_with_delete_function>; + +Optional compress(const string& data, int64_t level, const string& dict) noexcept { + ZSTD_customMem customMem{make_custom_alloc()}; + ZSTD_CCtxPtr ctx{ZSTD_createCCtx_advanced(customMem)}; + if (!ctx) { + kphp::log::warning("zstd_compress: can not create context"); + return false; + } + + size_t result{ZSTD_CCtx_setParameter(ctx.get(), ZSTD_c_compressionLevel, static_cast(level))}; + if (ZSTD_isError(result)) { + kphp::log::warning("zstd_compress: can not init context: {}", ZSTD_getErrorName(result)); + return false; + } + + result = ZSTD_CCtx_loadDictionary_byReference(ctx.get(), dict.c_str(), dict.size()); + if (ZSTD_isError(result)) { + kphp::log::warning("zstd_compress: can not load dict: {}", ZSTD_getErrorName(result)); + return false; + } + + kphp::log::assertion(ZSTD_CStreamOutSize() <= StringLibContext::STATIC_BUFFER_LENGTH); + ZSTD_outBuffer out{StringLibContext::get().static_buf.get(), StringLibContext::STATIC_BUFFER_LENGTH, 0}; + ZSTD_inBuffer in{data.c_str(), data.size(), 0}; + + string encoded_string{}; + do { + result = ZSTD_compressStream2(ctx.get(), &out, &in, ZSTD_e_end); + if (ZSTD_isError(result)) { + kphp::log::warning("zstd_compress: got zstd stream compression error: {}", ZSTD_getErrorName(result)); + return false; + } + encoded_string.append(static_cast(out.dst), out.pos); + out.pos = 0; + } while (result); + return encoded_string; +} + +Optional uncompress(const string& data, const string& dict) noexcept { + auto size{ZSTD_getFrameContentSize(data.c_str(), data.size())}; + if (size == ZSTD_CONTENTSIZE_ERROR) { + kphp::log::warning("zstd_uncompress: it was not compressed by zstd"); + return false; + } + + ZSTD_customMem customMem{make_custom_alloc()}; + ZSTD_DCtxPtr ctx{ZSTD_createDCtx_advanced(customMem)}; + if (!ctx) { + kphp::log::warning("zstd_uncompress: can not create context"); + return false; + } + + size_t result{ZSTD_DCtx_loadDictionary_byReference(ctx.get(), dict.c_str(), dict.size())}; + if (ZSTD_isError(result)) { + kphp::log::warning("zstd_uncompress: can not load dict: {}", ZSTD_getErrorName(result)); + return false; + } + + if (size != ZSTD_CONTENTSIZE_UNKNOWN) { + if (size > string::max_size()) { + kphp::log::warning("zstd_uncompress: trying to uncompress too large data"); + return false; + } + string decompressed{static_cast(size), false}; + result = ZSTD_decompressDCtx(ctx.get(), decompressed.buffer(), size, data.c_str(), data.size()); + if (ZSTD_isError(result)) { + kphp::log::warning("zstd_uncompress: got zstd error: {}", ZSTD_getErrorName(result)); + return false; + } + return decompressed; + } + + if (ZSTD_isError(result)) { + kphp::log::warning("zstd_uncompress: can not init stream: {}", ZSTD_getErrorName(result)); + return false; + } + + kphp::log::assertion(ZSTD_DStreamOutSize() <= StringLibContext::STATIC_BUFFER_LENGTH); + ZSTD_inBuffer in{data.c_str(), data.size(), 0}; + ZSTD_outBuffer out{StringLibContext::get().static_buf.get(), StringLibContext::STATIC_BUFFER_LENGTH, 0}; + + string decoded_string; + while (in.pos < in.size) { + if (out.pos == out.size) { + decoded_string.append(static_cast(out.dst), static_cast(out.pos)); + out.pos = 0; + } + + result = ZSTD_decompressStream(ctx.get(), &out, &in); + if (ZSTD_isError(result)) { + kphp::log::warning("zstd_uncompress: can not decompress stream: {}", ZSTD_getErrorName(result)); + return false; + } + if (result == 0) { + break; + } + } + decoded_string.append(static_cast(out.dst), static_cast(out.pos)); + return decoded_string; +} + +} // namespace zstd_impl_ diff --git a/runtime-light/stdlib/zstd/zstd-functions.h b/runtime-light/stdlib/zstd/zstd-functions.h new file mode 100644 index 0000000000..bffeeae180 --- /dev/null +++ b/runtime-light/stdlib/zstd/zstd-functions.h @@ -0,0 +1,37 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2024 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +#include "runtime-common/core/runtime-core.h" +#include "zstd/zstd.h" + +constexpr int DEFAULT_COMPRESS_LEVEL = 3; + +namespace zstd_impl_ { + +Optional compress(const string& data, int64_t level = DEFAULT_COMPRESS_LEVEL, const string& dict = string{}) noexcept; + +Optional uncompress(const string& data, const string& dict = string{}) noexcept; + +} // namespace zstd_impl_ + +inline Optional f$zstd_compress(const string& data, int64_t level = DEFAULT_COMPRESS_LEVEL) noexcept { + const int min_level = ZSTD_minCLevel(); + const int max_level = ZSTD_maxCLevel(); + if (min_level > level || level > max_level) { + php_warning("zstd_compress: compression level (%" PRIi64 ") must be within %d..%d or equal to 0", level, min_level, max_level); + return false; + } + + return zstd_impl_::compress(data, level); +} + +inline Optional f$zstd_uncompress(const string& data) noexcept; + +inline Optional f$zstd_compress_dict(const string& data, const string& dict) noexcept { + return zstd_impl_::compress(data, DEFAULT_COMPRESS_LEVEL, dict); +} + +inline Optional f$zstd_uncompress_dict(const string& data, const string& dict) noexcept; From 57f333d3338e301626a349fbe5c76bbc8be44b3b Mon Sep 17 00:00:00 2001 From: Nikita Siniachenko Date: Wed, 5 Nov 2025 12:47:54 +0300 Subject: [PATCH 2/7] removed k2-skip from zstd phpt tests --- .../kphp-light/stdlib/string-functions.txt | 18 ++++++++--------- runtime-light/stdlib/zstd/zstd-functions.cpp | 7 ++++--- runtime-light/stdlib/zstd/zstd-functions.h | 20 +++++++++++++------ tests/phpt/zstd/1_compress.php | 2 +- tests/phpt/zstd/2_uncompress.php | 2 +- tests/phpt/zstd/3_compress_uncompress.php | 2 +- 6 files changed, 30 insertions(+), 21 deletions(-) diff --git a/builtin-functions/kphp-light/stdlib/string-functions.txt b/builtin-functions/kphp-light/stdlib/string-functions.txt index da7249066a..66ea54a015 100644 --- a/builtin-functions/kphp-light/stdlib/string-functions.txt +++ b/builtin-functions/kphp-light/stdlib/string-functions.txt @@ -135,20 +135,20 @@ function unpack ($pattern ::: string, $data ::: string) ::: mixed[] | false; */ function base_convert ($number ::: string, $frombase ::: int, $tobase ::: int) ::: string; -// === UNSUPPORTED === - -/** @kphp-extern-func-info stub generation-required */ -function str_getcsv($str ::: string, string $delimiter ::: string = ",", string $enclosure ::: string = "\"", string $escape ::: string = "\\") ::: mixed[] | false; - -/** @kphp-extern-func-info stub generation-required */ +/** @kphp-extern-func-info */ function zstd_compress(string $data, int $level = 3) ::: string | false; -/** @kphp-extern-func-info stub generation-required */ +/** @kphp-extern-func-info */ function zstd_uncompress(string $data) ::: string | false; -/** @kphp-extern-func-info stub generation-required */ +/** @kphp-extern-func-info */ function zstd_compress_dict(string $data, string $dict) ::: string | false; -/** @kphp-extern-func-info stub generation-required */ +/** @kphp-extern-func-info */ function zstd_uncompress_dict(string $data, string $dict) ::: string | false; +// === UNSUPPORTED === + +/** @kphp-extern-func-info stub generation-required */ +function str_getcsv($str ::: string, string $delimiter ::: string = ",", string $enclosure ::: string = "\"", string $escape ::: string = "\\") ::: mixed[] | false; + /** @kphp-extern-func-info stub generation-required */ function getimagesize ($name ::: string) ::: mixed; diff --git a/runtime-light/stdlib/zstd/zstd-functions.cpp b/runtime-light/stdlib/zstd/zstd-functions.cpp index b3d3998b838..5152ec6b95 100644 --- a/runtime-light/stdlib/zstd/zstd-functions.cpp +++ b/runtime-light/stdlib/zstd/zstd-functions.cpp @@ -5,6 +5,7 @@ #include "runtime-light/stdlib/zstd/zstd-functions.h" #include "common/smart_ptrs/unique_ptr_with_delete_function.h" +#include "runtime-common/core/allocator/script-malloc-interface.h" #include "runtime-common/core/runtime-core.h" #include "runtime-common/stdlib/string/string-context.h" #include "runtime-light/stdlib/diagnostics/logs.h" @@ -18,8 +19,8 @@ namespace { static_assert(2 * ZSTD_BLOCKSIZE_MAX < StringLibContext::STATIC_BUFFER_LENGTH, "double block size is expected to be less then buffer size"); ZSTD_customMem make_custom_alloc() noexcept { - auto alloc = [](void*, size_t size) { return RuntimeAllocator::get().alloc_script_memory(size); }; - auto dealloc = [](void*, void* ptr) { return RuntimeAllocator::get().free_script_memory(ptr, size); }; + auto alloc = [](void*, size_t size) { return kphp::memory::script::alloc(size); }; + auto dealloc = [](void*, void* ptr) { return kphp::memory::script::free(ptr); }; return ZSTD_customMem{alloc, dealloc}; } @@ -115,7 +116,7 @@ Optional uncompress(const string& data, const string& dict) noexcept { ZSTD_inBuffer in{data.c_str(), data.size(), 0}; ZSTD_outBuffer out{StringLibContext::get().static_buf.get(), StringLibContext::STATIC_BUFFER_LENGTH, 0}; - string decoded_string; + string decoded_string{}; while (in.pos < in.size) { if (out.pos == out.size) { decoded_string.append(static_cast(out.dst), static_cast(out.pos)); diff --git a/runtime-light/stdlib/zstd/zstd-functions.h b/runtime-light/stdlib/zstd/zstd-functions.h index bffeeae180..d397a6ddfd 100644 --- a/runtime-light/stdlib/zstd/zstd-functions.h +++ b/runtime-light/stdlib/zstd/zstd-functions.h @@ -5,6 +5,10 @@ #pragma once #include "runtime-common/core/runtime-core.h" +#include "runtime-light/stdlib/diagnostics/logs.h" + +#define ZSTD_STATIC_LINKING_ONLY + #include "zstd/zstd.h" constexpr int DEFAULT_COMPRESS_LEVEL = 3; @@ -18,20 +22,24 @@ Optional uncompress(const string& data, const string& dict = string{}) n } // namespace zstd_impl_ inline Optional f$zstd_compress(const string& data, int64_t level = DEFAULT_COMPRESS_LEVEL) noexcept { - const int min_level = ZSTD_minCLevel(); - const int max_level = ZSTD_maxCLevel(); - if (min_level > level || level > max_level) { - php_warning("zstd_compress: compression level (%" PRIi64 ") must be within %d..%d or equal to 0", level, min_level, max_level); + const int32_t min_level{ZSTD_minCLevel()}; + const int32_t max_level{ZSTD_maxCLevel()}; + if (level < min_level || max_level < level) { + kphp::log::warning("zstd_compress: compression level ({}) must be within [{}..{}] or equal to 0", level, min_level, max_level); return false; } return zstd_impl_::compress(data, level); } -inline Optional f$zstd_uncompress(const string& data) noexcept; +inline Optional f$zstd_uncompress(const string& data) noexcept { + return zstd_impl_::uncompress(data); +} inline Optional f$zstd_compress_dict(const string& data, const string& dict) noexcept { return zstd_impl_::compress(data, DEFAULT_COMPRESS_LEVEL, dict); } -inline Optional f$zstd_uncompress_dict(const string& data, const string& dict) noexcept; +inline Optional f$zstd_uncompress_dict(const string& data, const string& dict) noexcept { + return zstd_impl_::uncompress(data, dict); +} diff --git a/tests/phpt/zstd/1_compress.php b/tests/phpt/zstd/1_compress.php index 2efb816a03..f4d4fbf3f3 100644 --- a/tests/phpt/zstd/1_compress.php +++ b/tests/phpt/zstd/1_compress.php @@ -1,4 +1,4 @@ -@ok k2_skip +@ok Date: Wed, 5 Nov 2025 15:31:00 +0300 Subject: [PATCH 3/7] runtime-light.cmake: added ZSTD::pic::zstd to runtime-light-pic linking --- runtime-light/runtime-light.cmake | 4 ++-- tests/phpt/zstd/3_compress_uncompress.php | 2 +- ...compress_uncompress_without_large_strings.php | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 tests/phpt/zstd/4_compress_uncompress_without_large_strings.php diff --git a/runtime-light/runtime-light.cmake b/runtime-light/runtime-light.cmake index 3cc64e6e03..d3745ef234 100644 --- a/runtime-light/runtime-light.cmake +++ b/runtime-light/runtime-light.cmake @@ -55,9 +55,9 @@ vk_add_library_pic(runtime-light-pic OBJECT ${RUNTIME_LIGHT_SRC}) target_compile_options(runtime-light-pic PUBLIC ${RUNTIME_LIGHT_COMPILE_FLAGS}) target_link_libraries(runtime-light-pic PUBLIC vk::pic::libc-alloc-wrapper) # it's mandatory to have alloc-wrapper first in the list of link libraries since we # want to use its symbols in all other libraries -target_link_libraries(runtime-light-pic PUBLIC KPHP_TIMELIB::pic::timelib PCRE2::pic::pcre2 ZLIB::pic::zlib) # third parties +target_link_libraries(runtime-light-pic PUBLIC KPHP_TIMELIB::pic::timelib PCRE2::pic::pcre2 ZLIB::pic::zlib ZSTD::pic::zstd) # third parties -set(RUNTIME_LIGHT_LINK_LIBS "${KPHP_TIMELIB_PIC_LIBRARIES} ${PCRE2_PIC_LIBRARIES} ${ZLIB_PIC_LIBRARIES}") +set(RUNTIME_LIGHT_LINK_LIBS "${KPHP_TIMELIB_PIC_LIBRARIES} ${PCRE2_PIC_LIBRARIES} ${ZLIB_PIC_LIBRARIES} ${ZSTD_PIC_LIBRARIES}") # ================================================================================================= diff --git a/tests/phpt/zstd/3_compress_uncompress.php b/tests/phpt/zstd/3_compress_uncompress.php index 9e36df6204..7cad8f6cfe 100644 --- a/tests/phpt/zstd/3_compress_uncompress.php +++ b/tests/phpt/zstd/3_compress_uncompress.php @@ -1,4 +1,4 @@ -@ok +@ok k2_skip Date: Mon, 10 Nov 2025 19:05:14 +0300 Subject: [PATCH 4/7] fixed some issues --- .../kphp-light/stdlib/string-functions.txt | 4 -- runtime-light/stdlib/zstd/zstd-functions.cpp | 40 +++++++++++------- runtime-light/stdlib/zstd/zstd-functions.h | 42 +++++++++---------- 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/builtin-functions/kphp-light/stdlib/string-functions.txt b/builtin-functions/kphp-light/stdlib/string-functions.txt index 66ea54a015..238be83d57 100644 --- a/builtin-functions/kphp-light/stdlib/string-functions.txt +++ b/builtin-functions/kphp-light/stdlib/string-functions.txt @@ -135,13 +135,9 @@ function unpack ($pattern ::: string, $data ::: string) ::: mixed[] | false; */ function base_convert ($number ::: string, $frombase ::: int, $tobase ::: int) ::: string; -/** @kphp-extern-func-info */ function zstd_compress(string $data, int $level = 3) ::: string | false; -/** @kphp-extern-func-info */ function zstd_uncompress(string $data) ::: string | false; -/** @kphp-extern-func-info */ function zstd_compress_dict(string $data, string $dict) ::: string | false; -/** @kphp-extern-func-info */ function zstd_uncompress_dict(string $data, string $dict) ::: string | false; // === UNSUPPORTED === diff --git a/runtime-light/stdlib/zstd/zstd-functions.cpp b/runtime-light/stdlib/zstd/zstd-functions.cpp index 5152ec6b95..bd29a66d63 100644 --- a/runtime-light/stdlib/zstd/zstd-functions.cpp +++ b/runtime-light/stdlib/zstd/zstd-functions.cpp @@ -1,19 +1,22 @@ // Compiler for PHP (aka KPHP) -// Copyright (c) 2024 LLC «V Kontakte» +// Copyright (c) 2025 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt #include "runtime-light/stdlib/zstd/zstd-functions.h" +#include +#include +#include + +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd/zstd.h" + #include "common/smart_ptrs/unique_ptr_with_delete_function.h" #include "runtime-common/core/allocator/script-malloc-interface.h" #include "runtime-common/core/runtime-core.h" #include "runtime-common/stdlib/string/string-context.h" #include "runtime-light/stdlib/diagnostics/logs.h" -#define ZSTD_STATIC_LINKING_ONLY - -#include "zstd/zstd.h" - namespace { static_assert(2 * ZSTD_BLOCKSIZE_MAX < StringLibContext::STATIC_BUFFER_LENGTH, "double block size is expected to be less then buffer size"); @@ -26,7 +29,7 @@ ZSTD_customMem make_custom_alloc() noexcept { } // namespace -namespace zstd_impl_ { +namespace kphp::zstd { template void free_ctx_wrapper(T* ptr) { @@ -36,7 +39,14 @@ void free_ctx_wrapper(T* ptr) { using ZSTD_CCtxPtr = vk::unique_ptr_with_delete_function>; using ZSTD_DCtxPtr = vk::unique_ptr_with_delete_function>; -Optional compress(const string& data, int64_t level, const string& dict) noexcept { +Optional compress(std::span data, int64_t level, std::span dict) noexcept { + const int32_t min_level{ZSTD_minCLevel()}; + const int32_t max_level{ZSTD_maxCLevel()}; + if (level < min_level || max_level < level) { + kphp::log::warning("zstd_compress: compression level ({}) must be within [{}..{}] or equal to 0", level, min_level, max_level); + return false; + } + ZSTD_customMem customMem{make_custom_alloc()}; ZSTD_CCtxPtr ctx{ZSTD_createCCtx_advanced(customMem)}; if (!ctx) { @@ -50,7 +60,7 @@ Optional compress(const string& data, int64_t level, const string& dict) return false; } - result = ZSTD_CCtx_loadDictionary_byReference(ctx.get(), dict.c_str(), dict.size()); + result = ZSTD_CCtx_loadDictionary_byReference(ctx.get(), dict.data(), dict.size()); if (ZSTD_isError(result)) { kphp::log::warning("zstd_compress: can not load dict: {}", ZSTD_getErrorName(result)); return false; @@ -58,7 +68,7 @@ Optional compress(const string& data, int64_t level, const string& dict) kphp::log::assertion(ZSTD_CStreamOutSize() <= StringLibContext::STATIC_BUFFER_LENGTH); ZSTD_outBuffer out{StringLibContext::get().static_buf.get(), StringLibContext::STATIC_BUFFER_LENGTH, 0}; - ZSTD_inBuffer in{data.c_str(), data.size(), 0}; + ZSTD_inBuffer in{data.data(), data.size(), 0}; string encoded_string{}; do { @@ -73,8 +83,8 @@ Optional compress(const string& data, int64_t level, const string& dict) return encoded_string; } -Optional uncompress(const string& data, const string& dict) noexcept { - auto size{ZSTD_getFrameContentSize(data.c_str(), data.size())}; +Optional uncompress(std::span data, std::span dict) noexcept { + auto size{ZSTD_getFrameContentSize(data.data(), data.size())}; if (size == ZSTD_CONTENTSIZE_ERROR) { kphp::log::warning("zstd_uncompress: it was not compressed by zstd"); return false; @@ -87,7 +97,7 @@ Optional uncompress(const string& data, const string& dict) noexcept { return false; } - size_t result{ZSTD_DCtx_loadDictionary_byReference(ctx.get(), dict.c_str(), dict.size())}; + size_t result{ZSTD_DCtx_loadDictionary_byReference(ctx.get(), dict.data(), dict.size())}; if (ZSTD_isError(result)) { kphp::log::warning("zstd_uncompress: can not load dict: {}", ZSTD_getErrorName(result)); return false; @@ -99,7 +109,7 @@ Optional uncompress(const string& data, const string& dict) noexcept { return false; } string decompressed{static_cast(size), false}; - result = ZSTD_decompressDCtx(ctx.get(), decompressed.buffer(), size, data.c_str(), data.size()); + result = ZSTD_decompressDCtx(ctx.get(), decompressed.buffer(), size, data.data(), data.size()); if (ZSTD_isError(result)) { kphp::log::warning("zstd_uncompress: got zstd error: {}", ZSTD_getErrorName(result)); return false; @@ -113,7 +123,7 @@ Optional uncompress(const string& data, const string& dict) noexcept { } kphp::log::assertion(ZSTD_DStreamOutSize() <= StringLibContext::STATIC_BUFFER_LENGTH); - ZSTD_inBuffer in{data.c_str(), data.size(), 0}; + ZSTD_inBuffer in{data.data(), data.size(), 0}; ZSTD_outBuffer out{StringLibContext::get().static_buf.get(), StringLibContext::STATIC_BUFFER_LENGTH, 0}; string decoded_string{}; @@ -136,4 +146,4 @@ Optional uncompress(const string& data, const string& dict) noexcept { return decoded_string; } -} // namespace zstd_impl_ +} // namespace kphp::zstd diff --git a/runtime-light/stdlib/zstd/zstd-functions.h b/runtime-light/stdlib/zstd/zstd-functions.h index d397a6ddfd..3ebb9661b3 100644 --- a/runtime-light/stdlib/zstd/zstd-functions.h +++ b/runtime-light/stdlib/zstd/zstd-functions.h @@ -1,45 +1,41 @@ // Compiler for PHP (aka KPHP) -// Copyright (c) 2024 LLC «V Kontakte» +// Copyright (c) 2025 LLC «V Kontakte» // Distributed under the GPL v3 License, see LICENSE.notice.txt #pragma once -#include "runtime-common/core/runtime-core.h" -#include "runtime-light/stdlib/diagnostics/logs.h" - -#define ZSTD_STATIC_LINKING_ONLY +#include +#include +#include -#include "zstd/zstd.h" - -constexpr int DEFAULT_COMPRESS_LEVEL = 3; +#include "runtime-common/core/runtime-core.h" -namespace zstd_impl_ { +namespace kphp::zstd { -Optional compress(const string& data, int64_t level = DEFAULT_COMPRESS_LEVEL, const string& dict = string{}) noexcept; +inline constexpr int64_t DEFAULT_COMPRESS_LEVEL = 3; -Optional uncompress(const string& data, const string& dict = string{}) noexcept; +// TODO what optional to use? php or std? +Optional compress(std::span data, int64_t level = DEFAULT_COMPRESS_LEVEL, + std::span dict = std::span{}) noexcept; -} // namespace zstd_impl_ +Optional uncompress(std::span data, std::span dict = std::span{}) noexcept; -inline Optional f$zstd_compress(const string& data, int64_t level = DEFAULT_COMPRESS_LEVEL) noexcept { - const int32_t min_level{ZSTD_minCLevel()}; - const int32_t max_level{ZSTD_maxCLevel()}; - if (level < min_level || max_level < level) { - kphp::log::warning("zstd_compress: compression level ({}) must be within [{}..{}] or equal to 0", level, min_level, max_level); - return false; - } +} // namespace kphp::zstd - return zstd_impl_::compress(data, level); +inline Optional f$zstd_compress(const string& data, int64_t level = kphp::zstd::DEFAULT_COMPRESS_LEVEL) noexcept { + return kphp::zstd::compress({reinterpret_cast(data.c_str()), static_cast(data.size())}, level); } inline Optional f$zstd_uncompress(const string& data) noexcept { - return zstd_impl_::uncompress(data); + return kphp::zstd::uncompress({reinterpret_cast(data.c_str()), static_cast(data.size())}); } inline Optional f$zstd_compress_dict(const string& data, const string& dict) noexcept { - return zstd_impl_::compress(data, DEFAULT_COMPRESS_LEVEL, dict); + return kphp::zstd::compress({reinterpret_cast(data.c_str()), static_cast(data.size())}, kphp::zstd::DEFAULT_COMPRESS_LEVEL, + {reinterpret_cast(dict.c_str()), static_cast(dict.size())}); } inline Optional f$zstd_uncompress_dict(const string& data, const string& dict) noexcept { - return zstd_impl_::uncompress(data, dict); + return kphp::zstd::uncompress({reinterpret_cast(data.c_str()), static_cast(data.size())}, + {reinterpret_cast(dict.c_str()), static_cast(dict.size())}); } From 8d1fa69eca59f90f609971d4c493864a18984aeb Mon Sep 17 00:00:00 2001 From: Nikita Siniachenko Date: Mon, 10 Nov 2025 19:40:39 +0300 Subject: [PATCH 5/7] fixed other issues --- runtime-light/stdlib/zstd/zstd-functions.cpp | 38 ++++++++------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/runtime-light/stdlib/zstd/zstd-functions.cpp b/runtime-light/stdlib/zstd/zstd-functions.cpp index bd29a66d63..559d1ae9d4 100644 --- a/runtime-light/stdlib/zstd/zstd-functions.cpp +++ b/runtime-light/stdlib/zstd/zstd-functions.cpp @@ -11,6 +11,7 @@ #define ZSTD_STATIC_LINKING_ONLY #include "zstd/zstd.h" +#include "common/containers/final_action.h" #include "common/smart_ptrs/unique_ptr_with_delete_function.h" #include "runtime-common/core/allocator/script-malloc-interface.h" #include "runtime-common/core/runtime-core.h" @@ -21,46 +22,35 @@ namespace { static_assert(2 * ZSTD_BLOCKSIZE_MAX < StringLibContext::STATIC_BUFFER_LENGTH, "double block size is expected to be less then buffer size"); -ZSTD_customMem make_custom_alloc() noexcept { - auto alloc = [](void*, size_t size) { return kphp::memory::script::alloc(size); }; - auto dealloc = [](void*, void* ptr) { return kphp::memory::script::free(ptr); }; - return ZSTD_customMem{alloc, dealloc}; -} +constexpr ZSTD_customMem zstd_allocator{[](void*, size_t size) { return kphp::memory::script::alloc(size); }, + [](void*, void* ptr) { return kphp::memory::script::free(ptr); }}; } // namespace namespace kphp::zstd { -template -void free_ctx_wrapper(T* ptr) { - Deleter(ptr); -} - -using ZSTD_CCtxPtr = vk::unique_ptr_with_delete_function>; -using ZSTD_DCtxPtr = vk::unique_ptr_with_delete_function>; - Optional compress(std::span data, int64_t level, std::span dict) noexcept { const int32_t min_level{ZSTD_minCLevel()}; const int32_t max_level{ZSTD_maxCLevel()}; if (level < min_level || max_level < level) { - kphp::log::warning("zstd_compress: compression level ({}) must be within [{}..{}] or equal to 0", level, min_level, max_level); + kphp::log::warning("zstd_compress: compression level ({}) must be within [{}..{}]", level, min_level, max_level); return false; } - ZSTD_customMem customMem{make_custom_alloc()}; - ZSTD_CCtxPtr ctx{ZSTD_createCCtx_advanced(customMem)}; + ZSTD_CCtx* ctx{ZSTD_createCCtx_advanced(zstd_allocator)}; if (!ctx) { kphp::log::warning("zstd_compress: can not create context"); return false; } + const auto finalizer{vk::finally([&ctx]() noexcept { ZSTD_freeCCtx(ctx); })}; - size_t result{ZSTD_CCtx_setParameter(ctx.get(), ZSTD_c_compressionLevel, static_cast(level))}; + size_t result{ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, static_cast(level))}; if (ZSTD_isError(result)) { kphp::log::warning("zstd_compress: can not init context: {}", ZSTD_getErrorName(result)); return false; } - result = ZSTD_CCtx_loadDictionary_byReference(ctx.get(), dict.data(), dict.size()); + result = ZSTD_CCtx_loadDictionary_byReference(ctx, dict.data(), dict.size()); if (ZSTD_isError(result)) { kphp::log::warning("zstd_compress: can not load dict: {}", ZSTD_getErrorName(result)); return false; @@ -72,7 +62,7 @@ Optional compress(std::span data, int64_t level, std::s string encoded_string{}; do { - result = ZSTD_compressStream2(ctx.get(), &out, &in, ZSTD_e_end); + result = ZSTD_compressStream2(ctx, &out, &in, ZSTD_e_end); if (ZSTD_isError(result)) { kphp::log::warning("zstd_compress: got zstd stream compression error: {}", ZSTD_getErrorName(result)); return false; @@ -90,14 +80,14 @@ Optional uncompress(std::span data, std::span uncompress(std::span data, std::span(size), false}; - result = ZSTD_decompressDCtx(ctx.get(), decompressed.buffer(), size, data.data(), data.size()); + result = ZSTD_decompressDCtx(ctx, decompressed.buffer(), size, data.data(), data.size()); if (ZSTD_isError(result)) { kphp::log::warning("zstd_uncompress: got zstd error: {}", ZSTD_getErrorName(result)); return false; @@ -133,7 +123,7 @@ Optional uncompress(std::span data, std::span Date: Mon, 10 Nov 2025 19:57:46 +0300 Subject: [PATCH 6/7] fixed other issues --- runtime-light/stdlib/zstd/zstd-functions.cpp | 30 ++++++++-------- runtime-light/stdlib/zstd/zstd-functions.h | 36 ++++++++++++++------ 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/runtime-light/stdlib/zstd/zstd-functions.cpp b/runtime-light/stdlib/zstd/zstd-functions.cpp index 559d1ae9d4..877433b127 100644 --- a/runtime-light/stdlib/zstd/zstd-functions.cpp +++ b/runtime-light/stdlib/zstd/zstd-functions.cpp @@ -6,13 +6,13 @@ #include #include +#include #include #define ZSTD_STATIC_LINKING_ONLY #include "zstd/zstd.h" #include "common/containers/final_action.h" -#include "common/smart_ptrs/unique_ptr_with_delete_function.h" #include "runtime-common/core/allocator/script-malloc-interface.h" #include "runtime-common/core/runtime-core.h" #include "runtime-common/stdlib/string/string-context.h" @@ -29,31 +29,31 @@ constexpr ZSTD_customMem zstd_allocator{[](void*, size_t size) { return kphp::me namespace kphp::zstd { -Optional compress(std::span data, int64_t level, std::span dict) noexcept { +std::optional compress(std::span data, int64_t level, std::span dict) noexcept { const int32_t min_level{ZSTD_minCLevel()}; const int32_t max_level{ZSTD_maxCLevel()}; if (level < min_level || max_level < level) { kphp::log::warning("zstd_compress: compression level ({}) must be within [{}..{}]", level, min_level, max_level); - return false; + return {}; } ZSTD_CCtx* ctx{ZSTD_createCCtx_advanced(zstd_allocator)}; if (!ctx) { kphp::log::warning("zstd_compress: can not create context"); - return false; + return {}; } const auto finalizer{vk::finally([&ctx]() noexcept { ZSTD_freeCCtx(ctx); })}; size_t result{ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, static_cast(level))}; if (ZSTD_isError(result)) { kphp::log::warning("zstd_compress: can not init context: {}", ZSTD_getErrorName(result)); - return false; + return {}; } result = ZSTD_CCtx_loadDictionary_byReference(ctx, dict.data(), dict.size()); if (ZSTD_isError(result)) { kphp::log::warning("zstd_compress: can not load dict: {}", ZSTD_getErrorName(result)); - return false; + return {}; } kphp::log::assertion(ZSTD_CStreamOutSize() <= StringLibContext::STATIC_BUFFER_LENGTH); @@ -65,7 +65,7 @@ Optional compress(std::span data, int64_t level, std::s result = ZSTD_compressStream2(ctx, &out, &in, ZSTD_e_end); if (ZSTD_isError(result)) { kphp::log::warning("zstd_compress: got zstd stream compression error: {}", ZSTD_getErrorName(result)); - return false; + return {}; } encoded_string.append(static_cast(out.dst), out.pos); out.pos = 0; @@ -73,43 +73,43 @@ Optional compress(std::span data, int64_t level, std::s return encoded_string; } -Optional uncompress(std::span data, std::span dict) noexcept { +std::optional uncompress(std::span data, std::span dict) noexcept { auto size{ZSTD_getFrameContentSize(data.data(), data.size())}; if (size == ZSTD_CONTENTSIZE_ERROR) { kphp::log::warning("zstd_uncompress: it was not compressed by zstd"); - return false; + return {}; } ZSTD_DCtx* ctx{ZSTD_createDCtx_advanced(zstd_allocator)}; if (!ctx) { kphp::log::warning("zstd_uncompress: can not create context"); - return false; + return {}; } const auto finalizer{vk::finally([&ctx]() noexcept { ZSTD_freeDCtx(ctx); })}; size_t result{ZSTD_DCtx_loadDictionary_byReference(ctx, dict.data(), dict.size())}; if (ZSTD_isError(result)) { kphp::log::warning("zstd_uncompress: can not load dict: {}", ZSTD_getErrorName(result)); - return false; + return {}; } if (size != ZSTD_CONTENTSIZE_UNKNOWN) { if (size > string::max_size()) { kphp::log::warning("zstd_uncompress: trying to uncompress too large data"); - return false; + return {}; } string decompressed{static_cast(size), false}; result = ZSTD_decompressDCtx(ctx, decompressed.buffer(), size, data.data(), data.size()); if (ZSTD_isError(result)) { kphp::log::warning("zstd_uncompress: got zstd error: {}", ZSTD_getErrorName(result)); - return false; + return {}; } return decompressed; } if (ZSTD_isError(result)) { kphp::log::warning("zstd_uncompress: can not init stream: {}", ZSTD_getErrorName(result)); - return false; + return {}; } kphp::log::assertion(ZSTD_DStreamOutSize() <= StringLibContext::STATIC_BUFFER_LENGTH); @@ -126,7 +126,7 @@ Optional uncompress(std::span data, std::span #include +#include #include #include "runtime-common/core/runtime-core.h" @@ -14,28 +15,43 @@ namespace kphp::zstd { inline constexpr int64_t DEFAULT_COMPRESS_LEVEL = 3; -// TODO what optional to use? php or std? -Optional compress(std::span data, int64_t level = DEFAULT_COMPRESS_LEVEL, - std::span dict = std::span{}) noexcept; +std::optional compress(std::span data, int64_t level = DEFAULT_COMPRESS_LEVEL, + std::span dict = std::span{}) noexcept; -Optional uncompress(std::span data, std::span dict = std::span{}) noexcept; +std::optional uncompress(std::span data, std::span dict = std::span{}) noexcept; } // namespace kphp::zstd inline Optional f$zstd_compress(const string& data, int64_t level = kphp::zstd::DEFAULT_COMPRESS_LEVEL) noexcept { - return kphp::zstd::compress({reinterpret_cast(data.c_str()), static_cast(data.size())}, level); + auto res{kphp::zstd::compress({reinterpret_cast(data.c_str()), static_cast(data.size())}, level)}; + if (!res) [[unlikely]] { + return false; + } + return res.value(); } inline Optional f$zstd_uncompress(const string& data) noexcept { - return kphp::zstd::uncompress({reinterpret_cast(data.c_str()), static_cast(data.size())}); + auto res{kphp::zstd::uncompress({reinterpret_cast(data.c_str()), static_cast(data.size())})}; + if (!res) [[unlikely]] { + return false; + } + return res.value(); } inline Optional f$zstd_compress_dict(const string& data, const string& dict) noexcept { - return kphp::zstd::compress({reinterpret_cast(data.c_str()), static_cast(data.size())}, kphp::zstd::DEFAULT_COMPRESS_LEVEL, - {reinterpret_cast(dict.c_str()), static_cast(dict.size())}); + auto res{kphp::zstd::compress({reinterpret_cast(data.c_str()), static_cast(data.size())}, kphp::zstd::DEFAULT_COMPRESS_LEVEL, + {reinterpret_cast(dict.c_str()), static_cast(dict.size())})}; + if (!res) [[unlikely]] { + return false; + } + return res.value(); } inline Optional f$zstd_uncompress_dict(const string& data, const string& dict) noexcept { - return kphp::zstd::uncompress({reinterpret_cast(data.c_str()), static_cast(data.size())}, - {reinterpret_cast(dict.c_str()), static_cast(dict.size())}); + auto res{kphp::zstd::uncompress({reinterpret_cast(data.c_str()), static_cast(data.size())}, + {reinterpret_cast(dict.c_str()), static_cast(dict.size())})}; + if (!res) [[unlikely]] { + return false; + } + return res.value(); } From cd65d7e6827b8d47b01165a9fa7b198cb1c25a4b Mon Sep 17 00:00:00 2001 From: Nikita Siniachenko Date: Tue, 11 Nov 2025 17:17:40 +0300 Subject: [PATCH 7/7] noexcept lambdas and std::addressof --- runtime-light/stdlib/zstd/zstd-functions.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime-light/stdlib/zstd/zstd-functions.cpp b/runtime-light/stdlib/zstd/zstd-functions.cpp index 877433b127..d8a986e02f 100644 --- a/runtime-light/stdlib/zstd/zstd-functions.cpp +++ b/runtime-light/stdlib/zstd/zstd-functions.cpp @@ -22,8 +22,8 @@ namespace { static_assert(2 * ZSTD_BLOCKSIZE_MAX < StringLibContext::STATIC_BUFFER_LENGTH, "double block size is expected to be less then buffer size"); -constexpr ZSTD_customMem zstd_allocator{[](void*, size_t size) { return kphp::memory::script::alloc(size); }, - [](void*, void* ptr) { return kphp::memory::script::free(ptr); }}; +constexpr ZSTD_customMem zstd_allocator{[](void*, size_t size) noexcept { return kphp::memory::script::alloc(size); }, + [](void*, void* ptr) noexcept { return kphp::memory::script::free(ptr); }}; } // namespace @@ -62,7 +62,7 @@ std::optional compress(std::span data, int64_t level, s string encoded_string{}; do { - result = ZSTD_compressStream2(ctx, &out, &in, ZSTD_e_end); + result = ZSTD_compressStream2(ctx, std::addressof(out), std::addressof(in), ZSTD_e_end); if (ZSTD_isError(result)) { kphp::log::warning("zstd_compress: got zstd stream compression error: {}", ZSTD_getErrorName(result)); return {}; @@ -123,7 +123,7 @@ std::optional uncompress(std::span data, std::span