feat: windows support [WIP]

CMakeLists.txt

@@ -3,8 +3,36 @@ cmake_policy(SET CMP0077 NEW)
 project(zvec)
 set(CC_CXX_STANDARD 17)
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type")
+if(MSVC)
+    set(INTTYPES_FORMAT VC7)
+    add_compile_options(/FS) #handle .pdb
+    add_compile_options(/EHsc) # def c++ exception behavior
+    add_compile_options(/Zc:preprocessor)
+    add_compile_options(/we4716) # -Werror=return-type
+
+
+    ###### reduce output length to make vibe coding work better :), which should be removed or solved later
+    add_compile_options(/wd4267 /wd4244)
+    add_compile_options(/wd4146) # unary minus operator applied to unsigned type #usage: uint32_t seg_id_{-1U};
+    add_compile_options(/wd4310) # warning C4310: cast truncates constant value
+    add_compile_options(/wd4530) #  warning C4530: C++ exception handler used, but unwind semantics are not enabled. Specify /EHsc
+    add_compile_options(/wd4251) #protobuf
+    add_compile_options(/wd4245 /wd4334 /wd4702  /wd4305 /wd4099)
+    add_compile_options(/wd4200) # flexible structure
+
+    # //warning C4334: '<<': result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?)
+    # keeps |= (1 << k);
+    add_compile_options(/wd4334)
+    ######
+
+    #TODO(windows): to be verified
+    add_definitions(-DARROW_STATIC -DPARQUET_STATIC -DARROW_ACERO_STATIC -DARROW_DS_STATIC -DARROW_COMPUTE_STATIC)
+
+
+else()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type")
+endif()
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-as-needed")
@@ -27,6 +55,7 @@ endif()
 
 include_directories(${PROJECT_ROOT_DIR}/src/include)
 include_directories(${PROJECT_ROOT_DIR}/src)
+include_directories(${PROJECT_ROOT_DIR})
 
 option(BUILD_PYTHON_BINDINGS "Build Python bindings using pybind11" OFF)
 message(STATUS "BUILD_PYTHON_BINDINGS:${BUILD_PYTHON_BINDINGS}")
@@ -99,4 +128,14 @@ if(BUILD_PYTHON_BINDINGS)
 
     message(STATUS "Zvec install path: ${ZVEC_PY_INSTALL_DIR}")
     install(TARGETS _zvec LIBRARY DESTINATION ${ZVEC_PY_INSTALL_DIR})
+
+    if(MSVC)
+        install(FILES
+            $<TARGET_FILE:glog>
+            $<TARGET_FILE:gflags>
+            $<TARGET_FILE:roaring>
+            $<TARGET_FILE:libprotobuf>
+            DESTINATION ${ZVEC_PY_INSTALL_DIR}
+        )
+    endif()
 endif()

cmake/bazel.cmake

@@ -621,6 +621,11 @@ function(_target_link_libraries _NAME)
       get_target_property(ALWAYS_LINK ${LIB} ALWAYS_LINK)
       if(ALWAYS_LINK)
         list(APPEND LOCAL_RESULT ${LIB})
+      elseif(MSVC AND TARGET ${LIB}_static)
+        get_target_property(_SIBLING_AL ${LIB}_static ALWAYS_LINK)
+        if(_SIBLING_AL)
+          list(APPEND LOCAL_RESULT ${LIB}_static)
+        endif()
       endif()
 
       get_target_property(DEP_LIBS ${LIB} INTERFACE_LINK_LIBRARIES)
@@ -652,6 +657,30 @@ function(_target_link_libraries _NAME)
 
   list(REMOVE_DUPLICATES ALL_LIBS_TO_PROCESS)
 
+  # On MSVC, each DLL has its own copy of template statics (e.g. Factory
+  # singletons), so registrations inside a DLL are invisible to the exe.
+  # Substitute SHARED libs with their ALWAYS_LINK _static counterparts and
+  # use /WHOLEARCHIVE so all registration code lives in the same module.
+  if(MSVC)
+    set(_SUBSTITUTED_LIBS "")
+    foreach(LIB ${ALL_LIBS_TO_PROCESS})
+      if(TARGET ${LIB} AND TARGET ${LIB}_static)
+        get_target_property(_LIB_TYPE ${LIB} TYPE)
+        get_target_property(_STATIC_AL ${LIB}_static ALWAYS_LINK)
+        if("${_LIB_TYPE}" STREQUAL "SHARED_LIBRARY" AND _STATIC_AL)
+          list(APPEND _SUBSTITUTED_LIBS ${LIB}_static)
+          list(APPEND ALL_ALWAYS_LINK_LIBS ${LIB}_static)
+          continue()
+        endif()
+      endif()
+      list(APPEND _SUBSTITUTED_LIBS ${LIB})
+    endforeach()
+    set(ALL_LIBS_TO_PROCESS ${_SUBSTITUTED_LIBS})
+    if(ALL_ALWAYS_LINK_LIBS)
+      list(REMOVE_DUPLICATES ALL_ALWAYS_LINK_LIBS)
+    endif()
+  endif()
+
   foreach(LIB ${ALL_LIBS_TO_PROCESS})
     if(NOT TARGET ${LIB})
       list(APPEND LINK_LIBS ${LIB})
@@ -672,7 +701,7 @@ function(_target_link_libraries _NAME)
       endif()
     else()
       # Microsoft Visual C++
-      list(APPEND LINK_LIBS /WHOLEARCHIVE:$<TARGET_FILE:${LIB}>)
+      list(APPEND MSVC_WHOLEARCHIVE_OPTS /WHOLEARCHIVE:$<TARGET_FILE:${LIB}>)
       get_target_property(OTHER_LINK_LIBS ${LIB} INTERFACE_LINK_LIBRARIES)
       if(OTHER_LINK_LIBS)
         foreach(OTHER_LIB ${OTHER_LINK_LIBS})
@@ -691,6 +720,9 @@ function(_target_link_libraries _NAME)
   endforeach()
 
   target_link_libraries(${_NAME} ${LINK_LIBS})
+  if(MSVC_WHOLEARCHIVE_OPTS)
+    target_link_options(${_NAME} PRIVATE ${MSVC_WHOLEARCHIVE_OPTS})
+  endif()
   if(LIBS_DEPS)
     add_dependencies(${_NAME} ${LIBS_DEPS})
     target_include_directories(${_NAME} PRIVATE "${LIBS_INCS}")

cmake/utils.cmake

@@ -12,7 +12,7 @@ function(apply_patch_once patch_name target_dir patch_file)
 
     #message(STATUS "Applying patch '${patch_name}' to ${target_dir} ...")
     execute_process(
-        COMMAND patch -p1 -i "${patch_file}"
+        COMMAND git apply --ignore-space-change --ignore-whitespace "${patch_file}"
         WORKING_DIRECTORY "${target_dir}"
         RESULT_VARIABLE patch_result
         OUTPUT_VARIABLE patch_stdout

src/ailego/buffer/buffer_manager.cc

@@ -103,7 +103,7 @@ struct BufferManager::BufferContext {
 
   ~BufferContext() {
     if (vector) {
-      free(vector);
+      ailego_aligned_free(vector);
     }
   }
 
@@ -256,15 +256,15 @@ bool BufferManager::BufferContext::read_vector() {
   }
   AILEGO_DEFER([this] { file.close(); });
   uint32_t len = id.vector().length;
-  auto ret = posix_memalign((void **)&vector, 64, len);  // 64-byte alignment
-  if (ret != 0 || vector == nullptr) {
+  vector = (uint8_t *)ailego_aligned_malloc(len, 64);  // 64-byte alignment
+  if (vector == nullptr) {
     LOG_ERROR("Failed to allocate buffer for file[%s]", file_name.c_str());
     return false;
   }
   uint32_t offset = id.vector().offset;
   if (file.read(offset, vector, len) != len) {
     LOG_ERROR("Failed to read file[%s]", file_name.c_str());
-    free(vector);
+    ailego_aligned_free(vector);
     vector = nullptr;
     return false;
   }
@@ -390,7 +390,7 @@ class BufferManager::BufferPool {
       if (victim->id.type == BufferID::TYPE::PARQUET) {
         victim->arrow_refs.clear();
       } else {
-        free(victim->vector);
+        ailego_aligned_free(victim->vector);
         victim->vector = nullptr;
       }
       victim->state = BufferContext::State::IDLE;
@@ -585,10 +585,15 @@ uint64_t BufferManager::total_size_in_bytes() const {
 }
 
 
-BufferManager::~BufferManager() {
+void BufferManager::cleanup() {
   for (auto pool : pools_) {
     delete pool;
   }
+  pools_.clear();
+}
+
+BufferManager::~BufferManager() {
+  cleanup();
 }
 
 

src/ailego/buffer/buffer_pool.cc

@@ -1,6 +1,25 @@
 #include <zvec/ailego/buffer/buffer_pool.h>
 #include <zvec/core/framework/index_logger.h>
 
+#if defined(_MSC_VER)
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <Windows.h>
+static ssize_t zvec_pread(int fd, void *buf, size_t count, size_t offset) {
+  HANDLE handle = reinterpret_cast<HANDLE>(_get_osfhandle(fd));
+  if (handle == INVALID_HANDLE_VALUE) return -1;
+  OVERLAPPED ov = {};
+  ov.Offset = static_cast<DWORD>(offset & 0xFFFFFFFF);
+  ov.OffsetHigh = static_cast<DWORD>(offset >> 32);
+  DWORD bytes_read = 0;
+  if (!ReadFile(handle, buf, static_cast<DWORD>(count), &bytes_read, &ov)) {
+    return -1;
+  }
+  return static_cast<ssize_t>(bytes_read);
+}
+#endif
+
 namespace zvec {
 namespace ailego {
 
@@ -167,13 +186,23 @@ void LPMap::recycle(moodycamel::ConcurrentQueue<char *> &free_buffers) {
 }
 
 VecBufferPool::VecBufferPool(const std::string &filename) {
+#if defined(_MSC_VER)
+  fd_ = _open(filename.c_str(), O_RDONLY | _O_BINARY);
+#else
   fd_ = open(filename.c_str(), O_RDONLY);
+#endif
   if (fd_ < 0) {
     throw std::runtime_error("Failed to open file: " + filename);
   }
+#if defined(_MSC_VER)
+  struct _stat64 st;
+  if (_fstat64(fd_, &st) < 0) {
+    _close(fd_);
+#else
   struct stat st;
   if (fstat(fd_, &st) < 0) {
     ::close(fd_);
+#endif
     throw std::runtime_error("Failed to stat file: " + filename);
   }
   file_size_ = st.st_size;
@@ -247,7 +276,11 @@ char *VecBufferPool::acquire_buffer(block_id_t block_id, size_t offset,
     }
   }
 
+#if defined(_MSC_VER)
+  ssize_t read_bytes = zvec_pread(fd_, buffer, size, offset);
+#else
   ssize_t read_bytes = pread(fd_, buffer, size, offset);
+#endif
   if (read_bytes != static_cast<ssize_t>(size)) {
     LOG_ERROR("Buffer pool failed to read file at offset: %zu", offset);
     free_buffers_.enqueue(buffer);
@@ -257,7 +290,11 @@ char *VecBufferPool::acquire_buffer(block_id_t block_id, size_t offset,
 }
 
 int VecBufferPool::get_meta(size_t offset, size_t length, char *buffer) {
+#if defined(_MSC_VER)
+  ssize_t read_bytes = zvec_pread(fd_, buffer, length, offset);
+#else
   ssize_t read_bytes = pread(fd_, buffer, length, offset);
+#endif
   if (read_bytes != static_cast<ssize_t>(length)) {
     LOG_ERROR("Buffer pool failed to read file at offset: %zu", offset);
     return -1;

src/ailego/internal/cpu_features.cc

@@ -15,7 +15,9 @@
 #include "cpu_features.h"
 #include <cstddef>
 
-#if !defined(_MSC_VER) && !defined(__ARM_ARCH)
+#if defined(_MSC_VER)
+#include <intrin.h>
+#elif !defined(__ARM_ARCH)
 #include <cpuid.h>
 #endif
 

src/ailego/io/file.cc

@@ -411,8 +411,9 @@ bool File::create(const char *path, size_t len, bool direct) {
 
   // Try opening or creating the file
   HANDLE file_handle =
-      CreateFileA(path, GENERIC_WRITE | GENERIC_READ, FILE_SHARE_READ, nullptr,
-                  CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
+      CreateFileA(path, GENERIC_WRITE | GENERIC_READ,
+                  FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+                  nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
   ailego_false_if_false(file_handle != INVALID_HANDLE_VALUE);
 
   // Truncate the file to the specified size
@@ -432,7 +433,8 @@ bool File::create(const char *path, size_t len, bool direct) {
     // Close and reopen file
     CloseHandle(file_handle);
     file_handle = CreateFileA(
-        path, GENERIC_WRITE | GENERIC_READ, FILE_SHARE_READ, nullptr,
+        path, GENERIC_WRITE | GENERIC_READ,
+        FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr,
         OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_NO_BUFFERING, nullptr);
     ailego_false_if_false(file_handle != INVALID_HANDLE_VALUE);
   }
@@ -453,7 +455,8 @@ bool File::open(const char *path, bool rdonly, bool direct) {
   }
   HANDLE file_handle =
       CreateFileA(path, (rdonly ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE),
-                  FILE_SHARE_READ, nullptr, OPEN_EXISTING, flags, nullptr);
+                  FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+                  nullptr, OPEN_EXISTING, flags, nullptr);
   ailego_false_if_false(file_handle != INVALID_HANDLE_VALUE);
 
   read_only_ = rdonly;
@@ -639,14 +642,28 @@ ssize_t File::offset(void) const {
 }
 
 void *File::MemoryMap(NativeHandle handle, ssize_t off, size_t len, int opts) {
-  LARGE_INTEGER file_size;
-  file_size.QuadPart = len;
+  // Root cause: Windows MapViewOfFile requires the file offset to be aligned to
+  // the allocation granularity (64 KB), but segment offsets were only
+  // page-aligned (4 KB). Also, CreateFileMapping was using len instead of
+  // off + len as the max size.
+  //
+  // Fix: Align the view offset down to allocation granularity, adjust the map
+  // length, and return base + excess. MemoryUnmap recovers the base by rounding
+  // down to granularity.
+
+  SYSTEM_INFO si;
+  GetSystemInfo(&si);
+  DWORD granularity = si.dwAllocationGranularity;
+  ssize_t aligned_off = (off / (ssize_t)granularity) * (ssize_t)granularity;
+  size_t excess = (size_t)(off - aligned_off);
+
+  LARGE_INTEGER max_size;
+  max_size.QuadPart = off + len;
 
-  // Create map object
   HANDLE file_mapping = CreateFileMapping(
       handle, nullptr,
       ((opts & File::MMAP_READONLY) ? PAGE_READONLY : PAGE_READWRITE),
-      file_size.HighPart, file_size.LowPart, nullptr);
+      max_size.HighPart, max_size.LowPart, nullptr);
   ailego_null_if_false(file_mapping != nullptr);
 
   DWORD desired_access = FILE_MAP_READ;
@@ -656,14 +673,17 @@ void *File::MemoryMap(NativeHandle handle, ssize_t off, size_t len, int opts) {
   if (!(opts & File::MMAP_SHARED)) {
     desired_access |= FILE_MAP_COPY;
   }
-  file_size.QuadPart = off;
 
-  // Map the whole file to memory and close handle
-  void *addr = MapViewOfFile(file_mapping, desired_access, file_size.HighPart,
-                             file_size.LowPart, 0);
+  LARGE_INTEGER view_offset;
+  view_offset.QuadPart = aligned_off;
+  size_t view_len = len + excess;
+
+  void *base = MapViewOfFile(file_mapping, desired_access, view_offset.HighPart,
+                             view_offset.LowPart, view_len);
   CloseHandle(file_mapping);
 
-  ailego_null_if_false(addr);
+  ailego_null_if_false(base);
+  void *addr = (char *)base + excess;
   if (opts & File::MMAP_LOCKED) {
     VirtualLock(addr, len);
   }
@@ -673,8 +693,17 @@ void *File::MemoryMap(NativeHandle handle, ssize_t off, size_t len, int opts) {
   return addr;
 }
 
-void *File::MemoryMap(size_t, int) {
-  return nullptr;
+void *File::MemoryMap(size_t len, int opts) {
+  void *addr =
+      VirtualAlloc(nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
+  ailego_null_if_false(addr);
+  if (opts & File::MMAP_LOCKED) {
+    VirtualLock(addr, len);
+  }
+  if (opts & File::MMAP_WARMUP) {
+    File::MemoryWarmup(addr, len);
+  }
+  return addr;
 }
 
 void *File::MemoryRemap(void *, size_t, void *, size_t) {
@@ -683,7 +712,14 @@ void *File::MemoryRemap(void *, size_t, void *, size_t) {
 
 void File::MemoryUnmap(void *addr, size_t /*len*/) {
   ailego_return_if_false(addr);
-  UnmapViewOfFile(addr);
+  MEMORY_BASIC_INFORMATION mbi;
+  if (VirtualQuery(addr, &mbi, sizeof(mbi))) {
+    if (mbi.Type == MEM_MAPPED) {
+      UnmapViewOfFile(mbi.AllocationBase);
+    } else {
+      VirtualFree(mbi.AllocationBase, 0, MEM_RELEASE);
+    }
+  }
 }
 
 bool File::MemoryFlush(void *addr, size_t /*len*/) {