Skip to content

s3fs configurations #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rguo123 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

s3fs configurations #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "quadfeather"
version = "2.2.0"
version = "2.3.0"
description = "Quadtree tiling from CSV/Apache Arrow for use with deepscatter in the browser."
readme = "README.md"
requires-python = ">=3.9"
Expand Down
6 changes: 3 additions & 3 deletions quadfeather/tiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ def __init__(
# Easier file system handling methods

def write_feather(self, write_path: Path, table: pa.Table, compression: Literal["zstd", "uncompressed"] = "zstd"):
with self.filesystem.open_output_stream(write_path.as_posix()) as f:
with self.filesystem.open_output_stream(write_path.as_posix(), metadata=None) as f:
feather.write_feather(table, f, compression=compression)

def read_feather(self, read_path: Path, columns: Optional[List[str]] = None) -> pa.Table:
Expand Down Expand Up @@ -713,7 +713,7 @@ def __init__(

def write_batch_to_filehandle(self, path: Path, batch: pa.Table):
if not path in self._open_filehandles:
open_filehandle = self.quadtree.filesystem.open_output_stream(path.as_posix())
open_filehandle = self.quadtree.filesystem.open_output_stream(path.as_posix(), metadata=None)
if path.suffix == ".feather":
self._open_filehandles[path] = ipc.new_file(open_filehandle, schema=batch.schema)
elif path.suffix == ".parquet":
Expand Down Expand Up @@ -1399,7 +1399,7 @@ def overflow_buffers(self):
p = self.overflow_loc
if k != "":
p = p.with_suffix(f".{k}.arrow")
self._sinks[k] = self.quadtree.filesystem.open_output_stream(p.as_posix())
self._sinks[k] = self.quadtree.filesystem.open_output_stream(p.as_posix(), metadata=None)
self._overflow_writers[k] = pa.ipc.new_file(
self._sinks[k], self.quadtree.schemas[k]
)
Expand Down
10 changes: 6 additions & 4 deletions tests/test_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
from pyarrow import fs
from quadfeather.tiler import *
from pathlib import Path
import s3fs

import pytest


def get_s3_filesystem(region: str = "us-east-2"):
return fs.S3FileSystem(
region=region,
)
filesystem = s3fs.S3FileSystem(s3_additional_kwargs={'ServerSideEncryption': 'AES256'}, client_kwargs={'region_name': region})
filesystem = fs.FSSpecHandler(filesystem)
return fs.PyFileSystem(filesystem)

@pytest.mark.skip(reason="This is a slow test that requires a real S3 bucket.")
def test_s3_filesystem(bucket_name: str,NUM_POINTS=100_000, TILE_SIZE=10_000):
def test_s3_filesystem(bucket_name: str, NUM_POINTS=100_000, TILE_SIZE=10_000):
fs = get_s3_filesystem()
basedir = Path(bucket_name) / "tiles"
print("Creating test data...")
Expand Down