Skip to content

Db class update #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mharshe wants to merge 3 commits into from
Closed

Db class update #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b0fce24
Add metadata class
mharshe Nov 14, 2021
4a887e9
Add missing API functions to DB class.
mharshe Nov 13, 2021
c026360
Add dummy methods to Python API for db class
mharshe Nov 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 34 additions & 2 deletions rocksdb/_rocksdb.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import cython
from libcpp.string cimport string
from libcpp.deque cimport deque
from libcpp.vector cimport vector
from libcpp.map cimport map
from cpython cimport bool as py_bool
from libcpp cimport bool as cpp_bool
from libc.stdint cimport uint32_t
Expand All @@ -29,6 +30,7 @@ from . cimport env
from . cimport table_factory
from . cimport memtablerep
from . cimport universal_compaction
from . cimport metadata

# Enums are the only exception for direct imports
# Their name als already unique enough
Expand Down Expand Up @@ -2382,8 +2384,26 @@ cdef class DB(object):
else:
return None

# def get_map_property(self, prop, ColumnFamilyHandle column_family=None):
# cdef map[string, string] value
# cdef Slice c_prop = bytes_to_slice(prop)
# cdef cpp_bool ret = False
# cdef db.ColumnFamilyHandle* cf_handle = NULL
# if column_family:
# cf_handle = column_family.get_handle()
# else:
# cf_handle = self.db.DefaultColumnFamily()

# with nogil:
# ret = self.db.GetMapProperty(cf_handle, c_prop, cython.address(value))

# if ret:
# return value
# else:
# return None

def get_live_files_metadata(self):
cdef vector[db.LiveFileMetaData] metadata
cdef vector[metadata.LiveFileMetaData] metadata

with nogil:
self.db.GetLiveFilesMetaData(cython.address(metadata))
Expand All @@ -2404,7 +2424,7 @@ cdef class DB(object):
return ret

def get_column_family_meta_data(self, ColumnFamilyHandle column_family=None):
cdef db.ColumnFamilyMetaData metadata
cdef metadata.ColumnFamilyMetaData metadata

cdef db.ColumnFamilyHandle* cf_handle = self.db.DefaultColumnFamily()
if column_family:
Expand Down Expand Up @@ -2540,6 +2560,18 @@ def repair_db(db_name, Options opts):
st = db.RepairDB(db_path, deref(opts.opts))
check_status(st)

# TODO Figure out API to add descriptors. See constructor which deals with vector of ColumnFamilyDescriptor
# def repair_db(db_name, Options opts, descriptors):
# cdef Status st
# cdef string db_path
# cdef vector[db.ColumnFamilyDescriptor] c_descriptors

# for d in descriptors:
# c_descriptors.push_back(<db.ColumnFamilyDescriptor>(d))

# db_path = path_to_string(db_name)
# st = db.RepairDB(db_path, deref(opts.opts), c_descriptors)
# check_status(st)

def list_column_families(db_name, Options opts):
cdef Status st
Expand Down
172 changes: 136 additions & 36 deletions rocksdb/db.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,18 @@ from .status cimport Status
from libcpp cimport bool as cpp_bool
from libcpp.string cimport string
from libcpp.vector cimport vector
from libcpp.map cimport map
from libcpp.unordered_map cimport unordered_map
from libcpp.memory cimport shared_ptr
from .types cimport SequenceNumber
from .slice_ cimport Slice
from .snapshot cimport Snapshot
from .iterator cimport Iterator
from .env cimport Env
from .metadata cimport ColumnFamilyMetaData
from .metadata cimport LiveFileMetaData
from .metadata cimport ExportImportFilesMetaData
from .table_properties cimport TableProperties

cdef extern from "rocksdb/write_batch.h" namespace "rocksdb":
cdef cppclass WriteBatch:
Expand Down Expand Up @@ -42,36 +50,11 @@ cdef extern from "cpp/write_batch_iter_helper.hpp" namespace "py_rocks":
cdef extern from "rocksdb/db.h" namespace "rocksdb":
string kDefaultColumnFamilyName

cdef struct LiveFileMetaData:
string name
int level
uint64_t size
string smallestkey
string largestkey
SequenceNumber smallest_seqno
SequenceNumber largest_seqno

# cdef struct SstFileMetaData:
# uint64_t size
# string name
# uint64_t file_number
# string db_path
# string smallestkey
# string largestkey
# SequenceNumber smallest_seqno
# SequenceNumber largest_seqno

# cdef struct LevelMetaData:
# int level
# uint64_t size
# string largestkey
# LiveFileMetaData files

cdef struct ColumnFamilyMetaData:
uint64_t size
uint64_t file_count
# string largestkey
# LevelMetaData levels
# todo TableProperties
ctypedef unordered_map[string, shared_ptr[const TableProperties]] TablePropertiesCollection

cdef struct GetMergeOperandsOptions:
uint64_t expected_max_number_of_operands

cdef cppclass Range:
Range(const Slice&, const Slice&)
Expand All @@ -88,6 +71,12 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
ColumnFamilyHandle*,
const Slice&) nogil except+

Status DeleteRange(
const options.WriteOptions&,
ColumnFamilyHandle*,
const Slice&,
const Slice&) nogil except+

Status Merge(
const options.WriteOptions&,
ColumnFamilyHandle*,
Expand All @@ -104,6 +93,14 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
const Slice&,
string*) nogil except+

# Status GetMergeOperands(
# const options.ReadOptions&,
# ColumnFamilyHandle*,
# const Slice&,
# PinnableSlice*,
# GetMergeOperandsOptions*,
# uint64_t*) nogil except+;

vector[Status] MultiGet(
const options.ReadOptions&,
const vector[ColumnFamilyHandle*]&,
Expand Down Expand Up @@ -141,6 +138,22 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
const Slice&,
string*) nogil except+

cpp_bool GetMapProperty(
ColumnFamilyHandle*,
const Slice&,
map[string, string]*) nogil except+

cpp_bool GetIntProperty(
ColumnFamilyHandle*,
const Slice&,
uint64_t*) nogil except+

Status ResetStats() nogil except+

cpp_bool GetAggregatedIntProperty(
const Slice&,
uint64_t*) nogil except+

void GetApproximateSizes(
ColumnFamilyHandle*,
const Range*
Expand All @@ -153,35 +166,116 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
const Slice*,
const Slice*) nogil except+

Status SetOptions(
ColumnFamilyHandle*,
const unordered_map[string, string]&) nogil except+

Status EnableAutoCompaction(
const vector[ColumnFamilyHandle*]&) nogil except+

void DisableManualCompaction() nogil except+
void EnableManualCompaction() nogil except+

Status CreateColumnFamily(
const options.ColumnFamilyOptions&,
const string&,
ColumnFamilyHandle**) nogil except+

Status CreateColumnFamilies(
const options.ColumnFamilyOptions&,
const vector[string]&,
vector[ColumnFamilyHandle*]*) nogil except+

Status CreateColumnFamilies(
const vector[ColumnFamilyDescriptor]&,
vector[ColumnFamilyHandle*]*) nogil except+

Status DropColumnFamily(
ColumnFamilyHandle*) nogil except+

Status DropColumnFamilies(
vector[ColumnFamilyHandle*]*) nogil except+

Status DestroyColumnFamilyHandle(
ColumnFamilyHandle*) nogil except+

int NumberLevels(ColumnFamilyHandle*) nogil except+
int MaxMemCompactionLevel(ColumnFamilyHandle*) nogil except+
int Level0StopWriteTrigger(ColumnFamilyHandle*) nogil except+
const string& GetName() nogil except+
Env* GetEnv() nogil except+
# TODO Mandar FileSystem* GetFileSystem() nogil except+
const options.Options& GetOptions(ColumnFamilyHandle*) nogil except+
Status Flush(const options.FlushOptions&, ColumnFamilyHandle*) nogil except+
Status Flush(
const options.FlushOptions&,
const vector[ColumnFamilyHandle*]&) nogil except+
Status FlushWAL(bool_cpp) nogil except+
Status SyncWAL() nogil except+
Status LockWAL() nogil except+
Status UnlockWAL() nogil except+

SequenceNumber GetLatestSequenceNumber()
cpp_bool SetPreserveDeletesSequenceNumber(SequenceNumber) nogil except+

Status DisableFileDeletions() nogil except+
Status EnableFileDeletions() nogil except+

Status Close() nogil except+
Status Resume() nogil except+
Status PauseBackgroundWork() nogil except+
Status ContinueBackgroundWork() nogil except+

Status GetDbIdentity(string&) nogil except+
ColumnFamilyHandle* DefaultColumnFamily()

# Following defined for #ifndef ROCKDDB_LITE
Status GetLiveFiles(vector[string]&,
uint64_t*,
cpp_bool) nogil except+
# TODO: Status GetSortedWalFiles(VectorLogPtr& files)
# TODO: SequenceNumber GetLatestSequenceNumber()
# TODO: Status GetCurrentWalFile(std::unique_ptr<LogFile>*)
# TODO: Status GetUpdatesSince(
# SequenceNumber seq_number,
# unique_ptr[TransactionLogIterator]*)

# SequenceNumber seq_number,
# unique_ptr[TransactionLogIterator]*,
# const TransactionLogIterator::ReadOptions&)
Status GetCreationTimeOfOldestFile(uint64_t*) nogil except+
Status DeleteFile(string) nogil except+
void GetLiveFilesMetaData(vector[LiveFileMetaData]*) nogil except+
void GetColumnFamilyMetaData(ColumnFamilyHandle*, ColumnFamilyMetaData*) nogil except+
ColumnFamilyHandle* DefaultColumnFamily()

Status IngestExternalFile(
ColumnFamilyHandle*,
const vector[string]&,
const options.IngestExternalFileOptions&) nogil except+
Status CreateColumnFamilyWithImport(
const options.ColumnFamilyOptions&,
const string&,
const options.ImportColumnFamilyOptions&,
const ExportImportFilesMetaData&,
ColumnFamilyHandle**) nogil except+
Status VerifyChecksum(const options.ReadOptions&) nogil except+

DB* GetRootDB() nogil except+
Status GetPropertiesOfAllTables(
ColumnFamilyHandle*,
TablePropertiesCollection*) nogil except+
Status GetPropertiesOfTablesInRange(
ColumnFamilyHandle*, const Range*, size_t,
TablePropertiesCollection*) nogil except+

Status SuggestCompactRange(ColumnFamilyHandle*,
const Slice*,
const Slice*) nogil except+
Status PromoteL0(ColumnFamilyHandle*, int) nogil except+
#TODO Status StartTrace(const TraceOptions&,
# std::unique_ptr<TraceWriter>&&) nogil except+
Status EndTrace() nogil except+
# TODO Status StartBlockCacheTrace(
# const TraceOptions&,
# std::unique_ptr<TraceWriter>&&) nogil except+
Status EndBlockCacheTrace() nogil except+
Status TryCatchUpWithPrimary() nogil except+

cdef Status DB_Open "rocksdb::DB::Open"(
const options.Options&,
Expand Down Expand Up @@ -209,7 +303,13 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
DB**,
cpp_bool) nogil except+

cdef Status DestroyDB(
const string&,
const options.Options&,
const vector[ColumnFamilyDescriptor]&) nogil except+

cdef Status RepairDB(const string& dbname, const options.Options&)
cdef Status RepairDB(const string& dbname, const options.Options&, const vector[ColumnFamilyDescriptor]&)

cdef Status ListColumnFamilies "rocksdb::DB::ListColumnFamilies" (
const options.Options&,
Expand All @@ -223,7 +323,7 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb":
cdef cppclass ColumnFamilyDescriptor:
ColumnFamilyDescriptor() nogil except+
ColumnFamilyDescriptor(
const string&,
const string&,
const options.ColumnFamilyOptions&) nogil except+
string name
options.ColumnFamilyOptions options
Expand Down
55 changes: 55 additions & 0 deletions rocksdb/metadata.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
from libcpp cimport bool as cpp_bool
from libcpp.string cimport string
from libcpp.vector cimport vector
from libc.stdint cimport uint64_t
from libc.stdint cimport int64_t
from .logger cimport Logger
from .types cimport SequenceNumber

cdef extern from "rocksdb/metadata.h" namespace "rocksdb":
cdef cppclass ColumnFamilyMetaData:
ColumnFamilyMetaData() except+
ColumnFamilyMetaData(const string&, uint64_t,
const vector[LevelMetaData]&&) except+
uint64_t size
uint64_t file_count
string name
vector[LevelMetaData] levels

cdef cppclass LevelMetaData:
int level
uint64_t size
vector[SstFileMetaData] files

cdef cppclass SstFileMetaData:
uint64_t size
string name
uint64_t file_number
string db_path
SequenceNumber smallest_seqno
SequenceNumber largest_seqno
string smallestkey
string largestkey
uint64_t num_reads_sampled
cpp_bool being_compacted
uint64_t num_entries
uint64_t num_deletions
uint64_t oldest_blob_file_number
uint64_t oldest_ancester_time
uint64_t file_creation_time
string file_checksum
string file_checksum_func_name

cdef cppclass LiveFileMetaData(SstFileMetaData):
string column_family_name
int level
uint64_t size
string smallestkey
string largestkey
SequenceNumber smallest_seqno
SequenceNumber largest_seqno

cdef cppclass ExportImportFilesMetaData:
string db_comparator_name
vector[LiveFileMetaData] files