diff --git a/rocksdb/_rocksdb.pyx b/rocksdb/_rocksdb.pyx index 8b3d116f..b25b68a0 100644 --- a/rocksdb/_rocksdb.pyx +++ b/rocksdb/_rocksdb.pyx @@ -3,6 +3,7 @@ import cython from libcpp.string cimport string from libcpp.deque cimport deque from libcpp.vector cimport vector +from libcpp.map cimport map from cpython cimport bool as py_bool from libcpp cimport bool as cpp_bool from libc.stdint cimport uint32_t @@ -29,6 +30,7 @@ from . cimport env from . cimport table_factory from . cimport memtablerep from . cimport universal_compaction +from . cimport metadata # Enums are the only exception for direct imports # Their name als already unique enough @@ -2382,8 +2384,26 @@ cdef class DB(object): else: return None + # def get_map_property(self, prop, ColumnFamilyHandle column_family=None): + # cdef map[string, string] value + # cdef Slice c_prop = bytes_to_slice(prop) + # cdef cpp_bool ret = False + # cdef db.ColumnFamilyHandle* cf_handle = NULL + # if column_family: + # cf_handle = column_family.get_handle() + # else: + # cf_handle = self.db.DefaultColumnFamily() + + # with nogil: + # ret = self.db.GetMapProperty(cf_handle, c_prop, cython.address(value)) + + # if ret: + # return value + # else: + # return None + def get_live_files_metadata(self): - cdef vector[db.LiveFileMetaData] metadata + cdef vector[metadata.LiveFileMetaData] metadata with nogil: self.db.GetLiveFilesMetaData(cython.address(metadata)) @@ -2404,7 +2424,7 @@ cdef class DB(object): return ret def get_column_family_meta_data(self, ColumnFamilyHandle column_family=None): - cdef db.ColumnFamilyMetaData metadata + cdef metadata.ColumnFamilyMetaData metadata cdef db.ColumnFamilyHandle* cf_handle = self.db.DefaultColumnFamily() if column_family: @@ -2540,6 +2560,18 @@ def repair_db(db_name, Options opts): st = db.RepairDB(db_path, deref(opts.opts)) check_status(st) +# TODO Figure out API to add descriptors. See constructor which deals with vector of ColumnFamilyDescriptor +# def repair_db(db_name, Options opts, descriptors): +# cdef Status st +# cdef string db_path +# cdef vector[db.ColumnFamilyDescriptor] c_descriptors + +# for d in descriptors: +# c_descriptors.push_back((d)) + +# db_path = path_to_string(db_name) +# st = db.RepairDB(db_path, deref(opts.opts), c_descriptors) +# check_status(st) def list_column_families(db_name, Options opts): cdef Status st diff --git a/rocksdb/db.pxd b/rocksdb/db.pxd index 6cc4d112..4ffe954c 100644 --- a/rocksdb/db.pxd +++ b/rocksdb/db.pxd @@ -4,10 +4,18 @@ from .status cimport Status from libcpp cimport bool as cpp_bool from libcpp.string cimport string from libcpp.vector cimport vector +from libcpp.map cimport map +from libcpp.unordered_map cimport unordered_map +from libcpp.memory cimport shared_ptr from .types cimport SequenceNumber from .slice_ cimport Slice from .snapshot cimport Snapshot from .iterator cimport Iterator +from .env cimport Env +from .metadata cimport ColumnFamilyMetaData +from .metadata cimport LiveFileMetaData +from .metadata cimport ExportImportFilesMetaData +from .table_properties cimport TableProperties cdef extern from "rocksdb/write_batch.h" namespace "rocksdb": cdef cppclass WriteBatch: @@ -42,36 +50,11 @@ cdef extern from "cpp/write_batch_iter_helper.hpp" namespace "py_rocks": cdef extern from "rocksdb/db.h" namespace "rocksdb": string kDefaultColumnFamilyName - cdef struct LiveFileMetaData: - string name - int level - uint64_t size - string smallestkey - string largestkey - SequenceNumber smallest_seqno - SequenceNumber largest_seqno - - # cdef struct SstFileMetaData: - # uint64_t size - # string name - # uint64_t file_number - # string db_path - # string smallestkey - # string largestkey - # SequenceNumber smallest_seqno - # SequenceNumber largest_seqno - - # cdef struct LevelMetaData: - # int level - # uint64_t size - # string largestkey - # LiveFileMetaData files - - cdef struct ColumnFamilyMetaData: - uint64_t size - uint64_t file_count - # string largestkey - # LevelMetaData levels + # todo TableProperties + ctypedef unordered_map[string, shared_ptr[const TableProperties]] TablePropertiesCollection + + cdef struct GetMergeOperandsOptions: + uint64_t expected_max_number_of_operands cdef cppclass Range: Range(const Slice&, const Slice&) @@ -88,6 +71,12 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb": ColumnFamilyHandle*, const Slice&) nogil except+ + Status DeleteRange( + const options.WriteOptions&, + ColumnFamilyHandle*, + const Slice&, + const Slice&) nogil except+ + Status Merge( const options.WriteOptions&, ColumnFamilyHandle*, @@ -104,6 +93,14 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb": const Slice&, string*) nogil except+ + # Status GetMergeOperands( + # const options.ReadOptions&, + # ColumnFamilyHandle*, + # const Slice&, + # PinnableSlice*, + # GetMergeOperandsOptions*, + # uint64_t*) nogil except+; + vector[Status] MultiGet( const options.ReadOptions&, const vector[ColumnFamilyHandle*]&, @@ -141,6 +138,22 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb": const Slice&, string*) nogil except+ + cpp_bool GetMapProperty( + ColumnFamilyHandle*, + const Slice&, + map[string, string]*) nogil except+ + + cpp_bool GetIntProperty( + ColumnFamilyHandle*, + const Slice&, + uint64_t*) nogil except+ + + Status ResetStats() nogil except+ + + cpp_bool GetAggregatedIntProperty( + const Slice&, + uint64_t*) nogil except+ + void GetApproximateSizes( ColumnFamilyHandle*, const Range* @@ -153,35 +166,116 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb": const Slice*, const Slice*) nogil except+ + Status SetOptions( + ColumnFamilyHandle*, + const unordered_map[string, string]&) nogil except+ + + Status EnableAutoCompaction( + const vector[ColumnFamilyHandle*]&) nogil except+ + + void DisableManualCompaction() nogil except+ + void EnableManualCompaction() nogil except+ + Status CreateColumnFamily( const options.ColumnFamilyOptions&, const string&, ColumnFamilyHandle**) nogil except+ + Status CreateColumnFamilies( + const options.ColumnFamilyOptions&, + const vector[string]&, + vector[ColumnFamilyHandle*]*) nogil except+ + + Status CreateColumnFamilies( + const vector[ColumnFamilyDescriptor]&, + vector[ColumnFamilyHandle*]*) nogil except+ + Status DropColumnFamily( ColumnFamilyHandle*) nogil except+ + Status DropColumnFamilies( + vector[ColumnFamilyHandle*]*) nogil except+ + + Status DestroyColumnFamilyHandle( + ColumnFamilyHandle*) nogil except+ + int NumberLevels(ColumnFamilyHandle*) nogil except+ int MaxMemCompactionLevel(ColumnFamilyHandle*) nogil except+ int Level0StopWriteTrigger(ColumnFamilyHandle*) nogil except+ const string& GetName() nogil except+ + Env* GetEnv() nogil except+ + # TODO Mandar FileSystem* GetFileSystem() nogil except+ const options.Options& GetOptions(ColumnFamilyHandle*) nogil except+ Status Flush(const options.FlushOptions&, ColumnFamilyHandle*) nogil except+ + Status Flush( + const options.FlushOptions&, + const vector[ColumnFamilyHandle*]&) nogil except+ + Status FlushWAL(bool_cpp) nogil except+ + Status SyncWAL() nogil except+ + Status LockWAL() nogil except+ + Status UnlockWAL() nogil except+ + + SequenceNumber GetLatestSequenceNumber() + cpp_bool SetPreserveDeletesSequenceNumber(SequenceNumber) nogil except+ + Status DisableFileDeletions() nogil except+ Status EnableFileDeletions() nogil except+ + Status Close() nogil except+ + Status Resume() nogil except+ + Status PauseBackgroundWork() nogil except+ + Status ContinueBackgroundWork() nogil except+ + + Status GetDbIdentity(string&) nogil except+ + ColumnFamilyHandle* DefaultColumnFamily() + # Following defined for #ifndef ROCKDDB_LITE + Status GetLiveFiles(vector[string]&, + uint64_t*, + cpp_bool) nogil except+ # TODO: Status GetSortedWalFiles(VectorLogPtr& files) - # TODO: SequenceNumber GetLatestSequenceNumber() + # TODO: Status GetCurrentWalFile(std::unique_ptr*) # TODO: Status GetUpdatesSince( - # SequenceNumber seq_number, - # unique_ptr[TransactionLogIterator]*) - + # SequenceNumber seq_number, + # unique_ptr[TransactionLogIterator]*, + # const TransactionLogIterator::ReadOptions&) + Status GetCreationTimeOfOldestFile(uint64_t*) nogil except+ Status DeleteFile(string) nogil except+ void GetLiveFilesMetaData(vector[LiveFileMetaData]*) nogil except+ void GetColumnFamilyMetaData(ColumnFamilyHandle*, ColumnFamilyMetaData*) nogil except+ - ColumnFamilyHandle* DefaultColumnFamily() + Status IngestExternalFile( + ColumnFamilyHandle*, + const vector[string]&, + const options.IngestExternalFileOptions&) nogil except+ + Status CreateColumnFamilyWithImport( + const options.ColumnFamilyOptions&, + const string&, + const options.ImportColumnFamilyOptions&, + const ExportImportFilesMetaData&, + ColumnFamilyHandle**) nogil except+ + Status VerifyChecksum(const options.ReadOptions&) nogil except+ + + DB* GetRootDB() nogil except+ + Status GetPropertiesOfAllTables( + ColumnFamilyHandle*, + TablePropertiesCollection*) nogil except+ + Status GetPropertiesOfTablesInRange( + ColumnFamilyHandle*, const Range*, size_t, + TablePropertiesCollection*) nogil except+ + + Status SuggestCompactRange(ColumnFamilyHandle*, + const Slice*, + const Slice*) nogil except+ + Status PromoteL0(ColumnFamilyHandle*, int) nogil except+ + #TODO Status StartTrace(const TraceOptions&, + # std::unique_ptr&&) nogil except+ + Status EndTrace() nogil except+ + # TODO Status StartBlockCacheTrace( + # const TraceOptions&, + # std::unique_ptr&&) nogil except+ + Status EndBlockCacheTrace() nogil except+ + Status TryCatchUpWithPrimary() nogil except+ cdef Status DB_Open "rocksdb::DB::Open"( const options.Options&, @@ -209,7 +303,13 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb": DB**, cpp_bool) nogil except+ + cdef Status DestroyDB( + const string&, + const options.Options&, + const vector[ColumnFamilyDescriptor]&) nogil except+ + cdef Status RepairDB(const string& dbname, const options.Options&) + cdef Status RepairDB(const string& dbname, const options.Options&, const vector[ColumnFamilyDescriptor]&) cdef Status ListColumnFamilies "rocksdb::DB::ListColumnFamilies" ( const options.Options&, @@ -223,7 +323,7 @@ cdef extern from "rocksdb/db.h" namespace "rocksdb": cdef cppclass ColumnFamilyDescriptor: ColumnFamilyDescriptor() nogil except+ ColumnFamilyDescriptor( - const string&, + const string&, const options.ColumnFamilyOptions&) nogil except+ string name options.ColumnFamilyOptions options diff --git a/rocksdb/metadata.pxd b/rocksdb/metadata.pxd new file mode 100644 index 00000000..cea3da73 --- /dev/null +++ b/rocksdb/metadata.pxd @@ -0,0 +1,55 @@ +from libcpp cimport bool as cpp_bool +from libcpp.string cimport string +from libcpp.vector cimport vector +from libc.stdint cimport uint64_t +from libc.stdint cimport int64_t +from .logger cimport Logger +from .types cimport SequenceNumber + +cdef extern from "rocksdb/metadata.h" namespace "rocksdb": + cdef cppclass ColumnFamilyMetaData: + ColumnFamilyMetaData() except+ + ColumnFamilyMetaData(const string&, uint64_t, + const vector[LevelMetaData]&&) except+ + uint64_t size + uint64_t file_count + string name + vector[LevelMetaData] levels + + cdef cppclass LevelMetaData: + int level + uint64_t size + vector[SstFileMetaData] files + + cdef cppclass SstFileMetaData: + uint64_t size + string name + uint64_t file_number + string db_path + SequenceNumber smallest_seqno + SequenceNumber largest_seqno + string smallestkey + string largestkey + uint64_t num_reads_sampled + cpp_bool being_compacted + uint64_t num_entries + uint64_t num_deletions + uint64_t oldest_blob_file_number + uint64_t oldest_ancester_time + uint64_t file_creation_time + string file_checksum + string file_checksum_func_name + + cdef cppclass LiveFileMetaData(SstFileMetaData): + string column_family_name + int level + uint64_t size + string smallestkey + string largestkey + SequenceNumber smallest_seqno + SequenceNumber largest_seqno + + cdef cppclass ExportImportFilesMetaData: + string db_comparator_name + vector[LiveFileMetaData] files +