Introducing the Graph module

.gitignore

@@ -10,4 +10,5 @@ recentMorphStoreProjectConf.log
 Dockerfile
 .DS_Store
 doc/doxygen/latex
-doc/doxygen/html   
+doc/doxygen/html
+.vscode/

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required( VERSION 3.10 )
 project( MorphStore )
 
-set( CMAKE_CXX_STANDARD 14 )
+set( CMAKE_CXX_STANDARD 17 )
 
 macro(morph_flag)
 	add_definitions(${ARGN})
@@ -28,9 +28,11 @@ set( LOG_FILE "recentMorphStoreProjectConf.log" )
 
 IF(CMAKE_BUILD_TYPE MATCHES Debug)
 	morph_flag(-g)
+	morph_flag(-DDEBUG)
 	message(STATUS "MorphStore is configured in DEBUG mode.")
 ELSEIF(CMAKE_BUILD_TYPE MATCHES Release)
 	morph_flag(-O2)
+	morph_flag(-DNDEBUG)
 	message(STATUS "MorphStore is configured in RELEASE mode.")
 ELSEIF(CMAKE_BUILD_TYPE MATCHES HighPerf)
 	morph_flag(-O3)
@@ -93,6 +95,10 @@ ENDIF(CODROID)
 # remove build type to allow for custom flag handling
 set(CMAKE_BUILD_TYPE "")
 
+# add resource directory for ldbc graph
+# (see https://github.com/ldbc/ldbc_snb_datagen for further instructions)
+morph_flag(-DLDBC_DIR="$ENV{HOME}/ldbc_snb_datagen/social_network/")
+
 # general compiler settings, meant for all subdirectories and tests
 morph_flag(-Werror)
 morph_flag(-pedantic)

doc/doxygen/pages/tutorials/quick_start.md

@@ -23,7 +23,7 @@ Ensure that you have the following tools installed before trying to build:
 - g++ >= version 8.2
 - cmake >= version 3.10
 
-Older versions may not build all test cases. Note that C++14 is necessary.
+Older versions may not build all test cases. Note that C++17 is necessary.
 
 
 To facilitate building and testing MorphStore, there is a script <i>build.sh</i> in the root folder.
@@ -42,9 +42,29 @@ build/src/examples/example_query
 ~~~
 
 This builds some example queries in debug mode and runs them. The source code of these queries can be found in the folder src/examples.
-They are runnig in scalar mode. Thus, every system providing C++14 support should be able to build and run them regardless of any (not) 
+They are runnig in scalar mode. Thus, every system providing C++17 support should be able to build and run them regardless of any (not) 
 available vector extensions.
 
+
+The Graph Module
+======================
+
+The graph module mainly contains the two different graph storage formats `Compressed Sparse Row (CSR)` and `Adjacency-List`, which differ in their representation of the graph topology . 
+These underlying graph model is a multi-graph, with properties for vertices and edges as well as types for both.
+The model is very similar to the Property-Graph model, except that vertices can only have one type (instead of multiple labels). 
+
+The columns describing the graph topology can be compressed using formats from the MorphStore. 
+Besides there exists simple implementations of the graph algorithms `breadth-first search (bfs)` and `PageRank`.
+
+
+To run all the test and micro-benchmarks, a LDBC graph has to be generated.
+Instructions of how to generate the graph, can be found at `https://github.com/ldbc/ldbc_snb_datagen`.
+By default the ldbc graph is expected to be at `"$HOME/ldbc_snb_datagen/social_network/"`. 
+This can be changed at `/Morphstore/Engine/CMakeLists.txt`.
+
+
+
+
 Test Vector Extensions
 ======================
 

include/core/morphing/decompress_column_block.h

@@ -0,0 +1,95 @@
+/**********************************************************************************************
+ * Copyright (C) 2020 by MorphStore-Team                                                      *
+ *                                                                                            *
+ * This file is part of MorphStore - a compression aware vectorized column store.             *
+ *                                                                                            *
+ * This program is free software: you can redistribute it and/or modify it under the          *
+ * terms of the GNU General Public License as published by the Free Software Foundation,      *
+ * either version 3 of the License, or (at your option) any later version.                    *
+ *                                                                                            *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;  *
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  *
+ * See the GNU General Public License for more details.                                       *
+ *                                                                                            *
+ * You should have received a copy of the GNU General Public License along with this program. *
+ * If not, see <http://www.gnu.org/licenses/>.                                                *
+ **********************************************************************************************/
+
+/**
+ * @file decompress_column_block.h
+ * @brief Decompressing single blocks of a column based on column_with_blockoffsets.
+ */
+
+#ifndef MORPHSTORE_CORE_MORPHING_DECOMPRESS_COLUMN_BLOCK_H
+#define MORPHSTORE_CORE_MORPHING_DECOMPRESS_COLUMN_BLOCK_H
+
+#include <core/morphing/morph_batch.h>
+#include <core/storage/column.h>
+#include <core/storage/column_with_blockoffsets.h>
+
+#include <assert.h>
+
+namespace morphstore {
+
+    /**
+     * @brief Decompressing a range column blocks (inclusive range)
+     *
+     * @param inCol The column with block-offsets
+     * @param start index of blocks to be decompressed
+     * @param end index of blocks to be decompressed
+     * @return Specified blocks uncompressed in a new column
+     */
+    template <class ve, class compr_f>
+    const column<uncompr_f> *decompress_column_blocks(column_with_blockoffsets<compr_f> *inCol, uint64_t start,
+                                                      uint64_t end) {
+        static_assert(compr_f::m_BlockSize != 1, "Decompressing column blocks of size 1 is not allowed");
+
+        auto block_size = compr_f::m_BlockSize;
+        auto block_count = inCol->get_block_offsets()->size();
+        auto inCol_value_count = inCol->get_column()->get_count_values();
+
+        // validating range
+        assert(start <= end);
+        assert(start < block_count);
+        assert(end < block_count);
+
+        bool last_block_uncompressed = !inCol->last_block_compressed();
+        bool last_block_included = end == (block_count - 1);
+
+        // pessimistic value_count (assuming all blocks are complete)
+        auto value_count = (end - start + 1) * block_size;
+
+        if (last_block_included && last_block_uncompressed) {
+            // correcting value_count estimation
+            value_count -= block_size - inCol_value_count % block_size;
+        }
+
+        // TODO: should actually be base_t?
+        auto alloc_size = value_count * sizeof(uint64_t);
+
+        auto decompr_col_blocks = new column<uncompr_f>(alloc_size);
+        decompr_col_blocks->set_meta_data(value_count, alloc_size);
+        uint8_t *out8 = decompr_col_blocks->get_data();
+
+        for (uint64_t block = start; block <= end; block++) {
+            const uint8_t *block_offset = inCol->get_block_offset(block);
+
+            if (block == end && last_block_included && last_block_uncompressed) {
+                // handle uncompressed part
+                morph_batch<ve, uncompr_f, uncompr_f>(block_offset, out8, inCol_value_count % block_size);
+            } else {
+                morph_batch<ve, uncompr_f, compr_f>(block_offset, out8, block_size);
+            }
+        }
+
+        return decompr_col_blocks;
+    }
+
+    template <class ve, class compr_f>
+    const column<uncompr_f> *decompress_column_block(column_with_blockoffsets<compr_f> *inCol, uint64_t block_index) {
+        return decompress_column_blocks<ve, compr_f>(inCol, block_index, block_index);
+    }
+
+} // namespace morphstore
+
+#endif // MORPHSTORE_CORE_MORPHING_DECOMPRESS_COLUMN_BLOCK_H

include/core/morphing/graph/morph_graph_col.h

@@ -0,0 +1,135 @@
+/**********************************************************************************************
+ * Copyright (C) 2020 by MorphStore-Team                                                      *
+ *                                                                                            *
+ * This file is part of MorphStore - a compression aware vectorized column store.             *
+ *                                                                                            *
+ * This program is free software: you can redistribute it and/or modify it under the          *
+ * terms of the GNU General Public License as published by the Free Software Foundation,      *
+ * either version 3 of the License, or (at your option) any later version.                    *
+ *                                                                                            *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;  *
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  *
+ * See the GNU General Public License for more details.                                       *
+ *                                                                                            *
+ * You should have received a copy of the GNU General Public License along with this program. *
+ * If not, see <http://www.gnu.org/licenses/>.                                                *
+ **********************************************************************************************/
+
+/**
+ * @file morph_graph_col.h
+ * @brief helper for morphing graph columns (template-free columns). Basically need to cast to template column as it
+ * cannot be derieved
+ * @todo Remove this helper and make graph formats accept templates (can use normal morph() then)
+ */
+
+#ifndef MORPHSTORE_GRAPH_MORPH_GRAPH_COL_H
+#define MORPHSTORE_GRAPH_MORPH_GRAPH_COL_H
+
+#include <core/storage/graph/graph_compr_format.h>
+#include <core/storage/column.h>
+#include <core/morphing/morph.h>
+
+#include <memory>
+
+namespace morphstore {
+    using column_uncompr = column<uncompr_f>;
+    using column_dyn_vbp = column<default_vbp>;
+    using column_delta = column<default_delta>;
+    using column_for = column<default_for>;
+
+    // casting the column to the actual column type before morphing (as compiler could not derive it)
+    // delete_old_col -> delete input column after morphing (if the result is not the input column)
+    const column_base *morph_graph_col(const column_base *column, const GraphCompressionFormat src_f,
+                                       const GraphCompressionFormat trg_f, bool delete_in_col = false) {
+        if (src_f == trg_f) {
+            return column;
+        }
+
+        const column_base *result = column;
+
+        switch (src_f) {
+        case GraphCompressionFormat::UNCOMPRESSED: {
+            const column_uncompr *old_col = dynamic_cast<const column_uncompr *>(column);
+            switch (trg_f) {
+            case GraphCompressionFormat::DELTA:
+                result = morph<ve, default_delta, uncompr_f>(old_col);
+                break;
+            case GraphCompressionFormat::FOR:
+                result = morph<ve, default_for, uncompr_f>(old_col);
+                break;
+            case GraphCompressionFormat::DYNAMIC_VBP:
+                result = morph<ve, default_vbp, uncompr_f>(old_col);
+                break;
+            case GraphCompressionFormat::UNCOMPRESSED:
+                // handled by src_f == trg_f
+                break;
+            }
+            break;
+        }
+        case GraphCompressionFormat::DELTA: {
+            if (trg_f == GraphCompressionFormat::UNCOMPRESSED) {
+                const column_delta *old_col = dynamic_cast<const column_delta *>(column);
+                result = morph<ve, uncompr_f, default_delta>(old_col);
+            } else {
+                // as direct morphing is not yet supported .. go via decompressing first
+                auto uncompr_col = morph_graph_col(column, src_f, GraphCompressionFormat::UNCOMPRESSED, false);
+                result = morph_graph_col(uncompr_col, GraphCompressionFormat::UNCOMPRESSED, trg_f, true);
+            }
+            break;
+        }
+        case GraphCompressionFormat::FOR: {
+            if (trg_f == GraphCompressionFormat::UNCOMPRESSED) {
+                const column_for *old_col = dynamic_cast<const column_for *>(column);
+                result = morph<ve, uncompr_f, default_for>(old_col);
+            } else {
+                // as direct morphing is not yet supported .. go via decompressing first
+                auto uncompr_col = morph_graph_col(column, src_f, GraphCompressionFormat::UNCOMPRESSED, false);
+                result = morph_graph_col(uncompr_col, GraphCompressionFormat::UNCOMPRESSED, trg_f, true);
+            }
+            break;
+        }
+        case GraphCompressionFormat::DYNAMIC_VBP: {
+            if (trg_f == GraphCompressionFormat::UNCOMPRESSED) {
+                const column_dyn_vbp *old_col = dynamic_cast<const column_dyn_vbp *>(column);
+                result = morph<ve, uncompr_f, default_vbp>(old_col);
+            } else {
+                // as direct morphing is not yet supported .. go via decompressing first
+                auto uncompr_col = morph_graph_col(column, src_f, GraphCompressionFormat::UNCOMPRESSED, false);
+                // delete_in_col = true as temporary uncompr_col should always be deleted
+                result = morph_graph_col(uncompr_col, GraphCompressionFormat::UNCOMPRESSED, trg_f, true);
+            }
+            break;
+        }
+        }
+
+        // free input column if possible
+        if (result != column && delete_in_col) {
+            delete column;
+        }
+
+        if (result == nullptr) {
+            throw std::runtime_error("Did not handle src: " + graph_compr_f_to_string(src_f) +
+                                     " trg: " + graph_compr_f_to_string(trg_f));
+        }
+
+        return result;
+    }
+
+    const column_uncompr *decompress_graph_col(const column_base *column, const GraphCompressionFormat src_f) {
+        return static_cast<const column_uncompr *>(
+            morph_graph_col(column, src_f, GraphCompressionFormat::UNCOMPRESSED, false));
+    }
+
+    double compression_ratio(const column_base *col, GraphCompressionFormat col_format) {
+        auto uncompr_col = decompress_graph_col(col, col_format);
+        auto ratio = uncompr_col->get_size_used_byte() / (double)col->get_size_used_byte();
+
+        if (col != uncompr_col) {
+            delete uncompr_col;
+        }
+
+        return ratio;
+    }
+} // namespace morphstore
+
+#endif // MORPHSTORE_GRAPH_MORPH_GRAPH_COL_H