Skip to content

Sparse PointData #556

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
JulianThijssen wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

Sparse PointData #556

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
513b1af
Add initial sparse matrix implementation
JulianThijssen Mar 4, 2024
e02ee0d
Fix issue with reading in sparse data from project files
JulianThijssen Mar 23, 2024
504cfa5
Add fromValue
alxvth Apr 5, 2024
0655792
Add getters and setters
JulianThijssen Apr 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ManiVault/src/plugins/PointData/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ set(POINTS_SOURCES
src/PointDataRange.h
src/PointView.h
src/RandomAccessRange.h
src/SparseMatrix.cpp
src/SparseMatrix.h
)

set(POINTS_HEADERS
Expand All @@ -53,6 +55,8 @@ set(POINTS_HEADERS
src/DimensionsPickerProxyModel.h
src/ModelResetter.h
src/CreateSetFromSelectionAction.h
src/SparseMatrix.cpp
src/SparseMatrix.h
)

# Note: This nesting of folders is intentional to allow uniform header inclusion in all projects
Expand Down
179 changes: 143 additions & 36 deletions ManiVault/src/plugins/PointData/src/PointData.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,10 @@ mv::Dataset<DatasetImpl> PointData::createDataSet(const QString& guid /*= ""*/)

std::uint32_t PointData::getNumPoints() const
{
return static_cast<unsigned int>(getSizeOfVector() / _numDimensions);
if (_isDense)
return static_cast<unsigned int>(getSizeOfVector() / _numDimensions);
else
return _numRows;
}

std::uint32_t PointData::getNumDimensions() const
Expand All @@ -65,8 +68,15 @@ std::uint64_t PointData::getNumberOfElements() const

std::uint64_t PointData::getRawDataSize() const
{
std::uint64_t elementSize = std::visit([](auto& vec) { return vec.empty() ? 0u : sizeof(vec[0]); }, _variantOfVectors);
return elementSize * getNumberOfElements();
if (_isDense)
{
std::uint64_t elementSize = std::visit([](auto& vec) { return vec.empty() ? 0u : sizeof(vec[0]); }, _variantOfVectors);
return elementSize * getNumberOfElements();
}
else
{
return (_numRows + 4) * sizeof(size_t) + _sparseData.getNumNonZeros() * 2 * sizeof(uint16_t);
}
}

void* PointData::getDataVoidPtr()
Expand Down Expand Up @@ -101,6 +111,18 @@ void PointData::setDimensionNames(const std::vector<QString>& dimNames)
qWarning() << "PointData: Number of dimension names does not equal the number of data dimensions";
}

std::vector<float> PointData::row(size_t rowIndex) const
{
if (!_isDense)
{
return _sparseData.getDenseRow(rowIndex);
}
else
{
qWarning() << ".row() not implemented for dense data";
}
}

float PointData::getValueAt(const std::size_t index) const
{
return std::visit([index](const auto& vec)
Expand All @@ -125,37 +147,97 @@ void PointData::fromVariantMap(const QVariantMap& variantMap)
variantMapMustContain(variantMap, "Data");
variantMapMustContain(variantMap, "NumberOfPoints");
variantMapMustContain(variantMap, "NumberOfDimensions");
variantMapMustContain(variantMap, "Dense");

const auto data = variantMap["Data"].toMap();
const auto numberOfPoints = static_cast<size_t>(variantMap["NumberOfPoints"].toInt());
const auto numberOfDimensions =variantMap["NumberOfDimensions"].toUInt();
const auto numberOfDimensions = variantMap["NumberOfDimensions"].toUInt();
const auto numberOfElements = numberOfPoints * numberOfDimensions;
const auto elementTypeIndex = static_cast<PointData::ElementTypeSpecifier>(data["TypeIndex"].toInt());
const auto rawData = data["Raw"].toMap();
const bool isDense = variantMap["Dense"].toBool();

_isDense = isDense;

setElementTypeSpecifier(elementTypeIndex);
resizeVector(numberOfElements);
populateDataBufferFromVariantMap(rawData, (char*)getDataVoidPtr());
_numDimensions = numberOfDimensions;

if (_isDense)
{
setElementTypeSpecifier(elementTypeIndex);
resizeVector(numberOfElements);
populateDataBufferFromVariantMap(rawData, (char*)getDataVoidPtr());
_numDimensions = numberOfDimensions;
}
else
{
variantMapMustContain(variantMap, "NumberOfNonZeroElements");

const auto numberOfNonZeroElements = variantMap["NumberOfNonZeroElements"].toULongLong();

std::vector<char> bytes((numberOfPoints + 1) * sizeof(size_t) + numberOfNonZeroElements * sizeof(uint16_t) * 2);

populateDataBufferFromVariantMap(rawData, bytes.data());
_numRows = numberOfPoints; // FIXME should be redundant
_numDimensions = numberOfDimensions;

size_t offset = 0;
std::vector<size_t> rowPointers(numberOfPoints + 1);
std::memcpy(rowPointers.data(), bytes.data() + offset, rowPointers.size() * sizeof(size_t));

offset += rowPointers.size() * sizeof(size_t);
std::vector<uint16_t> colIndices(numberOfNonZeroElements);
std::memcpy(colIndices.data(), bytes.data() + offset, colIndices.size() * sizeof(uint16_t));

offset += colIndices.size() * sizeof(uint16_t);
std::vector<uint16_t> values(numberOfNonZeroElements);
std::memcpy(values.data(), bytes.data() + offset, values.size() * sizeof(uint16_t));

_sparseData.setData(numberOfPoints, numberOfDimensions, rowPointers, colIndices, values);

qDebug() << "Loaded sparse data with" << _numRows << "points and" << _numDimensions << "dimensions.";
}
}

QVariantMap PointData::toVariantMap() const
{
const auto typeSpecifier = getElementTypeSpecifier();
const auto typeSpecifierName = getElementTypeNames()[static_cast<std::int32_t>(typeSpecifier)];
const auto typeIndex = static_cast<std::int32_t>(typeSpecifier);
const auto numberOfElements = getNumberOfElements();

QVariantMap rawData = rawDataToVariantMap((const char*)getDataConstVoidPtr(), getRawDataSize(), true);

return {
{ "TypeIndex", QVariant::fromValue(typeIndex) },
{ "TypeName", QVariant(typeSpecifierName) },
{ "Raw", QVariant::fromValue(rawData) },
{ "NumberOfElements", QVariant::fromValue(numberOfElements) }
};
if (_isDense)
{
const auto typeSpecifier = getElementTypeSpecifier();
const auto typeSpecifierName = getElementTypeNames()[static_cast<std::int32_t>(typeSpecifier)];
const auto typeIndex = static_cast<std::int32_t>(typeSpecifier);
const auto numberOfElements = getNumberOfElements();

QVariantMap rawData = rawDataToVariantMap((const char*)getDataConstVoidPtr(), getRawDataSize(), true);

return {
{ "TypeIndex", QVariant::fromValue(typeIndex) },
{ "TypeName", QVariant(typeSpecifierName) },
{ "Raw", QVariant::fromValue(rawData) },
{ "NumberOfElements", QVariant::fromValue(numberOfElements) }
};
}
else
{
const auto numberOfElements = getNumberOfElements();

std::vector<char> bytes;

const std::vector<size_t>& indexPointers = _sparseData.getIndexPointers();
const std::vector<uint16_t>& colIndices = _sparseData.getColIndices();
const std::vector<uint16_t>& values = _sparseData.getValues();

const char* indexPointersBytes = (const char*) (indexPointers.data());
const char* colIndicesBytes = (const char*) (colIndices.data());
const char* valuesBytes = (const char*) (values.data());

bytes.insert(bytes.end(), indexPointersBytes, indexPointersBytes + indexPointers.size() * sizeof(size_t));
bytes.insert(bytes.end(), colIndicesBytes, colIndicesBytes + colIndices.size() * sizeof(uint16_t));
bytes.insert(bytes.end(), valuesBytes, valuesBytes + values.size() * sizeof(uint16_t));

QVariantMap rawData = rawDataToVariantMap(bytes.data(), bytes.size(), true);

return {
{ "Raw", QVariant::fromValue(rawData) }
};
}
}

void PointData::extractFullDataForDimension(std::vector<float>& result, const int dimensionIndex) const
Expand All @@ -180,23 +262,38 @@ void PointData::extractFullDataForDimension(std::vector<float>& result, const in

void PointData::extractFullDataForDimensions(std::vector<mv::Vector2f>& result, const int dimensionIndex1, const int dimensionIndex2) const
{
CheckDimensionIndex(dimensionIndex1);
CheckDimensionIndex(dimensionIndex2);

result.resize(getNumPoints());
if (_isDense)
{
CheckDimensionIndex(dimensionIndex1);
CheckDimensionIndex(dimensionIndex2);

std::visit(
[&result, this, dimensionIndex1, dimensionIndex2](const auto& vec)
{
const auto resultSize = result.size();
result.resize(getNumPoints());

for (std::size_t i{}; i < resultSize; ++i)
std::visit(
[&result, this, dimensionIndex1, dimensionIndex2](const auto& vec)
{
const auto n = i * _numDimensions;
result[i].set(vec[n + dimensionIndex1], vec[n + dimensionIndex2]);
}
},
_variantOfVectors);
const auto resultSize = result.size();

for (std::size_t i{}; i < resultSize; ++i)
{
const auto n = i * _numDimensions;
result[i].set(vec[n + dimensionIndex1], vec[n + dimensionIndex2]);
}
},
_variantOfVectors);
}
else
{
result.resize(getNumPoints());

std::vector<float> col1 = _sparseData.getDenseCol(dimensionIndex1);
std::vector<float> col2 = _sparseData.getDenseCol(dimensionIndex2);

for (size_t i = 0; i < result.size(); i++)
{
result[i].set(col1[i], col2[i]);
}
}
}


Expand Down Expand Up @@ -723,6 +820,11 @@ void Points::setDimensionNames(const std::vector<QString>& dimNames)
mv::events().notifyDatasetDataDimensionsChanged(this);
}

std::vector<float> Points::row(size_t rowIndex) const
{
return getRawData<PointData>()->row(rowIndex);
}

float Points::getValueAt(const std::size_t index) const
{
return getRawData<PointData>()->getValueAt(index);
Expand Down Expand Up @@ -1056,6 +1158,11 @@ QVariantMap Points::toVariantMap() const
variantMap["DimensionNames"] = (dimensionNames.size() > 1000) ? rawDataToVariantMap((char*)dimensionsByteArray.data(), dimensionsByteArray.size(), true) : QVariant::fromValue(dimensionNames);
variantMap["NumberOfDimensions"] = getNumDimensions();
variantMap["Dimensions"] = _dimensionsPickerAction->toVariantMap();
variantMap["Dense"] = getRawData<PointData>()->isDense();
if (!getRawData<PointData>()->isDense())
{
variantMap["NumberOfNonZeroElements"] = QVariant::fromValue(getRawData<PointData>()->getNumNonZeroElements());
}

return variantMap;
}
Expand Down
62 changes: 60 additions & 2 deletions ManiVault/src/plugins/PointData/src/PointData.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "LinkedData.h"
#include "PointDataRange.h"
#include "Set.h"
#include "SparseMatrix.h"

#include "event/EventListener.h"

Expand Down Expand Up @@ -216,6 +217,8 @@ class POINTDATA_EXPORT PointData : public mv::plugin::RawData

std::uint64_t getNumberOfElements() const;

SparseMatrix<uint16_t, uint16_t>& getSparseData() { return _sparseData; }

/**
* Get amount of data occupied by the raw data
* @return Size of the raw data in bytes
Expand Down Expand Up @@ -269,7 +272,7 @@ class POINTDATA_EXPORT PointData : public mv::plugin::RawData
void extractFullDataForDimension(std::vector<float>& result, const int dimensionIndex) const;
void extractFullDataForDimensions(std::vector<mv::Vector2f>& result, const int dimensionIndex1, const int dimensionIndex2) const;
void extractDataForDimensions(std::vector<mv::Vector2f>& result, const int dimensionIndex1, const int dimensionIndex2, const std::vector<unsigned int>& indices) const;

template <typename ResultContainer, typename DimensionIndices>
void populateFullDataForDimensions(ResultContainer& resultContainer, const DimensionIndices& dimensionIndices) const
{
Expand Down Expand Up @@ -399,8 +402,38 @@ class POINTDATA_EXPORT PointData : public mv::plugin::RawData
_numDimensions = static_cast<unsigned int>(numDimensions);
}

template <typename ColIndexType, typename ValueType>
void setSparseData(size_t numRows, size_t numCols, std::vector<size_t> rowPointers, std::vector<ColIndexType> colIndices, std::vector<ValueType> values)
{
_numRows = numRows;
_numDimensions = numCols;
_sparseData.setData(numRows, numCols, rowPointers, colIndices, values);

_isDense = false;
}

void setSparseData(size_t numRows, size_t numCols)
{
_numRows = numRows;
_numDimensions = numCols;

_isDense = false;
}

bool isDense() const
{
return _isDense;
}

size_t getNumNonZeroElements()
{
return _sparseData.getNumNonZeros();
}

void setDimensionNames(const std::vector<QString>& dimNames);

std::vector<float> row(size_t rowIndex) const;

// Returns the value of the element at the specified position in the current
// data vector, converted to float.
// Will work fine, even when the internal data element type is not float.
Expand Down Expand Up @@ -432,6 +465,12 @@ class POINTDATA_EXPORT PointData : public mv::plugin::RawData
unsigned int _numDimensions = 1;

std::vector<QString> _dimNames;

// Sparse data
unsigned int _numRows = 0;
SparseMatrix<uint16_t, uint16_t> _sparseData;

bool _isDense = true;
};

// =============================================================================
Expand Down Expand Up @@ -634,6 +673,11 @@ class POINTDATA_EXPORT Points : public mv::DatasetImpl
mv::events().notifyDatasetDataDimensionsChanged(this);
}

SparseMatrix<uint16_t, uint16_t>& getSparseData()
{
return getRawData<PointData>()->getSparseData();
}

/// Just calls the corresponding member function of its PointData.
void setData(std::nullptr_t data, std::size_t numPoints, std::size_t numDimensions);

Expand Down Expand Up @@ -661,6 +705,17 @@ class POINTDATA_EXPORT Points : public mv::DatasetImpl
mv::events().notifyDatasetDataDimensionsChanged(this);
}

template <typename ColIndexType, typename ValueType>
void setSparseData(size_t numRows, size_t numCols, std::vector<size_t> rowPointers, std::vector<ColIndexType> colIndices, std::vector<ValueType> values)
{
getRawData<PointData>()->setSparseData(numRows, numCols, rowPointers, colIndices, values);
}

void setSparseData(size_t numRows, size_t numCols)
{
getRawData<PointData>()->setSparseData(numRows, numCols);
}

void extractDataForDimension(std::vector<float>& result, const int dimensionIndex) const;

void extractDataForDimensions(std::vector<mv::Vector2f>& result, const int dimensionIndex1, const int dimensionIndex2) const;
Expand Down Expand Up @@ -766,10 +821,13 @@ class POINTDATA_EXPORT Points : public mv::DatasetImpl
std::uint64_t getRawDataSize() const override {
if (isProxy())
return 0;
else
else {
return getRawData<PointData>()->getRawDataSize();
}
}

std::vector<float> row(size_t rowIndex) const;

// Returns the value of the element at the specified position in the current
// data vector, converted to float.
// Will work fine, even when the internal data element type is not float.
Expand Down
Loading
Loading