From 7139f7c3858edd57d6e40c8d825b3e081aa3e90c Mon Sep 17 00:00:00 2001 From: TonHai1111 Date: Sat, 28 Aug 2021 01:46:27 -0500 Subject: [PATCH 01/12] Fixed addSpatialRange with NULL bug. Added print interface --- PySTARE.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/PySTARE.h b/PySTARE.h index b2749c5..0490a11 100644 --- a/PySTARE.h +++ b/PySTARE.h @@ -197,6 +197,7 @@ class srange { SpatialRange range; STARE_SpatialIntervals sis; STARE_ArrayIndexSpatialValues sivs; + void print(){range.print();}; void add_intersect(const srange& one, const srange& other,bool compress) { // cout << " compress " << compress << endl << flush; @@ -225,7 +226,8 @@ class srange { SpatialRange *res = sr_intersect(one.range,other.range,compress); // cout << 1100 << " 11 nr = " << res->range->range->nranges() << endl << flush; // srange result; result.set_tag(999); - range.addSpatialRange(*res); + if(res != NULL) + range.addSpatialRange(*res); // STARE_SpatialIntervals sis_res = res->toSpatialIntervals(); // cout << 1150 << endl << flush; // range.addSpatialIntervals(sis_res); From 9c86110f66531ee083ea51e7b70cd97929369824 Mon Sep 17 00:00:00 2001 From: TonHai1111 Date: Thu, 9 Sep 2021 16:45:51 -0500 Subject: [PATCH 02/12] Fixed add_range interface --- PySTARE.h | 57 ++++++++++++++++++++----------------------------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/PySTARE.h b/PySTARE.h index 0490a11..de520c9 100644 --- a/PySTARE.h +++ b/PySTARE.h @@ -134,6 +134,7 @@ class StareResult { void copy (int64_t* indices, int len); void copy_as_values (int64_t* indices, int len); void copy_as_intervals(int64_t* indices, int len); + //void copy_as_list_list(int64_t* indices1, int len1, int64_t* indices2, int len2); void convert(); bool converted = false; STARE_SpatialIntervals sis; @@ -171,7 +172,9 @@ class srange { virtual ~srange(); void add_intervals(int64_t* indices, int len); - void add_range(const SpatialRange& r) { range.addSpatialRange(r); } + void add_range(const srange& r) { + range.addSpatialRange(r.range); + } // bool contains(int64_t siv); bool contains(long long siv); @@ -197,44 +200,26 @@ class srange { SpatialRange range; STARE_SpatialIntervals sis; STARE_ArrayIndexSpatialValues sivs; - void print(){range.print();}; void add_intersect(const srange& one, const srange& other,bool compress) { - // cout << " compress " << compress << endl << flush; - -// HstmRange *range1 = new HstmRange(range.range->range->RangeFromIntersection(other.range.range->range,compress)); // NOTE mlr Probably about the safest way to inst. SpatialRange. -// // #define DIAG -// #ifdef DIAG -// KeyPair kp; -// range1->reset(); -// range1->getNext(kp); -// cout << "sr_i range1,r->r,nr " << range1 << " " << range1->range << " " << range1->range->nranges() << " : " -// << setw(16) << setfill('0') << hex << kp.lo << " " -// << setw(16) << setfill('0') << hex << kp.hi << " " -// << dec -// << endl << flush; -// EmbeddedLevelNameEncoding leftJustified; -// leftJustified.setId(kp.lo); -// cout << "kp.lo lj " << setw(16) << setfill('0') << hex << leftJustified.getSciDBLeftJustifiedFormat() << endl << flush; -// leftJustified.setId(kp.hi); cout << "kp.hi lj " << setw(16) << setfill('0') << hex << leftJustified.getSciDBLeftJustifiedFormat() << endl << flush; -// cout << " r-r-my_los " << hex << range1->range->my_los << endl << flush; -// cout << dec; -// #endif -// cout << 1000 << endl << flush; - // SpatialRange *res = new SpatialRange(range1); - // Yay! Works: SpatialRange *res = (one.range & other.range); SpatialRange *res = sr_intersect(one.range,other.range,compress); - // cout << 1100 << " 11 nr = " << res->range->range->nranges() << endl << flush; - // srange result; result.set_tag(999); - if(res != NULL) - range.addSpatialRange(*res); - // STARE_SpatialIntervals sis_res = res->toSpatialIntervals(); - // cout << 1150 << endl << flush; - // range.addSpatialIntervals(sis_res); - // cout << 1200 << " 12 nr = " << range.range->range->nranges() << endl << flush; - // res->purge(); - // delete res; - // cout << 1300 << endl << flush; + if(res != NULL){ + range.addSpatialRange(*res); + //res->print(); + range.purge(); + } + } + std::list>* leftJoin(SpatialRange* sp){//srange + return range.leftJoin(sp);//2D numpy array + } + std::list>* innerJoin(SpatialRange* sp){//srange + return range.innerJoin(sp); + } + std::list>* fullJoin(SpatialRange* sp){//srange + return range.fullJoin(sp); + } + void print(){ + range.print(); } }; From 820142cf5eaf05ca1a6f9c0cb908e6b37d4c8e1d Mon Sep 17 00:00:00 2001 From: TonHai1111 Date: Tue, 14 Sep 2021 11:54:15 -0500 Subject: [PATCH 03/12] Added new join interfaces --- PySTARE.cpp | 37 +++++++++++++++++++++++++++++++++++++ PySTARE.h | 28 +++++++++++++++++----------- PySTARE.i | 21 +++++++++++++++++++++ 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/PySTARE.cpp b/PySTARE.cpp index 39e6d4a..d3cbc1c 100644 --- a/PySTARE.cpp +++ b/PySTARE.cpp @@ -175,7 +175,36 @@ StareResult _expand_intervals(int64_t* indices, int len, int resolution, bool mu result.add_indexValues(expandIntervalsMultiRes(si,resolution, multi_resolution)); return result; } +StareResult _convert_Join_Result(std::list>* temp){ + StareResult result; + STARE_ArrayIndexSpatialValues listValues; + STARE_ArrayIndexes listIndexes; + int index = 0; + std::list>::iterator it; + for(it = temp->begin(); it != temp->end(); ++it){ + listIndexes.push_back(index); + std::list::iterator it_j; + for(it_j = it->begin(); it_j != it->end(); ++it_j){ + listValues.push_back(*it_j); + index += 1; + } + } + result.addJoinResults(listValues, listIndexes); + return result; +} +StareResult _leftJoin(srange& one, srange& other){ + std::list>* temp = one.range.leftJoin(&(other.range)); + return _convert_Join_Result(temp); +} +StareResult _innerJoin(srange& one, srange& other){ + std::list>* temp = one.range.innerJoin(&(other.range)); + return _convert_Join_Result(temp); +} +StareResult _fullJoin(srange& one, srange& other){ + std::list>* temp = one.range.fullJoin(&(other.range)); + return _convert_Join_Result(temp); +} StareResult _to_neighbors(int64_t* indices, int len) { STARE_ArrayIndexSpatialValues sivs(indices, indices+len); StareResult result; @@ -724,6 +753,14 @@ void StareResult::copy (int64_t* indices, int len) { ; break; } } +void StareResult::copy_as_list_list(int64_t* indices1, int len1, int64_t* indices2, int len2){ + for(int i = 0; i < min(len1,(int)listValues.size()); ++i) { + indices1[i] = listValues[i]; + } + for(int i = 0; i < min(len2,(int)listIndexes.size()); ++i) { + indices2[i] = listIndexes[i]; + } +} void StareResult::copy_as_values (int64_t* indices, int len) { switch( sCase ) { case SpatialIntervals : diff --git a/PySTARE.h b/PySTARE.h index de520c9..8a067fb 100644 --- a/PySTARE.h +++ b/PySTARE.h @@ -118,7 +118,8 @@ void _set_temporal_resolutions_from_sorted_inplace (int64_t* indices_inplace, in /****/ -enum StareResultCase { SpatialIntervals, ArrayIndexSpatialValues }; +typedef std::vector STARE_ArrayIndexes; +enum StareResultCase { SpatialIntervals, ArrayIndexSpatialValues, ArrayIndexes }; class StareResult { public: @@ -134,9 +135,20 @@ class StareResult { void copy (int64_t* indices, int len); void copy_as_values (int64_t* indices, int len); void copy_as_intervals(int64_t* indices, int len); - //void copy_as_list_list(int64_t* indices1, int len1, int64_t* indices2, int len2); + void copy_as_list_list(int64_t* indices1, int len1, int64_t* indices2, int len2); void convert(); bool converted = false; + //For join + void addJoinResults(STARE_ArrayIndexSpatialValues values, std::vector indexes){ + this->listValues = values; + this->listIndexes = indexes; + this->sCase = ArrayIndexes; + } + int get_listValues_size(){ return listValues.size();} + int get_listIndexes_size(){ return listIndexes.size();} + STARE_ArrayIndexSpatialValues listValues; + STARE_ArrayIndexes listIndexes; + // STARE_SpatialIntervals sis; STARE_ArrayIndexSpatialValues sisvs; StareResultCase sCase; @@ -209,19 +221,13 @@ class srange { range.purge(); } } - std::list>* leftJoin(SpatialRange* sp){//srange - return range.leftJoin(sp);//2D numpy array - } - std::list>* innerJoin(SpatialRange* sp){//srange - return range.innerJoin(sp); - } - std::list>* fullJoin(SpatialRange* sp){//srange - return range.fullJoin(sp); - } void print(){ range.print(); } }; +StareResult _leftJoin(srange& one, srange& other); +StareResult _innerJoin(srange& one, srange& other); +StareResult _fullJoin(srange& one, srange& other); #endif diff --git a/PySTARE.i b/PySTARE.i index df5f9e8..532cbae 100644 --- a/PySTARE.i +++ b/PySTARE.i @@ -606,6 +606,27 @@ def expand_intervals(intervals, resolution, multi_resolution=False): result.copy_as_values(expanded_intervals) return expanded_intervals +def leftJoin(one, other): + result = _leftJoin(one,other) + values = numpy.zeros([result.get_listValues_size()],dtype=numpy.int64) + indexes = numpy.zeros([result.get_listIndexes_size()],dtype=numpy.int64) + result.copy_as_list_list(values, indexes) + return values, indexes + +def innerJoin(one, other): + result = _innerJoin(one,other) + values = numpy.zeros([result.get_listValues_size()],dtype=numpy.int64) + indexes = numpy.zeros([result.get_listIndexes_size()],dtype=numpy.int64) + result.copy_as_list_list(values, indexes) + return values, indexes + +def fullJoin(one, other): + result = _fullJoin(one,other) + values = numpy.zeros([result.get_listValues_size()],dtype=numpy.int64) + indexes = numpy.zeros([result.get_listIndexes_size()],dtype=numpy.int64) + result.copy_as_list_list(values, indexes) + return values, indexes + # result = numpy.full([result_size_limit],-1,dtype=numpy.int64) # result_size = numpy.full([1],-1,dtype=numpy.int64) # _expand_intervals(intervals,resolution,result,result_size) From f1af864257477d33d088875e56b134496584faf4 Mon Sep 17 00:00:00 2001 From: TonHai1111 Date: Wed, 15 Sep 2021 07:59:11 -0500 Subject: [PATCH 04/12] Fixed destructor functions for StareResult and srange --- PySTARE.cpp | 24 ++++++++++++++++++++++-- PySTARE.h | 3 ++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/PySTARE.cpp b/PySTARE.cpp index d3cbc1c..4d2124d 100644 --- a/PySTARE.cpp +++ b/PySTARE.cpp @@ -720,7 +720,20 @@ void _coarsen(int64_t* indices, int len, //////////////////////////////////////////////////////////////////////////////// // -StareResult::~StareResult() {} +StareResult::~StareResult() { + //if(!listValues.empty()){ + listValues.clear(); + //} + //if(!listIndexes.empty()){ + listIndexes.clear(); + //} + //if(!sis.empty()){ + sis.clear(); + //} + //if(!sisvs.empty()){ + sisvs.clear(); + //} +} int StareResult::get_size() { switch( sCase ) { case ArrayIndexSpatialValues : return get_size_as_values(); @@ -812,7 +825,14 @@ srange::srange(int64_t* indices, int len) { range.addSpatialIntervals(sis); } -srange::~srange() {} +srange::~srange() { + //if(!sis.empty()){ + sis.clear(); + //} + //if(!sivs.empty()){ + sivs.clear(); + //} +} void srange::add_intervals(int64_t* indices, int len) { STARE_SpatialIntervals sis(indices, indices+len); diff --git a/PySTARE.h b/PySTARE.h index 8a067fb..f314844 100644 --- a/PySTARE.h +++ b/PySTARE.h @@ -218,7 +218,8 @@ class srange { if(res != NULL){ range.addSpatialRange(*res); //res->print(); - range.purge(); + //range.purge(); + delete res; } } void print(){ From bcc18140462109a594deea4a7cf007e0ab67415f Mon Sep 17 00:00:00 2001 From: TonHai1111 Date: Wed, 15 Sep 2021 08:00:11 -0500 Subject: [PATCH 05/12] Added a test notebook test_HTMSubTree.ipynb --- tests/Test_HTMSubTree.ipynb | 243 ++++++++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 tests/Test_HTMSubTree.ipynb diff --git a/tests/Test_HTMSubTree.ipynb b/tests/Test_HTMSubTree.ipynb new file mode 100644 index 0000000..b7526e5 --- /dev/null +++ b/tests/Test_HTMSubTree.ipynb @@ -0,0 +1,243 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "6ce4f5ce", + "metadata": {}, + "outputs": [], + "source": [ + "import pystare\n", + "import numpy" + ] + }, + { + "cell_type": "markdown", + "id": "44d6c7d7", + "metadata": {}, + "source": [ + "## I. contains and add_intervals\n", + "* Check contains() and add_intervals() functions" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "16812afe", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "False\n", + "True\n" + ] + } + ], + "source": [ + "d = pystare.srange()\n", + "print(d.contains(2278628305110597130)) #OK\n", + "d.add_intervals([2278629908241951402, 2278628982922675882, 2278591990003059051, 2278628305110597130, 2278592394398889355, 2278590972621403051, 2278584699138640267, 2278590886317248235, 2278584833068297163, 2278645771925710699, 2278645935752009067, 2278635600733849834]) #OK\n", + "print(d.contains(2278628982922675882)) #OK\n", + "#d.purge()" + ] + }, + { + "cell_type": "markdown", + "id": "95d2b9ed", + "metadata": {}, + "source": [ + "## II. add_range" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5f82c5df", + "metadata": {}, + "outputs": [], + "source": [ + "c = pystare.srange()\n", + "c.add_intervals([2278584833068297163, 2278645771925710699, 2278645935752009067, 2278635600733849834]) #OK\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "6251b10b", + "metadata": {}, + "outputs": [], + "source": [ + "d.add_range(c)\n", + "#d.purge() " + ] + }, + { + "cell_type": "markdown", + "id": "0151f39c", + "metadata": {}, + "source": [ + "## III. extract_intervals\n", + "* extract_intervals()\n", + "* copy_as_intervals() with StareResults" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "2bf3b46a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2278584604132638731 2278584741571592203 2278590788885544971\n", + " 2278590926324498443 2278591888397172747 2278592300714033163\n", + " 2278627897402982410 2278628447158796298 2278629546670424074\n", + " 2278635593984376842 2278645764466933771 2278645901905887243]\n" + ] + } + ], + "source": [ + "d.extract_intervals()\n", + "sr = pystare.StareResult()\n", + "sr.add_intervals(d.sis)\n", + "temp = numpy.full([sr.get_size_as_intervals()], -1, dtype=numpy.int64)\n", + "sr.copy_as_intervals(temp)\n", + "print(temp)" + ] + }, + { + "cell_type": "markdown", + "id": "5054be6d", + "metadata": {}, + "source": [ + "## IV. Join\n", + "* leftJoin()\n", + "* innerJoin()\n", + "* fullJoin()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3dafda87", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=========LEFT JOIN==========\n", + "Stare IDs: \n", + "[2278584741571592203 2278584741571592203 2278635593984376842\n", + " 2278635593984376842 2278645764466933771 2278645764466933771\n", + " 2278645901905887243 2278645901905887243]\n", + "Offset: \n", + "[0 2 4 6]\n" + ] + } + ], + "source": [ + "a, b = pystare.leftJoin(c,d)\n", + "print(\"=========LEFT JOIN==========\")\n", + "print(\"Stare IDs: \")\n", + "print(a)\n", + "print(\"Offset: \")\n", + "print(b)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "15703246", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=========INNER JOIN==========\n", + "Stare IDs: \n", + "[2278584741571592203 2278584741571592203 2278635593984376842\n", + " 2278635593984376842 2278645764466933771 2278645764466933771\n", + " 2278645901905887243 2278645901905887243]\n", + "Offset: \n", + "[0 2 4 6]\n" + ] + } + ], + "source": [ + "print(\"=========INNER JOIN==========\")\n", + "a1, b1 = pystare.innerJoin(c,d)\n", + "print(\"Stare IDs: \")\n", + "print(a1)\n", + "print(\"Offset: \")\n", + "print(b1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "451add66", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=========FULL JOIN==========\n", + "Stare IDs: \n", + "[2278584604132638731 2278584741571592203 2278584741571592203\n", + " 2278590788885544971 2278590926324498443 2278591888397172747\n", + " 2278592300714033163 2278627897402982410 2278628447158796298\n", + " 2278629546670424074 2278635593984376842 2278635593984376842\n", + " 2278645764466933771 2278645764466933771 2278645901905887243\n", + " 2278645901905887243]\n", + "Offset: \n", + "[ 0 1 3 4 5 6 7 8 9 10 12 14]\n" + ] + } + ], + "source": [ + "print(\"=========FULL JOIN==========\")\n", + "a2, b2 = pystare.fullJoin(c,d)\n", + "print(\"Stare IDs: \")\n", + "print(a2)\n", + "print(\"Offset: \")\n", + "print(b2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57ab4435", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 9947ee50e96f72baac564797e350779884dad586 Mon Sep 17 00:00:00 2001 From: TonHai1111 Date: Fri, 17 Sep 2021 16:40:40 -0500 Subject: [PATCH 06/12] Clean code and verify _to_compressed_range, _intersect, _intersects and _intersect_multiresolution --- PySTARE.cpp | 85 ++++++++++++++++++++++++----------------------------- 1 file changed, 38 insertions(+), 47 deletions(-) diff --git a/PySTARE.cpp b/PySTARE.cpp index 4d2124d..1389685 100644 --- a/PySTARE.cpp +++ b/PySTARE.cpp @@ -320,10 +320,11 @@ void _intersects(int64_t* indices1, int len1, int64_t* indices2, int len2, int* intersects[i] = 0; int istat = r1.intersects(indices2[i]); if( istat != 0 ) { - intersects[i] = 1; + intersects[i] = 1; } } - } else if( method == 1 ) { + } + else if( method == 1 ) { // Binary sort and search sort(indices1,indices1+len1); for(int i=0; i end; - } else if(indices1[m] > test_siv) { - end = m-1; - done = start > end; - } else { - intersects[i] = 1; done = true; - } + m = (start+end)/2; + if(indices1[m] < test_siv) { + start = m+1; + done = start > end; + } + else if(indices1[m] > test_siv) { + end = m-1; + done = start > end; + } + else { + intersects[i] = 1; done = true; + } } if( intersects[i] == 0 ) { - if( (end >= 0) || (start < len1) ) { - if( 0 <= m-1 ) { - if( cmpSpatial(indices1[m-1],test_siv) != 0 ) { - intersects[i] = 1; - } - } - if( (0 <= m) && (m < len1) ) { - if( cmpSpatial(indices1[m],test_siv) != 0 ) { - intersects[i] = 1; - } - } - if( m+1 < len1 ) { - if( cmpSpatial(indices1[m+1],test_siv) != 0 ) { - intersects[i] = 1; - } - } - } + if( (end >= 0) || (start < len1) ) { + if( 0 <= m-1 ) { + if( cmpSpatial(indices1[m-1],test_siv) != 0 ) { + intersects[i] = 1; + } + } + if( (0 <= m) && (m < len1) ) { + if( cmpSpatial(indices1[m],test_siv) != 0 ) { + intersects[i] = 1; + } + } + if( m+1 < len1 ) { + if( cmpSpatial(indices1[m+1],test_siv) != 0 ) { + intersects[i] = 1; + } + } + } } } - } else { + } + else { // Fall-through - for(int i=0; i Date: Tue, 21 Sep 2021 11:37:51 -0500 Subject: [PATCH 07/12] Added a flag to disable tryGroupLeaves functionality --- PySTARE.cpp | 7 +++++++ PySTARE.h | 13 ++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/PySTARE.cpp b/PySTARE.cpp index 1389685..4c15d73 100644 --- a/PySTARE.cpp +++ b/PySTARE.cpp @@ -814,11 +814,18 @@ void StareResult::convert() { // srange::srange() {} +srange::srange(bool _isGroupLeaves) { + range.tree->isGroupLeaves = _isGroupLeaves; +} srange::srange(int64_t* indices, int len) { STARE_ArrayIndexSpatialValues sis(indices, indices+len); range.addSpatialIntervals(sis); } +srange::srange(int64_t* indices, int len, bool isGroupLeaves) { + STARE_ArrayIndexSpatialValues sis(indices, indices+len); + range.addSpatialIntervals(sis, isGroupLeaves); +} srange::~srange() { sis.clear(); diff --git a/PySTARE.h b/PySTARE.h index f314844..ec273f4 100644 --- a/PySTARE.h +++ b/PySTARE.h @@ -180,7 +180,9 @@ void _adapt_resolution_to_proximity(int64_t* indices, int len, int64_t* range_in class srange { public: srange(); + srange(bool isGroupLeaves); srange(int64_t* indices, int len); + srange(int64_t* indices, int len, bool isGroupLeaves); virtual ~srange(); void add_intervals(int64_t* indices, int len); @@ -211,7 +213,7 @@ class srange { SpatialRange range; STARE_SpatialIntervals sis; - STARE_ArrayIndexSpatialValues sivs; + STARE_ArrayIndexSpatialValues sivs; void add_intersect(const srange& one, const srange& other,bool compress) { SpatialRange *res = sr_intersect(one.range,other.range,compress); @@ -222,6 +224,15 @@ class srange { delete res; } } + void add_intersect(const srange& one, const srange& other,bool compress, bool isGroupLeaves) { + SpatialRange *res = sr_intersect(one.range,other.range,compress, isGroupLeaves); + if(res != NULL){ + range.addSpatialRange(*res); + //res->print(); + //range.purge(); + delete res; + } + } void print(){ range.print(); } From 7826100a049492f3baca9ea05b26fe74f69f34f1 Mon Sep 17 00:00:00 2001 From: griessbaum Date: Wed, 6 Oct 2021 17:03:03 -0700 Subject: [PATCH 08/12] merge --- README.md | 4 +-- pystare/PySTARE.cpp | 6 ++-- pystare/PySTARE.h | 6 ++-- pystare/spatial.py | 77 ++++++++++++++++++++++++++++++++++++++++----- pystare/temporal.py | 2 +- setup.py | 33 ++++++++++--------- 6 files changed, 94 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 5f57f06..93a9fb0 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ Build and install STARE e.g. with: git clone https://github.com/SpatioTemporal/STARE cd STARE mkdir build - cd build - cmake ../ + cd build + cmake -DCMAKE_INSTALL_PREFIX=~/stare -DSTARE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=NO .. make sudo make install diff --git a/pystare/PySTARE.cpp b/pystare/PySTARE.cpp index c6db765..79a5697 100644 --- a/pystare/PySTARE.cpp +++ b/pystare/PySTARE.cpp @@ -193,15 +193,15 @@ StareResult _convert_Join_Result(std::list>* temp){ return result; } -StareResult _leftJoin(srange& one, srange& other){ +StareResult _left_join(srange& one, srange& other){ std::list>* temp = one.range.leftJoin(&(other.range)); return _convert_Join_Result(temp); } -StareResult _innerJoin(srange& one, srange& other){ +StareResult _inner_join(srange& one, srange& other){ std::list>* temp = one.range.innerJoin(&(other.range)); return _convert_Join_Result(temp); } -StareResult _fullJoin(srange& one, srange& other){ +StareResult _full_join(srange& one, srange& other){ std::list>* temp = one.range.fullJoin(&(other.range)); return _convert_Join_Result(temp); } diff --git a/pystare/PySTARE.h b/pystare/PySTARE.h index 1f6cdfd..6f4d9fc 100644 --- a/pystare/PySTARE.h +++ b/pystare/PySTARE.h @@ -239,8 +239,8 @@ class srange { } }; -StareResult _leftJoin(srange& one, srange& other); -StareResult _innerJoin(srange& one, srange& other); -StareResult _fullJoin(srange& one, srange& other); +StareResult _left_join(srange& one, srange& other); +StareResult _inner_join(srange& one, srange& other); +StareResult _full_join(srange& one, srange& other); #endif diff --git a/pystare/spatial.py b/pystare/spatial.py index 1bf035e..afd5e25 100644 --- a/pystare/spatial.py +++ b/pystare/spatial.py @@ -493,13 +493,7 @@ def to_circular_cover(lat, lon, radius, level): def circular_cover_from(index, radius, level): """ - - Parameters - ----------- - - Returns - -------- - + TODO """ if level < 0 or level > 27: @@ -909,7 +903,7 @@ def triangulate_indices(indices): """ Prepare data for matplotlib.tri.Triangulate. - Usage + Examples ---------- >>> lons, lats, intmat = triangulate_indices(indices) # doctest: +SKIP >>> triang = tri.Triangulation(lons,lats,intmat) # doctest: +SKIP @@ -919,3 +913,70 @@ def triangulate_indices(indices): latv, lonv, lat_center, lon_center = to_vertices_latlon(indices) lons, lats, intmat = triangulate(latv, lonv) return lons, lats, intmat + + +def left_join(left, right): + """ Performs a left join + + Parameters + ------------ + left: array-like + left set of SIDs to join + right: array-like + right set of SIDs to join + + Examples + ---------- + >>> import numpy + + """ + result = pystare.core._left_join(left, right) + values = numpy.zeros([result.get_listValues_size()], dtype=numpy.int64) + indexes = numpy.zeros([result.get_listIndexes_size()], dtype=numpy.int64) + result.copy_as_list_list(values, indexes) + return values, indexes + + +def inner_join(left, right): + """ Performs an inner join + + Parameters + ------------ + left: array-like + left set of SIDs to join + right: array-like + right set of SIDs to join + + Examples + ---------- + >>> import numpy + + """ + result = pystare.core._inner_join(left, right) + values = numpy.zeros([result.get_listValues_size()], dtype=numpy.int64) + indexes = numpy.zeros([result.get_listIndexes_size()], dtype=numpy.int64) + result.copy_as_list_list(values, indexes) + return values, indexes + + +def full_join(left, right): + """ Performs a full/outer join + + Parameters + ------------ + left: array-like + left set of SIDs to join + right: array-like + right set of SIDs to join + + Examples + ---------- + >>> import numpy + + """ + result = pystare.core._full_join(left, right) + values = numpy.zeros([result.get_listValues_size()], dtype=numpy.int64) + indexes = numpy.zeros([result.get_listIndexes_size()], dtype=numpy.int64) + result.copy_as_list_list(values, indexes) + return values, indexes + diff --git a/pystare/temporal.py b/pystare/temporal.py index 2718815..6a984e3 100644 --- a/pystare/temporal.py +++ b/pystare/temporal.py @@ -130,7 +130,7 @@ def coarsest_resolution_finer_or_equal_ms(ms): |11 | - | - | - | 1 | Before/After Epoch | +----------+-------------+-------+-----+------+----------------------------+ - Arguments + Parameters ---------- ms: 1D array of ints resolution in milliseconds diff --git a/setup.py b/setup.py index 63adf41..6be46a7 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,4 @@ #!/usr/bin/env/python -"""Installation script -""" - import os import numpy from setuptools import setup, Extension @@ -10,19 +7,24 @@ import versioneer -STARE_LIB_DIRS = [os.environ.get('STARE_LIB_DIR', '/usr/local/lib/')] -STARE_INCLUDE_DIRS = [os.environ.get('STARE_INCLUDE_DIR', '/usr/local/include/STARE/')] +# Location of libSTARE.a +STARE_LIB_DIRS = [os.environ.get('STARE_LIB_DIR', + '/usr/local/lib/')] + +# Location of STARE.h +STARE_INCLUDE_DIRS = [os.environ.get('STARE_INCLUDE_DIR', + '/usr/local/include/STARE/')] INCLUDE_DIRS = STARE_INCLUDE_DIRS + [numpy.get_include()] -pystare = Extension(name='pystare._core', - sources=['pystare/PySTARE.i', 'pystare/PySTARE.cpp'], - swig_opts=['-c++'], - extra_compile_args=['-std=c++11'], - libraries=['STARE'], - library_dirs=STARE_LIB_DIRS, # Location of libSTARE.a - include_dirs=INCLUDE_DIRS, # Location of STARE.h - language='c++') +core = Extension(name='pystare._core', + sources=['pystare/PySTARE.i', 'pystare/PySTARE.cpp'], + swig_opts=['-c++'], + extra_compile_args=['-std=c++11'], + libraries=['STARE'], + library_dirs=STARE_LIB_DIRS, + include_dirs=INCLUDE_DIRS, + language='c++') class BuildPy(build_py): @@ -45,8 +47,5 @@ def run(self): setup( version=version, cmdclass=cmdclass, - ext_modules=[pystare], + ext_modules=[core], ) - - - From bc094fb7ef30c87560d2b62ee0c1da055118b140 Mon Sep 17 00:00:00 2001 From: griessbaum Date: Wed, 6 Oct 2021 17:05:31 -0700 Subject: [PATCH 09/12] git address #73 --- pystare/spatial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pystare/spatial.py b/pystare/spatial.py index afd5e25..e725bae 100644 --- a/pystare/spatial.py +++ b/pystare/spatial.py @@ -491,7 +491,7 @@ def to_circular_cover(lat, lon, radius, level): return range_indices -def circular_cover_from(index, radius, level): +def from_circular_cover(index, radius, level): """ TODO """ From 1cded1bb15521a862316dfc8825a6854dfc2f0f6 Mon Sep 17 00:00:00 2001 From: griessbaum Date: Wed, 6 Oct 2021 17:33:53 -0700 Subject: [PATCH 10/12] added join example --- examples/joins.ipynb | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 examples/joins.ipynb diff --git a/examples/joins.ipynb b/examples/joins.ipynb new file mode 100644 index 0000000..f574694 --- /dev/null +++ b/examples/joins.ipynb @@ -0,0 +1,36 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file From 50160b16f355d47cc0366fefd3043a845716ad57 Mon Sep 17 00:00:00 2001 From: griessbaum Date: Mon, 11 Oct 2021 15:21:02 -0700 Subject: [PATCH 11/12] added test stub for segfaults --- tests/test_subtree_segfaults.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tests/test_subtree_segfaults.py diff --git a/tests/test_subtree_segfaults.py b/tests/test_subtree_segfaults.py new file mode 100644 index 0000000..6a39f77 --- /dev/null +++ b/tests/test_subtree_segfaults.py @@ -0,0 +1,33 @@ +import numpy +import pystare + +lat = numpy.array([30, 45, 60], dtype=numpy.double) +lon = numpy.array([45, 60, 10], dtype=numpy.double) +indices1 = pystare.from_latlon(lat, lon, 12) +indices2 = numpy.array([0x100000000000000c], dtype=numpy.int64) + + +def test_issue79a(): + # This segfaults + intersected = pystare.intersection(indices1, indices2, multi_resolution=True) + + +def test_issue79b(): + # This segfaults + intersected = pystare.intersection(indices1, indices2, multi_resolution=False) + + +def test_issue79c(): + # This segfaults + out_length = 2 * max(len(indices1), len(indices2)) + intersection = numpy.full([out_length], -1, dtype=numpy.int64) + pystare.core._intersect_multiresolution(indices1, indices2, intersection) + + +def test_issue79d(): + # This segfaults + out_length = 2 * max(len(indices1), len(indices2)) + intersection = numpy.full([out_length], -1, dtype=numpy.int64) + pystare.core._intersect(indices1, indices2, intersection) + + From 729650f04b829a28c59455ae64fefd08258e6ea9 Mon Sep 17 00:00:00 2001 From: TonHai1111 Date: Wed, 17 Nov 2021 13:15:11 -0600 Subject: [PATCH 12/12] Added interface for merge list --- pystare/PySTARE.cpp | 17 +++++++++++++++++ pystare/PySTARE.h | 3 ++- pystare/spatial.py | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/pystare/PySTARE.cpp b/pystare/PySTARE.cpp index 79a5697..5fd2804 100644 --- a/pystare/PySTARE.cpp +++ b/pystare/PySTARE.cpp @@ -205,6 +205,23 @@ StareResult _full_join(srange& one, srange& other){ std::list>* temp = one.range.fullJoin(&(other.range)); return _convert_Join_Result(temp); } + +StareResult _merge_list(int64_t* indices1, int len1, int64_t* indices2, int len2){ + STARE_ArrayIndexSpatialValues a(indices1, len1 + indices1); + STARE_ArrayIndexSpatialValues b(indices2, len2 + indices2); + STARE_ArrayIndexSpatialValues res; + StareResult result; + int index = 0; + std::list *temp = mergeList(a, b); + std::list::iterator it; + for(it = temp->begin(); it != temp->end(); ++it) + res.push_back(*it); + temp->clear(); + delete temp; + result.add_indexValues(res); + return result; +} + StareResult _to_neighbors(int64_t* indices, int len) { STARE_ArrayIndexSpatialValues sivs(indices, indices+len); StareResult result; diff --git a/pystare/PySTARE.h b/pystare/PySTARE.h index 6f4d9fc..786977b 100644 --- a/pystare/PySTARE.h +++ b/pystare/PySTARE.h @@ -235,12 +235,13 @@ class srange { } } void print(){ - range.print(); + //range.print(); } }; StareResult _left_join(srange& one, srange& other); StareResult _inner_join(srange& one, srange& other); StareResult _full_join(srange& one, srange& other); +StareResult _merge_list(int64_t* indices1, int len1, int64_t* indices2, int len2); #endif diff --git a/pystare/spatial.py b/pystare/spatial.py index e725bae..8253794 100644 --- a/pystare/spatial.py +++ b/pystare/spatial.py @@ -980,3 +980,20 @@ def full_join(left, right): result.copy_as_list_list(values, indexes) return values, indexes +def merge_list(a, b): + """ + + Parameters + ----------- + + Returns + -------- + + Examples + -------- + + """ + result = pystare.core._merge_list(a, b) + range_indices = numpy.full([result.get_size_as_values()], -1, dtype=numpy.int64) + result.copy_as_values(range_indices) + return range_indices \ No newline at end of file