diff --git a/README.md b/README.md index 5f57f06..93a9fb0 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ Build and install STARE e.g. with: git clone https://github.com/SpatioTemporal/STARE cd STARE mkdir build - cd build - cmake ../ + cd build + cmake -DCMAKE_INSTALL_PREFIX=~/stare -DSTARE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=NO .. make sudo make install diff --git a/examples/joins.ipynb b/examples/joins.ipynb new file mode 100644 index 0000000..f574694 --- /dev/null +++ b/examples/joins.ipynb @@ -0,0 +1,36 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/pystare/PySTARE.cpp b/pystare/PySTARE.cpp index f524ecf..5fd2804 100644 --- a/pystare/PySTARE.cpp +++ b/pystare/PySTARE.cpp @@ -175,6 +175,52 @@ StareResult _expand_intervals(int64_t* indices, int len, int resolution, bool mu result.add_indexValues(expandIntervalsMultiRes(si,resolution, multi_resolution)); return result; } +StareResult _convert_Join_Result(std::list>* temp){ + StareResult result; + STARE_ArrayIndexSpatialValues listValues; + STARE_ArrayIndexes listIndexes; + int index = 0; + std::list>::iterator it; + for(it = temp->begin(); it != temp->end(); ++it){ + listIndexes.push_back(index); + std::list::iterator it_j; + for(it_j = it->begin(); it_j != it->end(); ++it_j){ + listValues.push_back(*it_j); + index += 1; + } + } + result.addJoinResults(listValues, listIndexes); + return result; + +} +StareResult _left_join(srange& one, srange& other){ + std::list>* temp = one.range.leftJoin(&(other.range)); + return _convert_Join_Result(temp); +} +StareResult _inner_join(srange& one, srange& other){ + std::list>* temp = one.range.innerJoin(&(other.range)); + return _convert_Join_Result(temp); +} +StareResult _full_join(srange& one, srange& other){ + std::list>* temp = one.range.fullJoin(&(other.range)); + return _convert_Join_Result(temp); +} + +StareResult _merge_list(int64_t* indices1, int len1, int64_t* indices2, int len2){ + STARE_ArrayIndexSpatialValues a(indices1, len1 + indices1); + STARE_ArrayIndexSpatialValues b(indices2, len2 + indices2); + STARE_ArrayIndexSpatialValues res; + StareResult result; + int index = 0; + std::list *temp = mergeList(a, b); + std::list::iterator it; + for(it = temp->begin(); it != temp->end(); ++it) + res.push_back(*it); + temp->clear(); + delete temp; + result.add_indexValues(res); + return result; +} StareResult _to_neighbors(int64_t* indices, int len) { STARE_ArrayIndexSpatialValues sivs(indices, indices+len); @@ -291,10 +337,11 @@ void _intersects(int64_t* indices1, int len1, int64_t* indices2, int len2, int* intersects[i] = 0; int istat = r1.intersects(indices2[i]); if( istat != 0 ) { - intersects[i] = 1; + intersects[i] = 1; } } - } else if( method == 1 ) { + } + else if( method == 1 ) { // Binary sort and search sort(indices1,indices1+len1); for(int i=0; i end; - } else if(indices1[m] > test_siv) { - end = m-1; - done = start > end; - } else { - intersects[i] = 1; done = true; - } + m = (start+end)/2; + if(indices1[m] < test_siv) { + start = m+1; + done = start > end; + } + else if(indices1[m] > test_siv) { + end = m-1; + done = start > end; + } + else { + intersects[i] = 1; done = true; + } } if( intersects[i] == 0 ) { - if( (end >= 0) || (start < len1) ) { - if( 0 <= m-1 ) { - if( cmpSpatial(indices1[m-1],test_siv) != 0 ) { - intersects[i] = 1; - } - } - if( (0 <= m) && (m < len1) ) { - if( cmpSpatial(indices1[m],test_siv) != 0 ) { - intersects[i] = 1; - } - } - if( m+1 < len1 ) { - if( cmpSpatial(indices1[m+1],test_siv) != 0 ) { - intersects[i] = 1; - } - } - } + if( (end >= 0) || (start < len1) ) { + if( 0 <= m-1 ) { + if( cmpSpatial(indices1[m-1],test_siv) != 0 ) { + intersects[i] = 1; + } + } + if( (0 <= m) && (m < len1) ) { + if( cmpSpatial(indices1[m],test_siv) != 0 ) { + intersects[i] = 1; + } + } + if( m+1 < len1 ) { + if( cmpSpatial(indices1[m+1],test_siv) != 0 ) { + intersects[i] = 1; + } + } + } } } - } else { + } + else { // Fall-through - for(int i=0; iisGroupLeaves = _isGroupLeaves; +} srange::srange(int64_t* indices, int len) { STARE_ArrayIndexSpatialValues sis(indices, indices+len); range.addSpatialIntervals(sis); } +srange::srange(int64_t* indices, int len, bool isGroupLeaves) { + STARE_ArrayIndexSpatialValues sis(indices, indices+len); + range.addSpatialIntervals(sis, isGroupLeaves); +} -srange::~srange() {} +srange::~srange() { + sis.clear(); + sivs.clear(); +} void srange::add_intervals(int64_t* indices, int len) { STARE_SpatialIntervals sis(indices, indices+len); diff --git a/pystare/PySTARE.h b/pystare/PySTARE.h index 814424f..786977b 100644 --- a/pystare/PySTARE.h +++ b/pystare/PySTARE.h @@ -119,7 +119,8 @@ void _set_temporal_resolutions_from_sorted_inplace (int64_t* indices_inplace, in /****/ -enum StareResultCase { SpatialIntervals, ArrayIndexSpatialValues }; +typedef std::vector STARE_ArrayIndexes; +enum StareResultCase { SpatialIntervals, ArrayIndexSpatialValues, ArrayIndexes }; class StareResult { public: @@ -135,8 +136,20 @@ class StareResult { void copy (int64_t* indices, int len); void copy_as_values (int64_t* indices, int len); void copy_as_intervals(int64_t* indices, int len); + void copy_as_list_list(int64_t* indices1, int len1, int64_t* indices2, int len2); void convert(); bool converted = false; + //For join + void addJoinResults(STARE_ArrayIndexSpatialValues values, std::vector indexes){ + this->listValues = values; + this->listIndexes = indexes; + this->sCase = ArrayIndexes; + } + int get_listValues_size(){ return listValues.size();} + int get_listIndexes_size(){ return listIndexes.size();} + STARE_ArrayIndexSpatialValues listValues; + STARE_ArrayIndexes listIndexes; + // STARE_SpatialIntervals sis; STARE_ArrayIndexSpatialValues sisvs; StareResultCase sCase; @@ -168,11 +181,15 @@ void _adapt_resolution_to_proximity(int64_t* indices, int len, int64_t* range_in class srange { public: srange(); + srange(bool isGroupLeaves); srange(int64_t* indices, int len); + srange(int64_t* indices, int len, bool isGroupLeaves); virtual ~srange(); void add_intervals(int64_t* indices, int len); - void add_range(const SpatialRange& r) { range.addSpatialRange(r); } + void add_range(const srange& r) { + range.addSpatialRange(r.range); + } // bool contains(int64_t siv); bool contains(long long siv); @@ -200,42 +217,31 @@ class srange { STARE_ArrayIndexSpatialValues sivs; void add_intersect(const srange& one, const srange& other,bool compress) { - // cout << " compress " << compress << endl << flush; - -// HstmRange *range1 = new HstmRange(range.range->range->RangeFromIntersection(other.range.range->range,compress)); // NOTE mlr Probably about the safest way to inst. SpatialRange. -// // #define DIAG -// #ifdef DIAG -// KeyPair kp; -// range1->reset(); -// range1->getNext(kp); -// cout << "sr_i range1,r->r,nr " << range1 << " " << range1->range << " " << range1->range->nranges() << " : " -// << setw(16) << setfill('0') << hex << kp.lo << " " -// << setw(16) << setfill('0') << hex << kp.hi << " " -// << dec -// << endl << flush; -// EmbeddedLevelNameEncoding leftJustified; -// leftJustified.setId(kp.lo); -// cout << "kp.lo lj " << setw(16) << setfill('0') << hex << leftJustified.getSciDBLeftJustifiedFormat() << endl << flush; -// leftJustified.setId(kp.hi); cout << "kp.hi lj " << setw(16) << setfill('0') << hex << leftJustified.getSciDBLeftJustifiedFormat() << endl << flush; -// cout << " r-r-my_los " << hex << range1->range->my_los << endl << flush; -// cout << dec; -// #endif -// cout << 1000 << endl << flush; - // SpatialRange *res = new SpatialRange(range1); - // Yay! Works: SpatialRange *res = (one.range & other.range); SpatialRange *res = sr_intersect(one.range,other.range,compress); - // cout << 1100 << " 11 nr = " << res->range->range->nranges() << endl << flush; - // srange result; result.set_tag(999); - range.addSpatialRange(*res); - // STARE_SpatialIntervals sis_res = res->toSpatialIntervals(); - // cout << 1150 << endl << flush; - // range.addSpatialIntervals(sis_res); - // cout << 1200 << " 12 nr = " << range.range->range->nranges() << endl << flush; - // res->purge(); - // delete res; - // cout << 1300 << endl << flush; + if(res != NULL){ + range.addSpatialRange(*res); + //res->print(); + //range.purge(); + delete res; + } + } + void add_intersect(const srange& one, const srange& other,bool compress, bool isGroupLeaves) { + SpatialRange *res = sr_intersect(one.range,other.range,compress, isGroupLeaves); + if(res != NULL){ + range.addSpatialRange(*res); + //res->print(); + //range.purge(); + delete res; + } + } + void print(){ + //range.print(); } }; +StareResult _left_join(srange& one, srange& other); +StareResult _inner_join(srange& one, srange& other); +StareResult _full_join(srange& one, srange& other); +StareResult _merge_list(int64_t* indices1, int len1, int64_t* indices2, int len2); #endif diff --git a/pystare/PySTARE.i b/pystare/PySTARE.i index ebcb810..622600a 100644 --- a/pystare/PySTARE.i +++ b/pystare/PySTARE.i @@ -563,6 +563,8 @@ namespace std { %} */ + %pythoncode %{%} + %include "PySTARE.h" diff --git a/pystare/spatial.py b/pystare/spatial.py index 1bf035e..8253794 100644 --- a/pystare/spatial.py +++ b/pystare/spatial.py @@ -491,15 +491,9 @@ def to_circular_cover(lat, lon, radius, level): return range_indices -def circular_cover_from(index, radius, level): +def from_circular_cover(index, radius, level): """ - - Parameters - ----------- - - Returns - -------- - + TODO """ if level < 0 or level > 27: @@ -909,7 +903,7 @@ def triangulate_indices(indices): """ Prepare data for matplotlib.tri.Triangulate. - Usage + Examples ---------- >>> lons, lats, intmat = triangulate_indices(indices) # doctest: +SKIP >>> triang = tri.Triangulation(lons,lats,intmat) # doctest: +SKIP @@ -919,3 +913,87 @@ def triangulate_indices(indices): latv, lonv, lat_center, lon_center = to_vertices_latlon(indices) lons, lats, intmat = triangulate(latv, lonv) return lons, lats, intmat + + +def left_join(left, right): + """ Performs a left join + + Parameters + ------------ + left: array-like + left set of SIDs to join + right: array-like + right set of SIDs to join + + Examples + ---------- + >>> import numpy + + """ + result = pystare.core._left_join(left, right) + values = numpy.zeros([result.get_listValues_size()], dtype=numpy.int64) + indexes = numpy.zeros([result.get_listIndexes_size()], dtype=numpy.int64) + result.copy_as_list_list(values, indexes) + return values, indexes + + +def inner_join(left, right): + """ Performs an inner join + + Parameters + ------------ + left: array-like + left set of SIDs to join + right: array-like + right set of SIDs to join + + Examples + ---------- + >>> import numpy + + """ + result = pystare.core._inner_join(left, right) + values = numpy.zeros([result.get_listValues_size()], dtype=numpy.int64) + indexes = numpy.zeros([result.get_listIndexes_size()], dtype=numpy.int64) + result.copy_as_list_list(values, indexes) + return values, indexes + + +def full_join(left, right): + """ Performs a full/outer join + + Parameters + ------------ + left: array-like + left set of SIDs to join + right: array-like + right set of SIDs to join + + Examples + ---------- + >>> import numpy + + """ + result = pystare.core._full_join(left, right) + values = numpy.zeros([result.get_listValues_size()], dtype=numpy.int64) + indexes = numpy.zeros([result.get_listIndexes_size()], dtype=numpy.int64) + result.copy_as_list_list(values, indexes) + return values, indexes + +def merge_list(a, b): + """ + + Parameters + ----------- + + Returns + -------- + + Examples + -------- + + """ + result = pystare.core._merge_list(a, b) + range_indices = numpy.full([result.get_size_as_values()], -1, dtype=numpy.int64) + result.copy_as_values(range_indices) + return range_indices \ No newline at end of file diff --git a/pystare/temporal.py b/pystare/temporal.py index 2d09ae3..b6bd983 100644 --- a/pystare/temporal.py +++ b/pystare/temporal.py @@ -130,7 +130,7 @@ def coarsest_resolution_finer_or_equal_ms(ms): |11 | - | - | - | 1 | Before/After Epoch | +----------+-------------+-------+-----+------+----------------------------+ - Arguments + Parameters ---------- ms: 1D array of ints resolution in milliseconds diff --git a/setup.py b/setup.py index 2523d1c..c247241 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,3 @@ -#!/usr/bin/env/python - import os import numpy from setuptools import setup, Extension @@ -8,19 +6,24 @@ import versioneer -STARE_LIB_DIRS = [os.environ.get('STARE_LIB_DIR', '/usr/local/lib/')] -STARE_INCLUDE_DIRS = [os.environ.get('STARE_INCLUDE_DIR', '/usr/local/include/STARE/')] +# Location of libSTARE.a +STARE_LIB_DIRS = [os.environ.get('STARE_LIB_DIR', + '/usr/local/lib/')] + +# Location of STARE.h +STARE_INCLUDE_DIRS = [os.environ.get('STARE_INCLUDE_DIR', + '/usr/local/include/STARE/')] INCLUDE_DIRS = STARE_INCLUDE_DIRS + [numpy.get_include()] -pystare = Extension(name='pystare._core', - sources=['pystare/PySTARE.i', 'pystare/PySTARE.cpp'], - swig_opts=['-c++'], - extra_compile_args=['-std=c++11'], - libraries=['STARE'], - library_dirs=STARE_LIB_DIRS, # Location of libSTARE.a - include_dirs=INCLUDE_DIRS, # Location of STARE.h - language='c++') +core = Extension(name='pystare._core', + sources=['pystare/PySTARE.i', 'pystare/PySTARE.cpp'], + swig_opts=['-c++'], + extra_compile_args=['-std=c++11'], + libraries=['STARE'], + library_dirs=STARE_LIB_DIRS, + include_dirs=INCLUDE_DIRS, + language='c++') class BuildPy(build_py): @@ -43,8 +46,5 @@ def run(self): setup( version=version, cmdclass=cmdclass, - ext_modules=[pystare], + ext_modules=[core], ) - - - diff --git a/tests/Test_HTMSubTree.ipynb b/tests/Test_HTMSubTree.ipynb new file mode 100644 index 0000000..b7526e5 --- /dev/null +++ b/tests/Test_HTMSubTree.ipynb @@ -0,0 +1,243 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "6ce4f5ce", + "metadata": {}, + "outputs": [], + "source": [ + "import pystare\n", + "import numpy" + ] + }, + { + "cell_type": "markdown", + "id": "44d6c7d7", + "metadata": {}, + "source": [ + "## I. contains and add_intervals\n", + "* Check contains() and add_intervals() functions" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "16812afe", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "False\n", + "True\n" + ] + } + ], + "source": [ + "d = pystare.srange()\n", + "print(d.contains(2278628305110597130)) #OK\n", + "d.add_intervals([2278629908241951402, 2278628982922675882, 2278591990003059051, 2278628305110597130, 2278592394398889355, 2278590972621403051, 2278584699138640267, 2278590886317248235, 2278584833068297163, 2278645771925710699, 2278645935752009067, 2278635600733849834]) #OK\n", + "print(d.contains(2278628982922675882)) #OK\n", + "#d.purge()" + ] + }, + { + "cell_type": "markdown", + "id": "95d2b9ed", + "metadata": {}, + "source": [ + "## II. add_range" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5f82c5df", + "metadata": {}, + "outputs": [], + "source": [ + "c = pystare.srange()\n", + "c.add_intervals([2278584833068297163, 2278645771925710699, 2278645935752009067, 2278635600733849834]) #OK\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "6251b10b", + "metadata": {}, + "outputs": [], + "source": [ + "d.add_range(c)\n", + "#d.purge() " + ] + }, + { + "cell_type": "markdown", + "id": "0151f39c", + "metadata": {}, + "source": [ + "## III. extract_intervals\n", + "* extract_intervals()\n", + "* copy_as_intervals() with StareResults" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "2bf3b46a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2278584604132638731 2278584741571592203 2278590788885544971\n", + " 2278590926324498443 2278591888397172747 2278592300714033163\n", + " 2278627897402982410 2278628447158796298 2278629546670424074\n", + " 2278635593984376842 2278645764466933771 2278645901905887243]\n" + ] + } + ], + "source": [ + "d.extract_intervals()\n", + "sr = pystare.StareResult()\n", + "sr.add_intervals(d.sis)\n", + "temp = numpy.full([sr.get_size_as_intervals()], -1, dtype=numpy.int64)\n", + "sr.copy_as_intervals(temp)\n", + "print(temp)" + ] + }, + { + "cell_type": "markdown", + "id": "5054be6d", + "metadata": {}, + "source": [ + "## IV. Join\n", + "* leftJoin()\n", + "* innerJoin()\n", + "* fullJoin()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3dafda87", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=========LEFT JOIN==========\n", + "Stare IDs: \n", + "[2278584741571592203 2278584741571592203 2278635593984376842\n", + " 2278635593984376842 2278645764466933771 2278645764466933771\n", + " 2278645901905887243 2278645901905887243]\n", + "Offset: \n", + "[0 2 4 6]\n" + ] + } + ], + "source": [ + "a, b = pystare.leftJoin(c,d)\n", + "print(\"=========LEFT JOIN==========\")\n", + "print(\"Stare IDs: \")\n", + "print(a)\n", + "print(\"Offset: \")\n", + "print(b)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "15703246", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=========INNER JOIN==========\n", + "Stare IDs: \n", + "[2278584741571592203 2278584741571592203 2278635593984376842\n", + " 2278635593984376842 2278645764466933771 2278645764466933771\n", + " 2278645901905887243 2278645901905887243]\n", + "Offset: \n", + "[0 2 4 6]\n" + ] + } + ], + "source": [ + "print(\"=========INNER JOIN==========\")\n", + "a1, b1 = pystare.innerJoin(c,d)\n", + "print(\"Stare IDs: \")\n", + "print(a1)\n", + "print(\"Offset: \")\n", + "print(b1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "451add66", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=========FULL JOIN==========\n", + "Stare IDs: \n", + "[2278584604132638731 2278584741571592203 2278584741571592203\n", + " 2278590788885544971 2278590926324498443 2278591888397172747\n", + " 2278592300714033163 2278627897402982410 2278628447158796298\n", + " 2278629546670424074 2278635593984376842 2278635593984376842\n", + " 2278645764466933771 2278645764466933771 2278645901905887243\n", + " 2278645901905887243]\n", + "Offset: \n", + "[ 0 1 3 4 5 6 7 8 9 10 12 14]\n" + ] + } + ], + "source": [ + "print(\"=========FULL JOIN==========\")\n", + "a2, b2 = pystare.fullJoin(c,d)\n", + "print(\"Stare IDs: \")\n", + "print(a2)\n", + "print(\"Offset: \")\n", + "print(b2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57ab4435", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/test_subtree_segfaults.py b/tests/test_subtree_segfaults.py new file mode 100644 index 0000000..6a39f77 --- /dev/null +++ b/tests/test_subtree_segfaults.py @@ -0,0 +1,33 @@ +import numpy +import pystare + +lat = numpy.array([30, 45, 60], dtype=numpy.double) +lon = numpy.array([45, 60, 10], dtype=numpy.double) +indices1 = pystare.from_latlon(lat, lon, 12) +indices2 = numpy.array([0x100000000000000c], dtype=numpy.int64) + + +def test_issue79a(): + # This segfaults + intersected = pystare.intersection(indices1, indices2, multi_resolution=True) + + +def test_issue79b(): + # This segfaults + intersected = pystare.intersection(indices1, indices2, multi_resolution=False) + + +def test_issue79c(): + # This segfaults + out_length = 2 * max(len(indices1), len(indices2)) + intersection = numpy.full([out_length], -1, dtype=numpy.int64) + pystare.core._intersect_multiresolution(indices1, indices2, intersection) + + +def test_issue79d(): + # This segfaults + out_length = 2 * max(len(indices1), len(indices2)) + intersection = numpy.full([out_length], -1, dtype=numpy.int64) + pystare.core._intersect(indices1, indices2, intersection) + +