From 06bd16d31f480faf809d77fe1a9b867aa80129fd Mon Sep 17 00:00:00 2001 From: Matthew Perry Date: Mon, 18 Nov 2013 14:33:59 -0800 Subject: [PATCH 001/344] palette256 if None --- TileStache/Core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TileStache/Core.py b/TileStache/Core.py index f9be3ccf..c89eee7d 100644 --- a/TileStache/Core.py +++ b/TileStache/Core.py @@ -655,6 +655,8 @@ def setSaveOptionsPNG(self, optimize=None, palette=None, palette256=None): if palette256 is not None: self.palette256 = bool(palette256) + else: + self.palette256 = None class KnownUnknown(Exception): """ There are known unknowns. That is to say, there are things that we now know we don't know. From 51e5eef7671e6138ea4922725c389f34d5dabfd0 Mon Sep 17 00:00:00 2001 From: Brett Camper Date: Tue, 28 Jan 2014 10:00:26 -0500 Subject: [PATCH 002/344] add support for OpenScienceMap formatted binary tiles to VecTiles provider OSciMap-formatting code mostly copied from: https://github.com/opensciencemap/TileStache/tree/master/TileStache/OSciMap4 todo: cleanup, add MultiResponse ('all') support --- .../Goodies/VecTiles/OSciMap4/GeomEncoder.py | 353 ++++++++++++++++++ .../VecTiles/OSciMap4/StaticKeys/__init__.py | 82 ++++ .../VecTiles/OSciMap4/StaticVals/__init__.py | 260 +++++++++++++ .../VecTiles/OSciMap4/TagRewrite/__init__.py | 109 ++++++ .../VecTiles/OSciMap4/TileData_v4.proto | 92 +++++ .../VecTiles/OSciMap4/TileData_v4_pb2.py | 203 ++++++++++ .../Goodies/VecTiles/OSciMap4/__init__.py | 0 TileStache/Goodies/VecTiles/oscimap.py | 190 ++++++++++ TileStache/Goodies/VecTiles/server.py | 41 +- 9 files changed, 1317 insertions(+), 13 deletions(-) create mode 100644 TileStache/Goodies/VecTiles/OSciMap4/GeomEncoder.py create mode 100644 TileStache/Goodies/VecTiles/OSciMap4/StaticKeys/__init__.py create mode 100644 TileStache/Goodies/VecTiles/OSciMap4/StaticVals/__init__.py create mode 100644 TileStache/Goodies/VecTiles/OSciMap4/TagRewrite/__init__.py create mode 100644 TileStache/Goodies/VecTiles/OSciMap4/TileData_v4.proto create mode 100644 TileStache/Goodies/VecTiles/OSciMap4/TileData_v4_pb2.py create mode 100644 TileStache/Goodies/VecTiles/OSciMap4/__init__.py create mode 100644 TileStache/Goodies/VecTiles/oscimap.py diff --git a/TileStache/Goodies/VecTiles/OSciMap4/GeomEncoder.py b/TileStache/Goodies/VecTiles/OSciMap4/GeomEncoder.py new file mode 100644 index 00000000..d3b05929 --- /dev/null +++ b/TileStache/Goodies/VecTiles/OSciMap4/GeomEncoder.py @@ -0,0 +1,353 @@ + +################################################################################ +# Copyright (c) QinetiQ Plc 2003 +# +# Licensed under the LGPL. For full license details see the LICENSE file. +################################################################################ + +""" +A parser for the Well Text Binary format of OpenGIS types. +""" +# +# 2.5d spec: http://gdal.velocet.ca/projects/opengis/twohalfdsf.html +# + +import sys, traceback, struct + + +# based on xdrlib.Unpacker +class _ExtendedUnPacker: + """ + A simple binary struct parser, only implements the types that are need for the WKB format. + """ + + def __init__(self,data): + self.reset(data) + self.setEndianness('XDR') + + def reset(self, data): + self.__buf = data + self.__pos = 0 + + def get_position(self): + return self.__pos + + def set_position(self, position): + self.__pos = position + + def get_buffer(self): + return self.__buf + + def done(self): + if self.__pos < len(self.__buf): + raise ExceptionWKBParser('unextracted data remains') + + def setEndianness(self,endianness): + if endianness == 'XDR': + self._endflag = '>' + elif endianness == 'NDR': + self._endflag = '<' + else: + raise ExceptionWKBParser('Attempt to set unknown endianness in ExtendedUnPacker') + + def unpack_byte(self): + i = self.__pos + self.__pos = j = i+1 + data = self.__buf[i:j] + if len(data) < 1: + raise EOFError + byte = struct.unpack('%sB' % self._endflag, data)[0] + return byte + + def unpack_uint32(self): + i = self.__pos + self.__pos = j = i+4 + data = self.__buf[i:j] + if len(data) < 4: + raise EOFError + uint32 = struct.unpack('%si' % self._endflag, data)[0] + return uint32 + + def unpack_short(self): + i = self.__pos + self.__pos = j = i+2 + data = self.__buf[i:j] + if len(data) < 2: + raise EOFError + short = struct.unpack('%sH' % self._endflag, data)[0] + return short + + def unpack_double(self): + i = self.__pos + self.__pos = j = i+8 + data = self.__buf[i:j] + if len(data) < 8: + raise EOFError + return struct.unpack('%sd' % self._endflag, data)[0] + +class ExceptionWKBParser(Exception): + '''This is the WKB Parser Exception class.''' + def __init__(self, value): + self.value = value + def __str__(self): + return `self.value` + +class GeomEncoder: + + _count = 0 + + def __init__(self, tileSize): + """ + Initialise a new WKBParser. + + """ + + self._typemap = {1: self.parsePoint, + 2: self.parseLineString, + 3: self.parsePolygon, + 4: self.parseMultiPoint, + 5: self.parseMultiLineString, + 6: self.parseMultiPolygon, + 7: self.parseGeometryCollection} + self.coordinates = [] + self.index = [] + self.position = 0 + self.lastX = 0 + self.lastY = 0 + self.dropped = 0 + self.num_points = 0 + self.isPoint = True + self.tileSize = tileSize - 1 + self.first = True + + def parseGeometry(self, geometry): + + + """ + A factory method for creating objects of the correct OpenGIS type. + """ + + self.coordinates = [] + self.index = [] + self.position = 0 + self.lastX = 0 + self.lastY = 0 + self.isPoly = False + self.isPoint = True; + self.dropped = 0; + self.first = True + # Used for exception strings + self._current_string = geometry + + reader = _ExtendedUnPacker(geometry) + + # Start the parsing + self._dispatchNextType(reader) + + + def _dispatchNextType(self,reader): + """ + Read a type id from the binary stream (reader) and call the correct method to parse it. + """ + + # Need to check endianess here! + endianness = reader.unpack_byte() + if endianness == 0: + reader.setEndianness('XDR') + elif endianness == 1: + reader.setEndianness('NDR') + else: + raise ExceptionWKBParser("Invalid endianness in WKB format.\n"\ + "The parser can only cope with XDR/big endian WKB format.\n"\ + "To force the WKB format to be in XDR use AsBinary(,'XDR'") + + + geotype = reader.unpack_uint32() + + mask = geotype & 0x80000000 # This is used to mask of the dimension flag. + + srid = geotype & 0x20000000 + # ignore srid ... + if srid != 0: + reader.unpack_uint32() + + dimensions = 2 + if mask == 0: + dimensions = 2 + else: + dimensions = 3 + + geotype = geotype & 0x1FFFFFFF + # Despatch to a method on the type id. + if self._typemap.has_key(geotype): + self._typemap[geotype](reader, dimensions) + else: + raise ExceptionWKBParser('Error type to dispatch with geotype = %s \n'\ + 'Invalid geometry in WKB string: %s' % (str(geotype), + str(self._current_string),)) + + def parseGeometryCollection(self, reader, dimension): + try: + num_geoms = reader.unpack_uint32() + + for _ in xrange(0,num_geoms): + self._dispatchNextType(reader) + + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + raise ExceptionWKBParser("Caught unhandled exception parsing GeometryCollection: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) + + + def parseLineString(self, reader, dimensions): + self.isPoint = False; + try: + num_points = reader.unpack_uint32() + + self.num_points = 0; + + for _ in xrange(0,num_points): + self.parsePoint(reader,dimensions) + + self.index.append(self.num_points) + #self.lastX = 0 + #self.lastY = 0 + self.first = True + + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + print error + raise ExceptionWKBParser("Caught unhandled exception parsing Linestring: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) + + + def parseMultiLineString(self, reader, dimensions): + try: + num_linestrings = reader.unpack_uint32() + + for _ in xrange(0,num_linestrings): + self._dispatchNextType(reader) + + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + raise ExceptionWKBParser("Caught unhandled exception parsing MultiLineString: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) + + + def parseMultiPoint(self, reader, dimensions): + try: + num_points = reader.unpack_uint32() + + for _ in xrange(0,num_points): + self._dispatchNextType(reader) + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + raise ExceptionWKBParser("Caught unhandled exception parsing MultiPoint: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) + + + def parseMultiPolygon(self, reader, dimensions): + try: + num_polygons = reader.unpack_uint32() + for n in xrange(0,num_polygons): + if n > 0: + self.index.append(0); + + self._dispatchNextType(reader) + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + raise ExceptionWKBParser("Caught unhandled exception parsing MultiPolygon: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) + + + def parsePoint(self, reader, dimensions): + x = reader.unpack_double() + y = reader.unpack_double() + + if dimensions == 3: + reader.unpack_double() + + xx = int(round(x)) + # flip upside down + yy = self.tileSize - int(round(y)) + + if self.first or xx - self.lastX != 0 or yy - self.lastY != 0: + self.coordinates.append(xx - self.lastX) + self.coordinates.append(yy - self.lastY) + self.num_points += 1 + else: + self.dropped += 1; + + self.first = False + self.lastX = xx + self.lastY = yy + + + def parsePolygon(self, reader, dimensions): + self.isPoint = False; + try: + num_rings = reader.unpack_uint32() + + for _ in xrange(0,num_rings): + self.parseLinearRing(reader,dimensions) + + self.isPoly = True + + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + raise ExceptionWKBParser("Caught unhandled exception parsing Polygon: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) + + def parseLinearRing(self, reader, dimensions): + self.isPoint = False; + try: + num_points = reader.unpack_uint32() + + self.num_points = 0; + + # skip the last point + for _ in xrange(0,num_points-1): + self.parsePoint(reader,dimensions) + + # skip the last point + reader.unpack_double() + reader.unpack_double() + if dimensions == 3: + reader.unpack_double() + + self.index.append(self.num_points) + + self.first = True + + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + raise ExceptionWKBParser("Caught unhandled exception parsing LinearRing: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) \ No newline at end of file diff --git a/TileStache/Goodies/VecTiles/OSciMap4/StaticKeys/__init__.py b/TileStache/Goodies/VecTiles/OSciMap4/StaticKeys/__init__.py new file mode 100644 index 00000000..9a594591 --- /dev/null +++ b/TileStache/Goodies/VecTiles/OSciMap4/StaticKeys/__init__.py @@ -0,0 +1,82 @@ +''' +Created on Sep 13, 2012 + +@author: jeff +''' + +strings = [ +'access', +'addr:housename', +'addr:housenumber', +'addr:interpolation', +'admin_level', +'aerialway', +'aeroway', +'amenity', +'area', +'barrier', +'bicycle', +'brand', +'bridge', +'boundary', +'building', +'construction', +'covered', +'culvert', +'cutting', +'denomination', +'disused', +'embankment', +'foot', +'generator:source', +'harbour', +'highway', +'historic', +'horse', +'intermittent', +'junction', +'landuse', +'layer', +'leisure', +'lock', +'man_made', +'military', +'motorcar', +'name', +'natural', +'oneway', +'operator', +'population', +'power', +'power_source', +'place', +'railway', +'ref', +'religion', +'route', +'service', +'shop', +'sport', +'surface', +'toll', +'tourism', +'tower:type', +'tracktype', +'tunnel', +'water', +'waterway', +'wetland', +'width', +'wood', + +'height', +'min_height', +'roof:shape', +'roof:height', +'rank'] + +keys = dict(zip(strings,range(0, len(strings)-1))) + +def getKeys(): + return keys + \ No newline at end of file diff --git a/TileStache/Goodies/VecTiles/OSciMap4/StaticVals/__init__.py b/TileStache/Goodies/VecTiles/OSciMap4/StaticVals/__init__.py new file mode 100644 index 00000000..0c0689f2 --- /dev/null +++ b/TileStache/Goodies/VecTiles/OSciMap4/StaticVals/__init__.py @@ -0,0 +1,260 @@ +vals = { +"yes" : 0, +"residential" : 1, +"service" : 2, +"unclassified" : 3, +"stream" : 4, +"track" : 5, +"water" : 6, +"footway" : 7, +"tertiary" : 8, +"private" : 9, +"tree" : 10, +"path" : 11, +"forest" : 12, +"secondary" : 13, +"house" : 14, +"no" : 15, +"asphalt" : 16, +"wood" : 17, +"grass" : 18, +"paved" : 19, +"primary" : 20, +"unpaved" : 21, +"bus_stop" : 22, +"parking" : 23, +"parking_aisle" : 24, +"rail" : 25, +"driveway" : 26, +"8" : 27, +"administrative" : 28, +"locality" : 29, +"turning_circle" : 30, +"crossing" : 31, +"village" : 32, +"fence" : 33, +"grade2" : 34, +"coastline" : 35, +"grade3" : 36, +"farmland" : 37, +"hamlet" : 38, +"hut" : 39, +"meadow" : 40, +"wetland" : 41, +"cycleway" : 42, +"river" : 43, +"school" : 44, +"trunk" : 45, +"gravel" : 46, +"place_of_worship" : 47, +"farm" : 48, +"grade1" : 49, +"traffic_signals" : 50, +"wall" : 51, +"garage" : 52, +"gate" : 53, +"motorway" : 54, +"living_street" : 55, +"pitch" : 56, +"grade4" : 57, +"industrial" : 58, +"road" : 59, +"ground" : 60, +"scrub" : 61, +"motorway_link" : 62, +"steps" : 63, +"ditch" : 64, +"swimming_pool" : 65, +"grade5" : 66, +"park" : 67, +"apartments" : 68, +"restaurant" : 69, +"designated" : 70, +"bench" : 71, +"survey_point" : 72, +"pedestrian" : 73, +"hedge" : 74, +"reservoir" : 75, +"riverbank" : 76, +"alley" : 77, +"farmyard" : 78, +"peak" : 79, +"level_crossing" : 80, +"roof" : 81, +"dirt" : 82, +"drain" : 83, +"garages" : 84, +"entrance" : 85, +"street_lamp" : 86, +"deciduous" : 87, +"fuel" : 88, +"trunk_link" : 89, +"information" : 90, +"playground" : 91, +"supermarket" : 92, +"primary_link" : 93, +"concrete" : 94, +"mixed" : 95, +"permissive" : 96, +"orchard" : 97, +"grave_yard" : 98, +"canal" : 99, +"garden" : 100, +"spur" : 101, +"paving_stones" : 102, +"rock" : 103, +"bollard" : 104, +"convenience" : 105, +"cemetery" : 106, +"post_box" : 107, +"commercial" : 108, +"pier" : 109, +"bank" : 110, +"hotel" : 111, +"cliff" : 112, +"retail" : 113, +"construction" : 114, +"-1" : 115, +"fast_food" : 116, +"coniferous" : 117, +"cafe" : 118, +"6" : 119, +"kindergarten" : 120, +"tower" : 121, +"hospital" : 122, +"yard" : 123, +"sand" : 124, +"public_building" : 125, +"cobblestone" : 126, +"destination" : 127, +"island" : 128, +"abandoned" : 129, +"vineyard" : 130, +"recycling" : 131, +"agricultural" : 132, +"isolated_dwelling" : 133, +"pharmacy" : 134, +"post_office" : 135, +"motorway_junction" : 136, +"pub" : 137, +"allotments" : 138, +"dam" : 139, +"secondary_link" : 140, +"lift_gate" : 141, +"siding" : 142, +"stop" : 143, +"main" : 144, +"farm_auxiliary" : 145, +"quarry" : 146, +"10" : 147, +"station" : 148, +"platform" : 149, +"taxiway" : 150, +"limited" : 151, +"sports_centre" : 152, +"cutline" : 153, +"detached" : 154, +"storage_tank" : 155, +"basin" : 156, +"bicycle_parking" : 157, +"telephone" : 158, +"terrace" : 159, +"town" : 160, +"suburb" : 161, +"bus" : 162, +"compacted" : 163, +"toilets" : 164, +"heath" : 165, +"works" : 166, +"tram" : 167, +"beach" : 168, +"culvert" : 169, +"fire_station" : 170, +"recreation_ground" : 171, +"bakery" : 172, +"police" : 173, +"atm" : 174, +"clothes" : 175, +"tertiary_link" : 176, +"waste_basket" : 177, +"attraction" : 178, +"viewpoint" : 179, +"bicycle" : 180, +"church" : 181, +"shelter" : 182, +"drinking_water" : 183, +"marsh" : 184, +"picnic_site" : 185, +"hairdresser" : 186, +"bridleway" : 187, +"retaining_wall" : 188, +"buffer_stop" : 189, +"nature_reserve" : 190, +"village_green" : 191, +"university" : 192, +"1" : 193, +"bar" : 194, +"townhall" : 195, +"mini_roundabout" : 196, +"camp_site" : 197, +"aerodrome" : 198, +"stile" : 199, +"9" : 200, +"car_repair" : 201, +"parking_space" : 202, +"library" : 203, +"pipeline" : 204, +"true" : 205, +"cycle_barrier" : 206, +"4" : 207, +"museum" : 208, +"spring" : 209, +"hunting_stand" : 210, +"disused" : 211, +"car" : 212, +"tram_stop" : 213, +"land" : 214, +"fountain" : 215, +"hiking" : 216, +"manufacture" : 217, +"vending_machine" : 218, +"kiosk" : 219, +"swamp" : 220, +"unknown" : 221, +"7" : 222, +"islet" : 223, +"shed" : 224, +"switch" : 225, +"rapids" : 226, +"office" : 227, +"bay" : 228, +"proposed" : 229, +"common" : 230, +"weir" : 231, +"grassland" : 232, +"customers" : 233, +"social_facility" : 234, +"hangar" : 235, +"doctors" : 236, +"stadium" : 237, +"give_way" : 238, +"greenhouse" : 239, +"guest_house" : 240, +"viaduct" : 241, +"doityourself" : 242, +"runway" : 243, +"bus_station" : 244, +"water_tower" : 245, +"golf_course" : 246, +"conservation" : 247, +"block" : 248, +"college" : 249, +"wastewater_plant" : 250, +"subway" : 251, +"halt" : 252, +"forestry" : 253, +"florist" : 254, +"butcher" : 255} + +def getValues(): + return vals diff --git a/TileStache/Goodies/VecTiles/OSciMap4/TagRewrite/__init__.py b/TileStache/Goodies/VecTiles/OSciMap4/TagRewrite/__init__.py new file mode 100644 index 00000000..3ce39679 --- /dev/null +++ b/TileStache/Goodies/VecTiles/OSciMap4/TagRewrite/__init__.py @@ -0,0 +1,109 @@ +import logging + +# TODO test the lua osm2pgsql for preprocessing ! +# +# fix tags from looking things up in wiki where a value should be used with a specific key, +# i.e. one combination has a wiki page and more use in taginfo and the other does not +# TODO add: +# natural=>meadow +# landuse=>greenhouse,public,scrub +# aeroway=>aerobridge +# leisure=>natural_reserve + +def fixTag(tag, zoomlevel): + drop = False + + if tag[1] is None: + drop = True + + key = tag[0].lower(); + + if key == 'highway': + # FIXME remove ; separated part of tags + return (key, tag[1].lower().split(';')[0]) + + # fixed in osm + #if key == 'leisure': + # value = tag[1].lower(); + # if value in ('village_green', 'recreation_ground'): + # return ('landuse', value) + # else: + # return (key, value) + + elif key == 'natural': + value = tag[1].lower(); + #if zoomlevel <= 9 and not value in ('water', 'wood'): + # return None + + if value in ('village_green', 'meadow'): + return ('landuse', value) + if value == 'mountain_range': + drop = True + else: + return (key, value) + + elif key == 'landuse': + value = tag[1].lower(); + #if zoomlevel <= 9 and not value in ('forest', 'military'): + # return None + + # strange for natural_reserve: more common this way round... + if value in ('park', 'natural_reserve'): + return ('leisure', value) + elif value == 'field': + # wiki: Although this landuse is rendered by Mapnik, it is not an officially + # recognised OpenStreetMap tag. Please use landuse=farmland instead. + return (key, 'farmland') + elif value in ('grassland', 'scrub'): + return ('natural', value) + else: + return (key, value) + + elif key == 'oneway': + value = tag[1].lower(); + if value in ('yes', '1', 'true'): + return (key, 'yes') + else: + drop = True + + elif key == 'area': + value = tag[1].lower(); + if value in ('yes', '1', 'true'): + return (key, 'yes') + # might be used to indicate that a closed way is not an area + elif value in ('no'): + return (key, 'no') + else: + drop = True + + elif key == 'bridge': + value = tag[1].lower(); + if value in ('yes', '1', 'true'): + return (key, 'yes') + elif value in ('no', '-1', '0', 'false'): + drop = True + else: + return (key, value) + + elif key == 'tunnel': + value = tag[1].lower(); + if value in ('yes', '1', 'true'): + return (key, 'yes') + elif value in ('no', '-1', '0', 'false'): + drop = True + else: + return (key, value) + + elif key == 'water': + value = tag[1].lower(); + if value in ('lake;pond'): + return (key, 'pond') + else: + return (key, value) + + if drop: + logging.debug('drop tag: %s %s' % (tag[0], tag[1])) + return None + + return tag + diff --git a/TileStache/Goodies/VecTiles/OSciMap4/TileData_v4.proto b/TileStache/Goodies/VecTiles/OSciMap4/TileData_v4.proto new file mode 100644 index 00000000..183954d7 --- /dev/null +++ b/TileStache/Goodies/VecTiles/OSciMap4/TileData_v4.proto @@ -0,0 +1,92 @@ +// Protocol Version 4 + +package org.oscim.database.oscimap4; + +message Data { + message Element { + + // number of geometry 'indices' + optional uint32 num_indices = 1 [default = 1]; + + // number of 'tags' + optional uint32 num_tags = 2 [default = 1]; + + // elevation per coordinate + // (pixel relative to ground meters) + // optional bool has_elevation = 3 [default = false]; + + // reference to tile.tags + repeated uint32 tags = 11 [packed = true]; + + // A list of number of coordinates for each geometry. + // - polygons are separated by one '0' index + // - for single points this can be omitted. + // e.g 2,2 for two lines with two points each, or + // 4,3,0,4,3 for two polygons with four points in + // the outer ring and 3 points in the inner. + + repeated uint32 indices = 12 [packed = true]; + + // single delta encoded coordinate x,y pairs scaled + // to a tile size of 4096 + // note: geometries start at x,y = tile size / 2 + + repeated sint32 coordinates = 13 [packed = true]; + + //---------------- optional items --------------- + // osm layer [-5 .. 5] -> [0 .. 10] + optional uint32 layer = 21 [default = 5]; + + // intended for symbol and label placement, not used + //optional uint32 rank = 32 [packed = true]; + + // elevation per coordinate + // (pixel relative to ground meters) + // repeated sint32 elevation = 33 [packed = true]; + + // building height, precision 1/10m + //repeated sint32 height = 34 [packed = true]; + + // building height, precision 1/10m + //repeated sint32 min_height = 35 [packed = true]; + } + + required uint32 version = 1; + + // tile creation time + optional uint64 timestamp = 2; + + // tile is completely water (not used yet) + optional bool water = 3; + + // number of 'tags' + required uint32 num_tags = 11; + optional uint32 num_keys = 12 [default = 0]; + optional uint32 num_vals = 13 [default = 0]; + + // strings referenced by tags + repeated string keys = 14; + // separate common attributes from label to + // allow + repeated string values = 15; + + // (key[0xfffffffc] | type[0x03]), value pairs + // key: uint32 -> reference to key-strings + // type 0: attribute -> uint32 reference to value-strings + // type 1: string -> uint32 reference to label-strings + // type 2: sint32 + // type 3: float + // value: uint32 interpreted according to 'type' + + repeated uint32 tags = 16 [packed = true]; + + + // linestring + repeated Element lines = 21; + + // polygons (MUST be implicitly closed) + repeated Element polygons = 22; + + // points (POIs) + repeated Element points = 23; +} diff --git a/TileStache/Goodies/VecTiles/OSciMap4/TileData_v4_pb2.py b/TileStache/Goodies/VecTiles/OSciMap4/TileData_v4_pb2.py new file mode 100644 index 00000000..0b1ab288 --- /dev/null +++ b/TileStache/Goodies/VecTiles/OSciMap4/TileData_v4_pb2.py @@ -0,0 +1,203 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! + +from google.protobuf import descriptor +from google.protobuf import message +from google.protobuf import reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + + +DESCRIPTOR = descriptor.FileDescriptor( + name='TileData_v4.proto', + package='org.oscim.database.oscimap4', + serialized_pb='\n\x11TileData_v4.proto\x12\x1borg.oscim.database.oscimap4\"\xe2\x03\n\x04\x44\x61ta\x12\x0f\n\x07version\x18\x01 \x02(\r\x12\x11\n\ttimestamp\x18\x02 \x01(\x04\x12\r\n\x05water\x18\x03 \x01(\x08\x12\x10\n\x08num_tags\x18\x0b \x02(\r\x12\x13\n\x08num_keys\x18\x0c \x01(\r:\x01\x30\x12\x13\n\x08num_vals\x18\r \x01(\r:\x01\x30\x12\x0c\n\x04keys\x18\x0e \x03(\t\x12\x0e\n\x06values\x18\x0f \x03(\t\x12\x10\n\x04tags\x18\x10 \x03(\rB\x02\x10\x01\x12\x38\n\x05lines\x18\x15 \x03(\x0b\x32).org.oscim.database.oscimap4.Data.Element\x12;\n\x08polygons\x18\x16 \x03(\x0b\x32).org.oscim.database.oscimap4.Data.Element\x12\x39\n\x06points\x18\x17 \x03(\x0b\x32).org.oscim.database.oscimap4.Data.Element\x1a\x88\x01\n\x07\x45lement\x12\x16\n\x0bnum_indices\x18\x01 \x01(\r:\x01\x31\x12\x13\n\x08num_tags\x18\x02 \x01(\r:\x01\x31\x12\x10\n\x04tags\x18\x0b \x03(\rB\x02\x10\x01\x12\x13\n\x07indices\x18\x0c \x03(\rB\x02\x10\x01\x12\x17\n\x0b\x63oordinates\x18\r \x03(\x11\x42\x02\x10\x01\x12\x10\n\x05layer\x18\x15 \x01(\r:\x01\x35') + + + + +_DATA_ELEMENT = descriptor.Descriptor( + name='Element', + full_name='org.oscim.database.oscimap4.Data.Element', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='num_indices', full_name='org.oscim.database.oscimap4.Data.Element.num_indices', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=True, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='num_tags', full_name='org.oscim.database.oscimap4.Data.Element.num_tags', index=1, + number=2, type=13, cpp_type=3, label=1, + has_default_value=True, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='tags', full_name='org.oscim.database.oscimap4.Data.Element.tags', index=2, + number=11, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), + descriptor.FieldDescriptor( + name='indices', full_name='org.oscim.database.oscimap4.Data.Element.indices', index=3, + number=12, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), + descriptor.FieldDescriptor( + name='coordinates', full_name='org.oscim.database.oscimap4.Data.Element.coordinates', index=4, + number=13, type=17, cpp_type=1, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), + descriptor.FieldDescriptor( + name='layer', full_name='org.oscim.database.oscimap4.Data.Element.layer', index=5, + number=21, type=13, cpp_type=3, label=1, + has_default_value=True, default_value=5, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=397, + serialized_end=533, +) + +_DATA = descriptor.Descriptor( + name='Data', + full_name='org.oscim.database.oscimap4.Data', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='version', full_name='org.oscim.database.oscimap4.Data.version', index=0, + number=1, type=13, cpp_type=3, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='timestamp', full_name='org.oscim.database.oscimap4.Data.timestamp', index=1, + number=2, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='water', full_name='org.oscim.database.oscimap4.Data.water', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='num_tags', full_name='org.oscim.database.oscimap4.Data.num_tags', index=3, + number=11, type=13, cpp_type=3, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='num_keys', full_name='org.oscim.database.oscimap4.Data.num_keys', index=4, + number=12, type=13, cpp_type=3, label=1, + has_default_value=True, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='num_vals', full_name='org.oscim.database.oscimap4.Data.num_vals', index=5, + number=13, type=13, cpp_type=3, label=1, + has_default_value=True, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='keys', full_name='org.oscim.database.oscimap4.Data.keys', index=6, + number=14, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='values', full_name='org.oscim.database.oscimap4.Data.values', index=7, + number=15, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='tags', full_name='org.oscim.database.oscimap4.Data.tags', index=8, + number=16, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), + descriptor.FieldDescriptor( + name='lines', full_name='org.oscim.database.oscimap4.Data.lines', index=9, + number=21, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='polygons', full_name='org.oscim.database.oscimap4.Data.polygons', index=10, + number=22, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='points', full_name='org.oscim.database.oscimap4.Data.points', index=11, + number=23, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_DATA_ELEMENT, ], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=51, + serialized_end=533, +) + +_DATA_ELEMENT.containing_type = _DATA; +_DATA.fields_by_name['lines'].message_type = _DATA_ELEMENT +_DATA.fields_by_name['polygons'].message_type = _DATA_ELEMENT +_DATA.fields_by_name['points'].message_type = _DATA_ELEMENT +DESCRIPTOR.message_types_by_name['Data'] = _DATA + +class Data(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + + class Element(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + DESCRIPTOR = _DATA_ELEMENT + + # @@protoc_insertion_point(class_scope:org.oscim.database.oscimap4.Data.Element) + DESCRIPTOR = _DATA + + # @@protoc_insertion_point(class_scope:org.oscim.database.oscimap4.Data) + +# @@protoc_insertion_point(module_scope) diff --git a/TileStache/Goodies/VecTiles/OSciMap4/__init__.py b/TileStache/Goodies/VecTiles/OSciMap4/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/TileStache/Goodies/VecTiles/oscimap.py b/TileStache/Goodies/VecTiles/oscimap.py new file mode 100644 index 00000000..8fc53f83 --- /dev/null +++ b/TileStache/Goodies/VecTiles/oscimap.py @@ -0,0 +1,190 @@ +import types +from OSciMap4 import TileData_v4_pb2 +from OSciMap4.GeomEncoder import GeomEncoder +from OSciMap4.StaticVals import getValues +from OSciMap4.StaticKeys import getKeys +from OSciMap4.TagRewrite import fixTag +from TileStache.Core import KnownUnknown +import re +import logging +import struct + +statickeys = getKeys() +staticvals = getValues() + +# custom keys/values start at attrib_offset +attrib_offset = 256 + +# coordindates are scaled to this range within tile +extents = 4096 + +def encode(file, features, coord): + tile = VectorTile(extents) + + for feature in features: + tile.addFeature(feature, coord) + + tile.complete() + + data = tile.out.SerializeToString() + file.write(struct.pack(">I", len(data))) + file.write(data) + +class VectorTile: + """ + """ + def __init__(self, extents): + self.geomencoder = GeomEncoder(extents) + + # TODO count to sort by number of occurrences + self.keydict = {} + self.cur_key = attrib_offset + + self.valdict = {} + self.cur_val = attrib_offset + + self.tagdict = {} + self.num_tags = 0 + + self.out = TileData_v4_pb2.Data() + self.out.version = 4 + + + def complete(self): + if self.num_tags == 0: + logging.info("empty tags") + + self.out.num_tags = self.num_tags + + if self.cur_key - attrib_offset > 0: + self.out.num_keys = self.cur_key - attrib_offset + + if self.cur_val - attrib_offset > 0: + self.out.num_vals = self.cur_val - attrib_offset + + def addFeature(self, row, coord): + geom = self.geomencoder + tags = [] + + #height = None + layer = None + + for tag in row[1].iteritems(): + if tag[1] is None: + continue + tag = (str(tag[0]), str(tag[1])) + + # use unsigned int for layer. i.e. map to 0..10 + if "layer" == tag[0]: + layer = self.getLayer(tag[1]) + continue + + tag = fixTag(tag, coord.zoom) + logging.info(tag) + + if tag is None: + continue + + tags.append(self.getTagId(tag)) + + if len(tags) == 0: + logging.debug('missing tags') + return + + geom.parseGeometry(row[0]) + feature = None; + + if geom.isPoint: + feature = self.out.points.add() + # add number of points (for multi-point) + if len(geom.coordinates) > 2: + logging.info('points %s' %len(geom.coordinates)) + feature.indices.add(geom.coordinates/2) + else: + # empty geometry + if len(geom.index) == 0: + logging.debug('empty geom: %s %s' % row[1]) + return + + if geom.isPoly: + feature = self.out.polygons.add() + else: + feature = self.out.lines.add() + + # add coordinate index list (coordinates per geometry) + feature.indices.extend(geom.index) + + # add indice count (number of geometries) + if len(feature.indices) > 1: + feature.num_indices = len(feature.indices) + + # add coordinates + feature.coordinates.extend(geom.coordinates) + + # add tags + feature.tags.extend(tags) + if len(tags) > 1: + feature.num_tags = len(tags) + + # add osm layer + if layer is not None and layer != 5: + feature.layer = layer + + #logging.debug('tags %d, indices %d' %(len(tags),len(feature.indices))) + + + def getLayer(self, val): + try: + l = max(min(10, int(val)) + 5, 0) + if l != 0: + return l + except ValueError: + logging.debug("layer invalid %s" %val) + + return None + + def getKeyId(self, key): + if key in statickeys: + return statickeys[key] + + if key in self.keydict: + return self.keydict[key] + + self.out.keys.append(key); + + r = self.cur_key + self.keydict[key] = r + self.cur_key += 1 + return r + + def getAttribId(self, var): + if var in staticvals: + return staticvals[var] + + if var in self.valdict: + return self.valdict[var] + + self.out.values.append(var); + + r = self.cur_val + self.valdict[var] = r + self.cur_val += 1 + return r + + + def getTagId(self, tag): + # logging.debug(tag) + + if self.tagdict.has_key(tag): + return self.tagdict[tag] + + key = self.getKeyId(tag[0].decode('utf-8')) + val = self.getAttribId(tag[1].decode('utf-8')) + + self.out.tags.append(key) + self.out.tags.append(val) + #logging.info("add tag %s - %d/%d" %(tag, key, val)) + r = self.num_tags + self.tagdict[tag] = r + self.num_tags += 1 + return r diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index c7f90c5d..9b97dd8f 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -22,7 +22,7 @@ def connect(*args, **kwargs): raise err -from . import mvt, geojson, topojson +from . import mvt, geojson, topojson, oscimap from ...Geography import SphericalMercator from ModestMaps.Core import Point @@ -154,8 +154,8 @@ def renderTile(self, width, height, srs, coord): self.columns[query] = query_columns(self.dbinfo, self.srid, query, bounds) tolerance = self.simplify * tolerances[coord.zoom] if coord.zoom < self.simplify_until else None - - return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip) + + return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord) def getTypeByExtension(self, extension): ''' Get mime-type and format by file extension, one of "mvt", "json" or "topojson". @@ -168,7 +168,10 @@ def getTypeByExtension(self, extension): elif extension.lower() == 'topojson': return 'application/json', 'TopoJSON' - + + elif extension.lower() == 'vtm': + return 'image/png', 'OpenScienceMap' # TODO: make this proper stream type, app only seems to work with png + else: raise ValueError(extension) @@ -232,7 +235,7 @@ def __exit__(self, type, value, traceback): class Response: ''' ''' - def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip): + def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip, coord): ''' Create a new response object with Postgres connection info and a query. bounds argument is a 4-tuple with (xmin, ymin, xmax, ymax). @@ -241,12 +244,14 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli self.bounds = bounds self.zoom = zoom self.clip = clip - + self.coord = coord + bbox = 'ST_MakeBox2D(ST_MakePoint(%.2f, %.2f), ST_MakePoint(%.2f, %.2f))' % bounds geo_query = build_query(srid, subquery, columns, bbox, tolerance, True, clip) merc_query = build_query(srid, subquery, columns, bbox, tolerance, False, clip) - self.query = dict(TopoJSON=geo_query, JSON=geo_query, MVT=merc_query) - + oscimap_query = build_query(srid, subquery, columns, bbox, tolerance, False, clip, oscimap.extents) + self.query = dict(TopoJSON=geo_query, JSON=geo_query, MVT=merc_query, OpenScienceMap=oscimap_query) + def save(self, out, format): ''' ''' @@ -279,7 +284,10 @@ def save(self, out, format): ll = SphericalMercator().projLocation(Point(*self.bounds[0:2])) ur = SphericalMercator().projLocation(Point(*self.bounds[2:4])) topojson.encode(out, features, (ll.lon, ll.lat, ur.lon, ur.lat), self.clip) - + + elif format == 'OpenScienceMap': + oscimap.encode(out, features, self.coord) + else: raise ValueError(format) @@ -302,7 +310,10 @@ def save(self, out, format): ll = SphericalMercator().projLocation(Point(*self.bounds[0:2])) ur = SphericalMercator().projLocation(Point(*self.bounds[2:4])) topojson.encode(out, [], (ll.lon, ll.lat, ur.lon, ur.lat), False) - + + elif format == 'OpenScienceMap': + oscimap.encode(out, [], self.coord) + else: raise ValueError(format) @@ -357,8 +368,8 @@ def query_columns(dbinfo, srid, subquery, bounds): column_names = set(row.keys()) return column_names - -def build_query(srid, subquery, subcolumns, bbox, tolerance, is_geo, is_clipped): + +def build_query(srid, subquery, subcolumns, bbox, tolerance, is_geo, is_clipped, scale=None): ''' Build and return an PostGIS query. ''' bbox = 'ST_SetSRID(%s, %d)' % (bbox, srid) @@ -372,7 +383,11 @@ def build_query(srid, subquery, subcolumns, bbox, tolerance, is_geo, is_clipped) if is_geo: geom = 'ST_Transform(%s, 4326)' % geom - + + # TODO: move this out of the query? + if scale: + geom = 'ST_TransScale(%s, -ST_XMin(%s), -ST_YMin(%s), (%d / (ST_XMax(%s) - ST_XMin(%s))), (%d / (ST_YMax(%s) - ST_YMin(%s))))' % (geom, bbox, bbox, scale, bbox, bbox, scale, bbox, bbox) + subquery = subquery.replace('!bbox!', bbox) columns = ['q."%s"' % c for c in subcolumns if c not in ('__geometry__', )] From 88fdf92d8652028b7d4da691f160a8c10d97c1ff Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Thu, 30 Jan 2014 13:52:32 -0500 Subject: [PATCH 003/344] adding support for opensciencemap (vtm) multiprovider. moving db/feature stuff into get_features --- TileStache/Goodies/VecTiles/server.py | 54 +++++++++++++++++---------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 9b97dd8f..e1669400 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -212,6 +212,9 @@ def getTypeByExtension(self, extension): elif extension.lower() == 'topojson': return 'application/json', 'TopoJSON' + + elif extension.lower() == 'vtm': + return 'image/png', 'OpenScienceMap' # TODO: make this proper stream type, app only seems to work with png else: raise ValueError(extension) @@ -255,24 +258,7 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli def save(self, out, format): ''' ''' - with Connection(self.dbinfo) as db: - db.execute(self.query[format]) - - features = [] - - for row in db.fetchall(): - if row['__geometry__'] is None: - continue - - wkb = bytes(row['__geometry__']) - prop = dict([(k, v) for (k, v) in row.items() - if k not in ('__geometry__', '__id__')]) - - if '__id__' in row: - features.append((wkb, prop, row['__id__'])) - - else: - features.append((wkb, prop)) + features = get_features(self.dbinfo, self.query[format]) if format == 'MVT': mvt.encode(out, features) @@ -326,7 +312,6 @@ def __init__(self, config, names, coord): self.config = config self.names = names self.coord = coord - def save(self, out, format): ''' ''' @@ -335,6 +320,16 @@ def save(self, out, format): elif format == 'JSON': geojson.merge(out, self.names, self.config, self.coord) + + elif format == 'OpenScienceMap': + features = [] + layers = [self.config.layers[name] for name in self.names] + for layer in layers: + width, height = layer.dim, layer.dim + tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) + if isinstance(tile,EmptyResponse): continue + features.extend(get_features(tile.dbinfo, tile.query["OpenScienceMap"])) + oscimap.encode(out, features, self.coord) else: raise ValueError(format) @@ -369,6 +364,27 @@ def query_columns(dbinfo, srid, subquery, bounds): column_names = set(row.keys()) return column_names +def get_features(dbinfo, query): + with Connection(dbinfo) as db: + db.execute(query) + + features = [] + + for row in db.fetchall(): + if row['__geometry__'] is None: + continue + + wkb = bytes(row['__geometry__']) + prop = dict([(k, v) for (k, v) in row.items() + if k not in ('__geometry__', '__id__')]) + + if '__id__' in row: + features.append((wkb, prop, row['__id__'])) + + else: + features.append((wkb, prop)) + return features + def build_query(srid, subquery, subcolumns, bbox, tolerance, is_geo, is_clipped, scale=None): ''' Build and return an PostGIS query. ''' From 303f6fca4ed74ceec10912fcae45ba8e441d49bd Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Thu, 30 Jan 2014 13:54:02 -0500 Subject: [PATCH 004/344] adding pbf_test --- .../Goodies/VecTiles/OSciMap4/pbf_test.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 TileStache/Goodies/VecTiles/OSciMap4/pbf_test.py diff --git a/TileStache/Goodies/VecTiles/OSciMap4/pbf_test.py b/TileStache/Goodies/VecTiles/OSciMap4/pbf_test.py new file mode 100644 index 00000000..ef56ba43 --- /dev/null +++ b/TileStache/Goodies/VecTiles/OSciMap4/pbf_test.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +# +# protoc --python_out=. proto/TileData.proto +# + +import sys +import TileData_v4_pb2 + +if __name__ == "__main__" : + if len(sys.argv) != 2 : + print>>sys.stderr, "Usage:", sys.argv[0], "" + sys.exit(1) + + tile = TileData_v4_pb2.Data() + + try: + f = open(sys.argv[1], "rb") + tile.ParseFromString(f.read()[4:]) + f.close() + except IOError: + print sys.argv[1] + ": Could not open file. Creating a new one." + + print tile \ No newline at end of file From 687925e6b762aaaac4906ab1fcd59f33427cb86b Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Thu, 30 Jan 2014 13:56:10 -0500 Subject: [PATCH 005/344] EmptyResponse instance has no attribute 'coord' --- TileStache/Goodies/VecTiles/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index e1669400..adebcd58 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -298,7 +298,7 @@ def save(self, out, format): topojson.encode(out, [], (ll.lon, ll.lat, ur.lon, ur.lat), False) elif format == 'OpenScienceMap': - oscimap.encode(out, [], self.coord) + oscimap.encode(out, [], None) else: raise ValueError(format) From cd1a82fa3ac4b7afeeabf118950d31ff12727e80 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Fri, 31 Jan 2014 14:04:11 -0500 Subject: [PATCH 006/344] Moving lines that write json stream to a file into its own function --- TileStache/Goodies/VecTiles/geojson.py | 29 ++++++++++---------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/TileStache/Goodies/VecTiles/geojson.py b/TileStache/Goodies/VecTiles/geojson.py index 66383889..0a1eeae2 100644 --- a/TileStache/Goodies/VecTiles/geojson.py +++ b/TileStache/Goodies/VecTiles/geojson.py @@ -91,21 +91,8 @@ def encode(file, features, zoom, is_clipped): feature.update(dict(clipped=True)) geojson = dict(type='FeatureCollection', features=features) - encoder = json.JSONEncoder(separators=(',', ':')) - encoded = encoder.iterencode(geojson) - flt_fmt = '%%.%df' % precisions[zoom] - - for token in encoded: - if charfloat_pat.match(token): - # in python 2.7, we see a character followed by a float literal - file.write(token[0] + flt_fmt % float(token[1:])) - - elif float_pat.match(token): - # in python 2.6, we see a simple float literal - file.write(flt_fmt % float(token)) - - else: - file.write(token) + + write_to_file(file, geojson, zoom) def merge(file, names, config, coord): ''' Retrieve a list of GeoJSON tile responses and merge them into one. @@ -115,9 +102,15 @@ def merge(file, names, config, coord): inputs = get_tiles(names, config, coord) output = dict(zip(names, inputs)) + write_to_file(file, output, coord.zoom) + +def write_to_file(file, geojson, zoom): + ''' Write GeoJSON stream to a file + + ''' encoder = json.JSONEncoder(separators=(',', ':')) - encoded = encoder.iterencode(output) - flt_fmt = '%%.%df' % precisions[coord.zoom] + encoded = encoder.iterencode(geojson) + flt_fmt = '%%.%df' % precisions[zoom] for token in encoded: if charfloat_pat.match(token): @@ -129,4 +122,4 @@ def merge(file, names, config, coord): file.write(flt_fmt % float(token)) else: - file.write(token) + file.write(token) \ No newline at end of file From dd40e37668f9e7053bf1cc1ecaf5b8ef49a11165 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Mon, 3 Feb 2014 15:42:30 -0500 Subject: [PATCH 007/344] adding OsciMap4 to the packages array --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8004e5db..7abf61c3 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,11 @@ def is_installed(name): 'TileStache.Goodies', 'TileStache.Goodies.Caches', 'TileStache.Goodies.Providers', - 'TileStache.Goodies.VecTiles'], + 'TileStache.Goodies.VecTiles', + 'TileStache.Goodies.VecTiles/OSciMap4/StaticKeys', + 'TileStache.Goodies.VecTiles/OSciMap4/StaticVals', + 'TileStache.Goodies.VecTiles/OSciMap4/TagRewrite', + 'TileStache.Goodies.VecTiles/OSciMap4'], scripts=['scripts/tilestache-compose.py', 'scripts/tilestache-seed.py', 'scripts/tilestache-clean.py', 'scripts/tilestache-server.py', 'scripts/tilestache-render.py', 'scripts/tilestache-list.py'], data_files=[('share/tilestache', ['TileStache/Goodies/Providers/DejaVuSansMono-alphanumeric.ttf'])], download_url='http://tilestache.org/download/TileStache-%(version)s.tar.gz' % locals(), From 03a9016b0eb365e109c17d3400edf047e4967f54 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Wed, 5 Feb 2014 16:13:04 -0500 Subject: [PATCH 008/344] moving get_tiles into server.py and using it for both formats. --- TileStache/Goodies/VecTiles/geojson.py | 34 ++------------------ TileStache/Goodies/VecTiles/server.py | 30 ++++++++++++++++-- TileStache/Goodies/VecTiles/topojson.py | 42 ++++--------------------- 3 files changed, 36 insertions(+), 70 deletions(-) diff --git a/TileStache/Goodies/VecTiles/geojson.py b/TileStache/Goodies/VecTiles/geojson.py index 0a1eeae2..af014c64 100644 --- a/TileStache/Goodies/VecTiles/geojson.py +++ b/TileStache/Goodies/VecTiles/geojson.py @@ -6,8 +6,6 @@ from shapely.wkb import loads from shapely.geometry import asShape -from ... import getTile -from ...Core import KnownUnknown from .ops import transform float_pat = compile(r'^-?\d+\.\d+(e-?\d+)?$') @@ -16,32 +14,6 @@ # floating point lat/lon precision for each zoom level, good to ~1/4 pixel. precisions = [int(ceil(log(1< 1: - raise KnownUnknown('%s.get_tiles encountered incompatible transforms: %s' % (__name__, list(unique_xforms))) - - return topojsons - def update_arc_indexes(geometry, merged_arcs, old_arcs): ''' Updated geometry arc indexes, and add arcs to merged_arcs along the way. @@ -190,12 +156,16 @@ def encode(file, features, bounds, is_clipped): json.dump(result, file, separators=(',', ':')) -def merge(file, names, config, coord): +def merge(file, names, inputs, config, coord): ''' Retrieve a list of TopoJSON tile responses and merge them into one. get_tiles() retrieves data and performs basic integrity checks. ''' - inputs = get_tiles(names, config, coord) + transforms = [topo['transform'] for topo in inputs] + unique_xforms = set([tuple(xform['scale'] + xform['translate']) for xform in transforms]) + + if len(unique_xforms) > 1: + raise KnownUnknown('%s.merge encountered incompatible transforms: %s' % (__name__, list(unique_xforms))) output = { 'type': 'Topology', From 778a91fff21c2867da19e333b44b2a504840a9e7 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Tue, 18 Mar 2014 16:08:14 -0400 Subject: [PATCH 009/344] first pass - extending multiprovider to handle requests that have a "+" in the path (seperating individual layers for ex: /buildings+places+pois/123/132.json) - requires the config file to have a empty layer called "+" with class "TileStache.Goodies.VecTiles:MultiProvider" and empty names []. --- TileStache/Goodies/VecTiles/server.py | 6 +++++- TileStache/__init__.py | 24 +++++++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index c7f90c5d..7d3568bc 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -195,7 +195,11 @@ class MultiProvider: def __init__(self, layer, names): self.layer = layer self.names = names - + + def __call__(self, layer, names): + self.layer = layer + self.names = names + def renderTile(self, width, height, srs, coord): ''' Render a single tile, return a Response instance. ''' diff --git a/TileStache/__init__.py b/TileStache/__init__.py index ba117951..bdfc0e91 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -138,6 +138,19 @@ def mergePathInfo(layer, coord, extension): return '/%(layer)s/%(z)d/%(x)d/%(y)d.%(extension)s' % locals() +def isNotValidLayer(layer, config): + if not layer: + return True + if (layer not in config.layers): + if (layer.find("+") != -1): + multi_providers = list(ll for ll in config.layers if hasattr(config.layers[ll].provider, 'names')) + for l in layer.split("+"): + if ((l not in config.layers) or (l in multi_providers)): + return True + return False + return True + return False + def requestLayer(config, path_info): """ Return a Layer. @@ -175,10 +188,15 @@ def requestLayer(config, path_info): layername = splitPathInfo(path_info)[0] - if layername not in config.layers: + if isNotValidLayer(layername, config): raise Core.KnownUnknown('"%s" is not a layer I know about. Here are some that I do know about: %s.' % (layername, ', '.join(sorted(config.layers.keys())))) - return config.layers[layername] + customLayer = layername.find("+")!=-1 + + if customLayer: + config.layers["+"].provider(config.layers["+"], **{'names': layername.split("+")}) + + return config.layers[layername] if not customLayer else config.layers["+"] def requestHandler(config_hint, path_info, query_string=None): """ Generate a mime-type and response body for a given request. @@ -369,7 +387,7 @@ def __call__(self, environ, start_response): # WSGI behavior is different from CGI behavior, because we may not want # to return a chatty rummy for likely-deployed WSGI vs. testing CGI. # - if layer and layer not in self.config.layers: + if isNotValidLayer(layer, self.config): return self._response(start_response, 404) path_info = environ.get('PATH_INFO', None) From 48b09924831d0c39d042517aa725c306656c541f Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Thu, 3 Apr 2014 17:19:27 -0400 Subject: [PATCH 010/344] unknownLayerMessage: A message that notifies that the given layer is unknown and lists out the known layers. --- TileStache/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/TileStache/__init__.py b/TileStache/__init__.py index ba117951..4018146a 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -66,6 +66,11 @@ def getTile(layer, coord, extension, ignore_cached=False): return mime, body +def unknownLayerMessage(config, unknown_layername): + """ A message that notifies that the given layer is unknown and lists out the known layers. + """ + return '"%s" is not a layer I know about. \nHere are some that I do know about: \n %s.' % (unknown_layername, '\n '.join(sorted(config.layers.keys()))) + def getPreview(layer): """ Get a type string and dynamic map viewer HTML for a given layer. """ @@ -176,7 +181,7 @@ def requestLayer(config, path_info): layername = splitPathInfo(path_info)[0] if layername not in config.layers: - raise Core.KnownUnknown('"%s" is not a layer I know about. Here are some that I do know about: %s.' % (layername, ', '.join(sorted(config.layers.keys())))) + raise Core.KnownUnknown(unknownLayerMessage(config, layername)) return config.layers[layername] @@ -370,7 +375,7 @@ def __call__(self, environ, start_response): # to return a chatty rummy for likely-deployed WSGI vs. testing CGI. # if layer and layer not in self.config.layers: - return self._response(start_response, 404) + return self._response(start_response, 404, str(unknownLayerMessage(self.config, layer))) path_info = environ.get('PATH_INFO', None) query_string = environ.get('QUERY_STRING', None) From 905eb314c694dd86357475520827451675d5c6f7 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Mon, 7 Apr 2014 15:45:03 -0400 Subject: [PATCH 011/344] printing and logging all exceptions in addition to core.knownunknowns --- TileStache/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/TileStache/__init__.py b/TileStache/__init__.py index 4018146a..1db6bc54 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -268,10 +268,11 @@ def requestHandler2(config_hint, path_info, query_string=None, script_name=''): headers.setdefault('Expires', expires.strftime('%a %d %b %Y %H:%M:%S GMT')) headers.setdefault('Cache-Control', 'public, max-age=%d' % layer.max_cache_age) - except Core.KnownUnknown, e: + except (Core.KnownUnknown, Exception), e: + logging.exception(e) out = StringIO() - print >> out, 'Known unknown!' + print >> out, 'Known unknown!' if isinstance(e,Core.KnownUnknown) else 'Exception!' print >> out, e print >> out, '' print >> out, '\n'.join(Core._rummy()) From 9958eb9ccad6082fa9820d48f7bf0e7136aca7af Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Mon, 7 Apr 2014 15:47:53 -0400 Subject: [PATCH 012/344] descriptive error messages --- TileStache/Goodies/VecTiles/server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index c7f90c5d..2cf3b91b 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -170,7 +170,7 @@ def getTypeByExtension(self, extension): return 'application/json', 'TopoJSON' else: - raise ValueError(extension) + raise ValueError(extension + " is not a valid extension") class MultiProvider: ''' VecTiles provider to gather PostGIS tiles into a single multi-response. @@ -211,7 +211,7 @@ def getTypeByExtension(self, extension): return 'application/json', 'TopoJSON' else: - raise ValueError(extension) + raise ValueError(extension + " is not a valid extension for responses with multiple layers") class Connection: ''' Context manager for Postgres connections. @@ -281,7 +281,7 @@ def save(self, out, format): topojson.encode(out, features, (ll.lon, ll.lat, ur.lon, ur.lat), self.clip) else: - raise ValueError(format) + raise ValueError(format + " is not supported") class EmptyResponse: ''' Simple empty response renders valid MVT or GeoJSON with no features. @@ -304,7 +304,7 @@ def save(self, out, format): topojson.encode(out, [], (ll.lon, ll.lat, ur.lon, ur.lat), False) else: - raise ValueError(format) + raise ValueError(format + " is not supported") class MultiResponse: ''' @@ -326,7 +326,7 @@ def save(self, out, format): geojson.merge(out, self.names, self.config, self.coord) else: - raise ValueError(format) + raise ValueError(format + " is not supported for responses with multiple layers") def query_columns(dbinfo, srid, subquery, bounds): ''' Get information about the columns returned for a subquery. From 2c89ac80333fda2245a660424e144d4a784d7e93 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Mon, 7 Apr 2014 17:05:06 -0400 Subject: [PATCH 013/344] using comma-delimited list instead of + --- TileStache/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/TileStache/__init__.py b/TileStache/__init__.py index bdfc0e91..3164198d 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -142,9 +142,9 @@ def isNotValidLayer(layer, config): if not layer: return True if (layer not in config.layers): - if (layer.find("+") != -1): + if (layer.find(",") != -1): multi_providers = list(ll for ll in config.layers if hasattr(config.layers[ll].provider, 'names')) - for l in layer.split("+"): + for l in layer.split(","): if ((l not in config.layers) or (l in multi_providers)): return True return False @@ -191,12 +191,12 @@ def requestLayer(config, path_info): if isNotValidLayer(layername, config): raise Core.KnownUnknown('"%s" is not a layer I know about. Here are some that I do know about: %s.' % (layername, ', '.join(sorted(config.layers.keys())))) - customLayer = layername.find("+")!=-1 + customLayer = layername.find(",")!=-1 if customLayer: - config.layers["+"].provider(config.layers["+"], **{'names': layername.split("+")}) + config.layers[","].provider(config.layers[","], **{'names': layername.split(",")}) - return config.layers[layername] if not customLayer else config.layers["+"] + return config.layers[layername] if not customLayer else config.layers[","] def requestHandler(config_hint, path_info, query_string=None): """ Generate a mime-type and response body for a given request. From b74540594dd0e2de6cb997dcb31aa334879abeff Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Mon, 7 Apr 2014 17:14:08 -0400 Subject: [PATCH 014/344] isNotValidLayer -> isValidLayer --- TileStache/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/TileStache/__init__.py b/TileStache/__init__.py index 3164198d..d5a98f96 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -138,18 +138,18 @@ def mergePathInfo(layer, coord, extension): return '/%(layer)s/%(z)d/%(x)d/%(y)d.%(extension)s' % locals() -def isNotValidLayer(layer, config): +def isValidLayer(layer, config): if not layer: - return True + return False if (layer not in config.layers): if (layer.find(",") != -1): multi_providers = list(ll for ll in config.layers if hasattr(config.layers[ll].provider, 'names')) for l in layer.split(","): if ((l not in config.layers) or (l in multi_providers)): - return True - return False - return True - return False + return False + return True + return False + return True def requestLayer(config, path_info): """ Return a Layer. @@ -188,7 +188,7 @@ def requestLayer(config, path_info): layername = splitPathInfo(path_info)[0] - if isNotValidLayer(layername, config): + if not isValidLayer(layername, config): raise Core.KnownUnknown('"%s" is not a layer I know about. Here are some that I do know about: %s.' % (layername, ', '.join(sorted(config.layers.keys())))) customLayer = layername.find(",")!=-1 @@ -387,7 +387,7 @@ def __call__(self, environ, start_response): # WSGI behavior is different from CGI behavior, because we may not want # to return a chatty rummy for likely-deployed WSGI vs. testing CGI. # - if isNotValidLayer(layer, self.config): + if not isValidLayer(layer, self.config): return self._response(start_response, 404) path_info = environ.get('PATH_INFO', None) From 7e89e1dd4f215d678471baa5aa00fce887a13117 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Mon, 7 Apr 2014 17:35:11 -0400 Subject: [PATCH 015/344] global _delimiter set to comma --- TileStache/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/TileStache/__init__.py b/TileStache/__init__.py index d5a98f96..00a32147 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -46,6 +46,9 @@ _pathinfo_pat = re.compile(r'^/?(?P\w.+)/(?P\d+)/(?P-?\d+)/(?P-?\d+)\.(?P\w+)$') _preview_pat = re.compile(r'^/?(?P\w.+)/(preview\.html)?$') +# symbol used to separate layers when specifying more than one layer +_delimiter = ',' + def getTile(layer, coord, extension, ignore_cached=False): ''' Get a type string and tile binary for a given request layer tile. @@ -142,9 +145,9 @@ def isValidLayer(layer, config): if not layer: return False if (layer not in config.layers): - if (layer.find(",") != -1): + if (layer.find(_delimiter) != -1): multi_providers = list(ll for ll in config.layers if hasattr(config.layers[ll].provider, 'names')) - for l in layer.split(","): + for l in layer.split(_delimiter): if ((l not in config.layers) or (l in multi_providers)): return False return True @@ -191,12 +194,12 @@ def requestLayer(config, path_info): if not isValidLayer(layername, config): raise Core.KnownUnknown('"%s" is not a layer I know about. Here are some that I do know about: %s.' % (layername, ', '.join(sorted(config.layers.keys())))) - customLayer = layername.find(",")!=-1 + customLayer = layername.find(_delimiter)!=-1 if customLayer: - config.layers[","].provider(config.layers[","], **{'names': layername.split(",")}) + config.layers[_delimiter].provider(config.layers[_delimiter], **{'names': layername.split(_delimiter)}) - return config.layers[layername] if not customLayer else config.layers[","] + return config.layers[layername] if not customLayer else config.layers[_delimiter] def requestHandler(config_hint, path_info, query_string=None): """ Generate a mime-type and response body for a given request. From 98f843522c634f8af7344deb4fffd67d29f9bde8 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Mon, 7 Apr 2014 18:40:51 -0400 Subject: [PATCH 016/344] not dependendent on the config file anymore --- TileStache/Config.py | 8 +++++++- TileStache/__init__.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/TileStache/Config.py b/TileStache/Config.py index 456ceaef..9ce2d665 100644 --- a/TileStache/Config.py +++ b/TileStache/Config.py @@ -124,6 +124,10 @@ def __init__(self, cache, dirpath): self.dirpath = dirpath self.layers = {} + # adding custom_layer to extend multiprovider to support comma separated layernames + self.custom_layer_name = "," + self.custom_layer_dict = {'provider': {'class': 'TileStache.Goodies.VecTiles:MultiProvider', 'kwargs': {'names': []}}} + self.index = 'text/plain', 'TileStache bellows hello.' class Bounds: @@ -204,7 +208,7 @@ def buildConfiguration(config_dict, dirpath='.'): URL including the "file://" prefix. """ scheme, h, path, p, q, f = urlparse(dirpath) - + if scheme in ('', 'file'): sys.path.insert(0, path) @@ -216,6 +220,8 @@ def buildConfiguration(config_dict, dirpath='.'): for (name, layer_dict) in config_dict.get('layers', {}).items(): config.layers[name] = _parseConfigfileLayer(layer_dict, config, dirpath) + config.layers[config.custom_layer_name] = _parseConfigfileLayer(config.custom_layer_dict, config, dirpath) + if 'index' in config_dict: index_href = urljoin(dirpath, config_dict['index']) index_body = urlopen(index_href).read() diff --git a/TileStache/__init__.py b/TileStache/__init__.py index 00a32147..3e5905d8 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -197,9 +197,9 @@ def requestLayer(config, path_info): customLayer = layername.find(_delimiter)!=-1 if customLayer: - config.layers[_delimiter].provider(config.layers[_delimiter], **{'names': layername.split(_delimiter)}) + config.layers[config.custom_layer_name].provider(config.layers[config.custom_layer_name], **{'names': layername.split(_delimiter)}) - return config.layers[layername] if not customLayer else config.layers[_delimiter] + return config.layers[layername] if not customLayer else config.layers[config.custom_layer_name] def requestHandler(config_hint, path_info, query_string=None): """ Generate a mime-type and response body for a given request. From 7e3b2099c54aaee4700554872ba0690b026f7bfb Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Mon, 7 Apr 2014 18:43:15 -0400 Subject: [PATCH 017/344] camelCase to snake_case --- TileStache/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/TileStache/__init__.py b/TileStache/__init__.py index 3e5905d8..0a3bf867 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -194,12 +194,12 @@ def requestLayer(config, path_info): if not isValidLayer(layername, config): raise Core.KnownUnknown('"%s" is not a layer I know about. Here are some that I do know about: %s.' % (layername, ', '.join(sorted(config.layers.keys())))) - customLayer = layername.find(_delimiter)!=-1 + custom_layer = layername.find(_delimiter)!=-1 - if customLayer: + if custom_layer: config.layers[config.custom_layer_name].provider(config.layers[config.custom_layer_name], **{'names': layername.split(_delimiter)}) - return config.layers[layername] if not customLayer else config.layers[config.custom_layer_name] + return config.layers[layername] if not custom_layer else config.layers[config.custom_layer_name] def requestHandler(config_hint, path_info, query_string=None): """ Generate a mime-type and response body for a given request. From 2186f358fa2ba82cf0634960ca62052a4ea95bab Mon Sep 17 00:00:00 2001 From: Brett Camper Date: Tue, 8 Apr 2014 10:49:07 -0400 Subject: [PATCH 018/344] VecTiles: increase max zoom supported for pixel tolerances --- TileStache/Goodies/VecTiles/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index adebcd58..c426497b 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -26,7 +26,7 @@ def connect(*args, **kwargs): from ...Geography import SphericalMercator from ModestMaps.Core import Point -tolerances = [6378137 * 2 * pi / (2 ** (zoom + 8)) for zoom in range(20)] +tolerances = [6378137 * 2 * pi / (2 ** (zoom + 8)) for zoom in range(22)] class Provider: ''' VecTiles provider for PostGIS data sources. From 35123fd3d9e11cc831d2eacf7edbfde27ca29122 Mon Sep 17 00:00:00 2001 From: Brett Camper Date: Tue, 8 Apr 2014 10:50:47 -0400 Subject: [PATCH 019/344] VecTiles: add support to pad tile bounding box by a specified amount(used by OpenScienceMap to hide seams between tiles) moves bounding box SQL construction to build_query function --- TileStache/Goodies/VecTiles/oscimap.py | 3 +++ TileStache/Goodies/VecTiles/server.py | 14 +++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/TileStache/Goodies/VecTiles/oscimap.py b/TileStache/Goodies/VecTiles/oscimap.py index 8fc53f83..19fcf816 100644 --- a/TileStache/Goodies/VecTiles/oscimap.py +++ b/TileStache/Goodies/VecTiles/oscimap.py @@ -18,6 +18,9 @@ # coordindates are scaled to this range within tile extents = 4096 +# tiles are padded by this number of pixels for the current zoom level (OSciMap uses this to cover up seams between tiles) +padding = 5 + def encode(file, features, coord): tile = VectorTile(extents) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index c426497b..8ca4239c 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -249,10 +249,9 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli self.clip = clip self.coord = coord - bbox = 'ST_MakeBox2D(ST_MakePoint(%.2f, %.2f), ST_MakePoint(%.2f, %.2f))' % bounds - geo_query = build_query(srid, subquery, columns, bbox, tolerance, True, clip) - merc_query = build_query(srid, subquery, columns, bbox, tolerance, False, clip) - oscimap_query = build_query(srid, subquery, columns, bbox, tolerance, False, clip, oscimap.extents) + geo_query = build_query(srid, subquery, columns, bounds, tolerance, True, clip) + merc_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip) + oscimap_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, oscimap.padding * tolerances[coord.zoom], oscimap.extents) self.query = dict(TopoJSON=geo_query, JSON=geo_query, MVT=merc_query, OpenScienceMap=oscimap_query) def save(self, out, format): @@ -385,9 +384,10 @@ def get_features(dbinfo, query): features.append((wkb, prop)) return features -def build_query(srid, subquery, subcolumns, bbox, tolerance, is_geo, is_clipped, scale=None): +def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clipped, padding=0, scale=None): ''' Build and return an PostGIS query. ''' + bbox = 'ST_MakeBox2D(ST_MakePoint(%.2f, %.2f), ST_MakePoint(%.2f, %.2f))' % (bounds[0] - padding, bounds[1] - padding, bounds[2] + padding, bounds[3] + padding) bbox = 'ST_SetSRID(%s, %d)' % (bbox, srid) geom = 'q.__geometry__' @@ -400,9 +400,9 @@ def build_query(srid, subquery, subcolumns, bbox, tolerance, is_geo, is_clipped, if is_geo: geom = 'ST_Transform(%s, 4326)' % geom - # TODO: move this out of the query? if scale: - geom = 'ST_TransScale(%s, -ST_XMin(%s), -ST_YMin(%s), (%d / (ST_XMax(%s) - ST_XMin(%s))), (%d / (ST_YMax(%s) - ST_YMin(%s))))' % (geom, bbox, bbox, scale, bbox, bbox, scale, bbox, bbox) + # scale applies to the un-padded bounds, e.g. geometry in the padding area "spills over" past the scale range + geom = 'ST_TransScale(%s, %.2f, %.2f, (%.2f / (%.2f - %.2f)), (%.2f / (%.2f - %.2f)))' % (geom, -bounds[0], -bounds[1], scale, bounds[2], bounds[0], scale, bounds[3], bounds[1]) subquery = subquery.replace('!bbox!', bbox) columns = ['q."%s"' % c for c in subcolumns if c not in ('__geometry__', )] From 98e5b2085f08144649a09ee7f42598a1da8637fe Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Fri, 25 Apr 2014 14:06:01 -0400 Subject: [PATCH 020/344] first pass: adding mapbox vector tile support. (geometry encoding is a WIP for now) --- .../Goodies/VecTiles/Mapbox/GeomEncoder.py | 343 ++++++++++++++++++ .../Goodies/VecTiles/Mapbox/__init__.py | 0 .../Goodies/VecTiles/Mapbox/vector_tile.proto | 92 +++++ .../VecTiles/Mapbox/vector_tile_pb2.py | 298 +++++++++++++++ TileStache/Goodies/VecTiles/mapbox.py | 141 +++++++ TileStache/Goodies/VecTiles/server.py | 85 +++-- setup.py | 3 +- 7 files changed, 935 insertions(+), 27 deletions(-) create mode 100644 TileStache/Goodies/VecTiles/Mapbox/GeomEncoder.py create mode 100644 TileStache/Goodies/VecTiles/Mapbox/__init__.py create mode 100644 TileStache/Goodies/VecTiles/Mapbox/vector_tile.proto create mode 100644 TileStache/Goodies/VecTiles/Mapbox/vector_tile_pb2.py create mode 100644 TileStache/Goodies/VecTiles/mapbox.py diff --git a/TileStache/Goodies/VecTiles/Mapbox/GeomEncoder.py b/TileStache/Goodies/VecTiles/Mapbox/GeomEncoder.py new file mode 100644 index 00000000..0bc143f0 --- /dev/null +++ b/TileStache/Goodies/VecTiles/Mapbox/GeomEncoder.py @@ -0,0 +1,343 @@ +""" +A parser for the Well Text Binary format of OpenGIS types. +""" + +import sys, traceback, struct + + +# based on xdrlib.Unpacker +class _ExtendedUnPacker: + """ + A simple binary struct parser, only implements the types that are need for the WKB format. + """ + + def __init__(self,data): + self.reset(data) + self.setEndianness('XDR') + + def reset(self, data): + self.__buf = data + self.__pos = 0 + + def get_position(self): + return self.__pos + + def set_position(self, position): + self.__pos = position + + def get_buffer(self): + return self.__buf + + def done(self): + if self.__pos < len(self.__buf): + raise ExceptionWKBParser('unextracted data remains') + + def setEndianness(self,endianness): + if endianness == 'XDR': + self._endflag = '>' + elif endianness == 'NDR': + self._endflag = '<' + else: + raise ExceptionWKBParser('Attempt to set unknown endianness in ExtendedUnPacker') + + def unpack_byte(self): + i = self.__pos + self.__pos = j = i+1 + data = self.__buf[i:j] + if len(data) < 1: + raise EOFError + byte = struct.unpack('%sB' % self._endflag, data)[0] + return byte + + def unpack_uint32(self): + i = self.__pos + self.__pos = j = i+4 + data = self.__buf[i:j] + if len(data) < 4: + raise EOFError + uint32 = struct.unpack('%si' % self._endflag, data)[0] + return uint32 + + def unpack_short(self): + i = self.__pos + self.__pos = j = i+2 + data = self.__buf[i:j] + if len(data) < 2: + raise EOFError + short = struct.unpack('%sH' % self._endflag, data)[0] + return short + + def unpack_double(self): + i = self.__pos + self.__pos = j = i+8 + data = self.__buf[i:j] + if len(data) < 8: + raise EOFError + return struct.unpack('%sd' % self._endflag, data)[0] + +class ExceptionWKBParser(Exception): + '''This is the WKB Parser Exception class.''' + def __init__(self, value): + self.value = value + def __str__(self): + return `self.value` + +class GeomEncoder: + + _count = 0 + + def __init__(self, tileSize): + """ + Initialise a new WKBParser. + + """ + + self._typemap = {1: self.parsePoint, + 2: self.parseLineString, + 3: self.parsePolygon, + 4: self.parseMultiPoint, + 5: self.parseMultiLineString, + 6: self.parseMultiPolygon, + 7: self.parseGeometryCollection} + self.coordinates = [] + self.index = [] + self.position = 0 + self.lastX = 0 + self.lastY = 0 + self.dropped = 0 + self.num_points = 0 + self.isPoint = True + self.tileSize = tileSize - 1 + self.first = True + + def parseGeometry(self, geometry): + + + """ + A factory method for creating objects of the correct OpenGIS type. + """ + + self.coordinates = [] + self.index = [] + self.position = 0 + self.lastX = 0 + self.lastY = 0 + self.isPoly = False + self.isPoint = True; + self.dropped = 0; + self.first = True + # Used for exception strings + self._current_string = geometry + + reader = _ExtendedUnPacker(geometry) + + # Start the parsing + self._dispatchNextType(reader) + + + def _dispatchNextType(self,reader): + """ + Read a type id from the binary stream (reader) and call the correct method to parse it. + """ + + # Need to check endianess here! + endianness = reader.unpack_byte() + if endianness == 0: + reader.setEndianness('XDR') + elif endianness == 1: + reader.setEndianness('NDR') + else: + raise ExceptionWKBParser("Invalid endianness in WKB format.\n"\ + "The parser can only cope with XDR/big endian WKB format.\n"\ + "To force the WKB format to be in XDR use AsBinary(,'XDR'") + + + geotype = reader.unpack_uint32() + + mask = geotype & 0x80000000 # This is used to mask of the dimension flag. + + srid = geotype & 0x20000000 + # ignore srid ... + if srid != 0: + reader.unpack_uint32() + + dimensions = 2 + if mask == 0: + dimensions = 2 + else: + dimensions = 3 + + geotype = geotype & 0x1FFFFFFF + # Despatch to a method on the type id. + if self._typemap.has_key(geotype): + self._typemap[geotype](reader, dimensions) + else: + raise ExceptionWKBParser('Error type to dispatch with geotype = %s \n'\ + 'Invalid geometry in WKB string: %s' % (str(geotype), + str(self._current_string),)) + + def parseGeometryCollection(self, reader, dimension): + try: + num_geoms = reader.unpack_uint32() + + for _ in xrange(0,num_geoms): + self._dispatchNextType(reader) + + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + raise ExceptionWKBParser("Caught unhandled exception parsing GeometryCollection: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) + + + def parseLineString(self, reader, dimensions): + self.isPoint = False; + try: + num_points = reader.unpack_uint32() + + self.num_points = 0; + + for _ in xrange(0,num_points): + self.parsePoint(reader,dimensions) + + self.index.append(self.num_points) + #self.lastX = 0 + #self.lastY = 0 + self.first = True + + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + print error + raise ExceptionWKBParser("Caught unhandled exception parsing Linestring: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) + + + def parseMultiLineString(self, reader, dimensions): + try: + num_linestrings = reader.unpack_uint32() + + for _ in xrange(0,num_linestrings): + self._dispatchNextType(reader) + + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + raise ExceptionWKBParser("Caught unhandled exception parsing MultiLineString: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) + + + def parseMultiPoint(self, reader, dimensions): + try: + num_points = reader.unpack_uint32() + + for _ in xrange(0,num_points): + self._dispatchNextType(reader) + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + raise ExceptionWKBParser("Caught unhandled exception parsing MultiPoint: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) + + + def parseMultiPolygon(self, reader, dimensions): + try: + num_polygons = reader.unpack_uint32() + for n in xrange(0,num_polygons): + if n > 0: + self.index.append(0); + + self._dispatchNextType(reader) + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + raise ExceptionWKBParser("Caught unhandled exception parsing MultiPolygon: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) + + + def parsePoint(self, reader, dimensions): + x = reader.unpack_double() + y = reader.unpack_double() + + if dimensions == 3: + reader.unpack_double() + + xx = int(round(x)) + # flip upside down + yy = self.tileSize - int(round(y)) + + if self.first or xx - self.lastX != 0 or yy - self.lastY != 0: + self.coordinates.append(xx - self.lastX) + self.coordinates.append(yy - self.lastY) + self.num_points += 1 + else: + self.dropped += 1; + + self.first = False + self.lastX = xx + self.lastY = yy + + + def parsePolygon(self, reader, dimensions): + self.isPoint = False; + try: + num_rings = reader.unpack_uint32() + + for _ in xrange(0,num_rings): + self.parseLinearRing(reader,dimensions) + + self.isPoly = True + + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + raise ExceptionWKBParser("Caught unhandled exception parsing Polygon: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) + + def parseLinearRing(self, reader, dimensions): + self.isPoint = False; + try: + num_points = reader.unpack_uint32() + + self.num_points = 0; + + # skip the last point + for _ in xrange(0,num_points-1): + self.parsePoint(reader,dimensions) + + # skip the last point + reader.unpack_double() + reader.unpack_double() + if dimensions == 3: + reader.unpack_double() + + self.index.append(self.num_points) + + self.first = True + + except: + _, value, tb = sys.exc_info()[:3] + error = ("%s , %s \n" % (type, value)) + for bits in traceback.format_exception(type,value,tb): + error = error + bits + '\n' + del tb + raise ExceptionWKBParser("Caught unhandled exception parsing LinearRing: %s \n"\ + "Traceback: %s\n" % (str(self._current_string),error)) \ No newline at end of file diff --git a/TileStache/Goodies/VecTiles/Mapbox/__init__.py b/TileStache/Goodies/VecTiles/Mapbox/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/TileStache/Goodies/VecTiles/Mapbox/vector_tile.proto b/TileStache/Goodies/VecTiles/Mapbox/vector_tile.proto new file mode 100644 index 00000000..110f37c3 --- /dev/null +++ b/TileStache/Goodies/VecTiles/Mapbox/vector_tile.proto @@ -0,0 +1,92 @@ +// Protocol Version 1 + +package mapnik.vector; + +option optimize_for = LITE_RUNTIME; + +message tile { + enum GeomType { + Unknown = 0; + Point = 1; + LineString = 2; + Polygon = 3; + } + + // Variant type encoding + message value { + // Exactly one of these values may be present in a valid message + optional string string_value = 1; + optional float float_value = 2; + optional double double_value = 3; + optional int64 int_value = 4; + optional uint64 uint_value = 5; + optional sint64 sint_value = 6; + optional bool bool_value = 7; + + extensions 8 to max; + } + + message feature { + optional uint64 id = 1; + + // Tags of this feature. Even numbered values refer to the nth + // value in the keys list on the tile message, odd numbered + // values refer to the nth value in the values list on the tile + // message. + repeated uint32 tags = 2 [ packed = true ]; + + // The type of geometry stored in this feature. + optional GeomType type = 3 [ default = Unknown ]; + + // Contains a stream of commands and parameters (vertices). The + // repeat count is shifted to the left by 3 bits. This means + // that the command has 3 bits (0-7). The repeat count + // indicates how often this command is to be repeated. Defined + // commands are: + // - MoveTo: 1 (2 parameters follow) + // - LineTo: 2 (2 parameters follow) + // - ClosePath: 7 (no parameters follow) + // + // Ex.: MoveTo(3, 6), LineTo(8, 12), LineTo(20, 34), ClosePath + // Encoded as: [ 9 3 6 18 5 6 12 22 15 ] + // == command type 7 (ClosePath), length 1 + // ===== relative LineTo(+12, +22) == LineTo(20, 34) + // === relative LineTo(+5, +6) == LineTo(8, 12) + // == [00010 010] = command type 2 (LineTo), length 2 + // === relative MoveTo(+3, +6) + // == [00001 001] = command type 1 (MoveTo), length 1 + // Commands are encoded as uint32 varints, vertex parameters are + // encoded as sint32 varints (zigzag). Vertex parameters are + // also encoded as deltas to the previous position. The original + // position is (0,0) + repeated uint32 geometry = 4 [ packed = true ]; + } + + message layer { + // Any compliant implementation must first read the version + // number encoded in this message and choose the correct + // implementation for this version number before proceeding to + // decode other parts of this message. + required uint32 version = 15 [ default = 1 ]; + + required string name = 1; + + // The actual features in this tile. + repeated feature features = 2; + + // Dictionary encoding for keys + repeated string keys = 3; + + // Dictionary encoding for values + repeated value values = 4; + + // The bounding box in this tile spans from 0..4095 units + optional uint32 extent = 5 [ default = 4096 ]; + + extensions 16 to max; + } + + repeated layer layers = 3; + + extensions 16 to 8191; +} diff --git a/TileStache/Goodies/VecTiles/Mapbox/vector_tile_pb2.py b/TileStache/Goodies/VecTiles/Mapbox/vector_tile_pb2.py new file mode 100644 index 00000000..6c1bee2f --- /dev/null +++ b/TileStache/Goodies/VecTiles/Mapbox/vector_tile_pb2.py @@ -0,0 +1,298 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: vector_tile.proto + +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='vector_tile.proto', + package='mapnik.vector', + serialized_pb='\n\x11vector_tile.proto\x12\rmapnik.vector\"\xc5\x04\n\x04tile\x12)\n\x06layers\x18\x03 \x03(\x0b\x32\x19.mapnik.vector.tile.layer\x1a\xa1\x01\n\x05value\x12\x14\n\x0cstring_value\x18\x01 \x01(\t\x12\x13\n\x0b\x66loat_value\x18\x02 \x01(\x02\x12\x14\n\x0c\x64ouble_value\x18\x03 \x01(\x01\x12\x11\n\tint_value\x18\x04 \x01(\x03\x12\x12\n\nuint_value\x18\x05 \x01(\x04\x12\x12\n\nsint_value\x18\x06 \x01(\x12\x12\x12\n\nbool_value\x18\x07 \x01(\x08*\x08\x08\x08\x10\x80\x80\x80\x80\x02\x1ar\n\x07\x66\x65\x61ture\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x10\n\x04tags\x18\x02 \x03(\rB\x02\x10\x01\x12\x33\n\x04type\x18\x03 \x01(\x0e\x32\x1c.mapnik.vector.tile.GeomType:\x07Unknown\x12\x14\n\x08geometry\x18\x04 \x03(\rB\x02\x10\x01\x1a\xb1\x01\n\x05layer\x12\x12\n\x07version\x18\x0f \x02(\r:\x01\x31\x12\x0c\n\x04name\x18\x01 \x02(\t\x12-\n\x08\x66\x65\x61tures\x18\x02 \x03(\x0b\x32\x1b.mapnik.vector.tile.feature\x12\x0c\n\x04keys\x18\x03 \x03(\t\x12)\n\x06values\x18\x04 \x03(\x0b\x32\x19.mapnik.vector.tile.value\x12\x14\n\x06\x65xtent\x18\x05 \x01(\r:\x04\x34\x30\x39\x36*\x08\x08\x10\x10\x80\x80\x80\x80\x02\"?\n\x08GeomType\x12\x0b\n\x07Unknown\x10\x00\x12\t\n\x05Point\x10\x01\x12\x0e\n\nLineString\x10\x02\x12\x0b\n\x07Polygon\x10\x03*\x05\x08\x10\x10\x80@B\x02H\x03') + + + +_TILE_GEOMTYPE = _descriptor.EnumDescriptor( + name='GeomType', + full_name='mapnik.vector.tile.GeomType', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='Unknown', index=0, number=0, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='Point', index=1, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='LineString', index=2, number=2, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='Polygon', index=3, number=3, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=548, + serialized_end=611, +) + + +_TILE_VALUE = _descriptor.Descriptor( + name='value', + full_name='mapnik.vector.tile.value', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='string_value', full_name='mapnik.vector.tile.value.string_value', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='float_value', full_name='mapnik.vector.tile.value.float_value', index=1, + number=2, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='double_value', full_name='mapnik.vector.tile.value.double_value', index=2, + number=3, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='int_value', full_name='mapnik.vector.tile.value.int_value', index=3, + number=4, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='uint_value', full_name='mapnik.vector.tile.value.uint_value', index=4, + number=5, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='sint_value', full_name='mapnik.vector.tile.value.sint_value', index=5, + number=6, type=18, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='bool_value', full_name='mapnik.vector.tile.value.bool_value', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=True, + extension_ranges=[(8, 536870912), ], + serialized_start=89, + serialized_end=250, +) + +_TILE_FEATURE = _descriptor.Descriptor( + name='feature', + full_name='mapnik.vector.tile.feature', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='mapnik.vector.tile.feature.id', index=0, + number=1, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='tags', full_name='mapnik.vector.tile.feature.tags', index=1, + number=2, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), + _descriptor.FieldDescriptor( + name='type', full_name='mapnik.vector.tile.feature.type', index=2, + number=3, type=14, cpp_type=8, label=1, + has_default_value=True, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='geometry', full_name='mapnik.vector.tile.feature.geometry', index=3, + number=4, type=13, cpp_type=3, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=252, + serialized_end=366, +) + +_TILE_LAYER = _descriptor.Descriptor( + name='layer', + full_name='mapnik.vector.tile.layer', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='version', full_name='mapnik.vector.tile.layer.version', index=0, + number=15, type=13, cpp_type=3, label=2, + has_default_value=True, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='name', full_name='mapnik.vector.tile.layer.name', index=1, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='features', full_name='mapnik.vector.tile.layer.features', index=2, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='keys', full_name='mapnik.vector.tile.layer.keys', index=3, + number=3, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='values', full_name='mapnik.vector.tile.layer.values', index=4, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='extent', full_name='mapnik.vector.tile.layer.extent', index=5, + number=5, type=13, cpp_type=3, label=1, + has_default_value=True, default_value=4096, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=True, + extension_ranges=[(16, 536870912), ], + serialized_start=369, + serialized_end=546, +) + +_TILE = _descriptor.Descriptor( + name='tile', + full_name='mapnik.vector.tile', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='layers', full_name='mapnik.vector.tile.layers', index=0, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_TILE_VALUE, _TILE_FEATURE, _TILE_LAYER, ], + enum_types=[ + _TILE_GEOMTYPE, + ], + options=None, + is_extendable=True, + extension_ranges=[(16, 8192), ], + serialized_start=37, + serialized_end=618, +) + +_TILE_VALUE.containing_type = _TILE; +_TILE_FEATURE.fields_by_name['type'].enum_type = _TILE_GEOMTYPE +_TILE_FEATURE.containing_type = _TILE; +_TILE_LAYER.fields_by_name['features'].message_type = _TILE_FEATURE +_TILE_LAYER.fields_by_name['values'].message_type = _TILE_VALUE +_TILE_LAYER.containing_type = _TILE; +_TILE.fields_by_name['layers'].message_type = _TILE_LAYER +_TILE_GEOMTYPE.containing_type = _TILE; +DESCRIPTOR.message_types_by_name['tile'] = _TILE + +class tile(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + + class value(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _TILE_VALUE + + # @@protoc_insertion_point(class_scope:mapnik.vector.tile.value) + + class feature(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _TILE_FEATURE + + # @@protoc_insertion_point(class_scope:mapnik.vector.tile.feature) + + class layer(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _TILE_LAYER + + # @@protoc_insertion_point(class_scope:mapnik.vector.tile.layer) + DESCRIPTOR = _TILE + + # @@protoc_insertion_point(class_scope:mapnik.vector.tile) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), 'H\003') +_TILE_FEATURE.fields_by_name['tags'].has_options = True +_TILE_FEATURE.fields_by_name['tags']._options = _descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001') +_TILE_FEATURE.fields_by_name['geometry'].has_options = True +_TILE_FEATURE.fields_by_name['geometry']._options = _descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001') +# @@protoc_insertion_point(module_scope) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py new file mode 100644 index 00000000..8ede2559 --- /dev/null +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -0,0 +1,141 @@ +import types +from Mapbox import vector_tile_pb2 +from Mapbox.GeomEncoder import GeomEncoder +from shapely.wkb import loads + +from TileStache.Core import KnownUnknown +import re +import logging +import struct + +# coordindates are scaled to this range within tile +extents = 4096 + +def decode(file): + ''' Stub function to decode a mapbox vector tile file into a list of features. + + Not currently implemented, modeled on geojson.decode(). + ''' + raise NotImplementedError('mapbox.decode() not yet written') + +def encode(file, features, coord, layer_name): + tile = VectorTile(extents) + + tile.addFeatures(features, coord, extents, layer_name) + + # tile.complete() + + data = tile.tile.SerializeToString() + logging.info(tile.tile) + file.write(struct.pack(">I", len(data))) + file.write(data) + +def merge(file, features, coord): + ''' Retrieve a list of GeoJSON tile responses and merge them into one. + + get_tiles() retrieves data and performs basic integrity checks. + ''' + tile = VectorTile(extents) + + for feature in features: + name = feature['name'] + feats= feature['features'] + tile.addFeatures(feats, coord, extents, name) + + data = tile.tile.SerializeToString() + logging.info(tile.tile) + file.write(struct.pack(">I", len(data))) + file.write(data) + +class VectorTile: + """ + """ + def __init__(self, extents, layer_name=""): + self.geomencoder = GeomEncoder(extents) + + self.tile = vector_tile_pb2.tile() + self.feature_count = 0 + self.keys = [] + self.values = [] + self.pixels = [] + + + # def complete(self): + # if self.cur_key - attrib_offset > 0: + # self.tile.num_keys = self.cur_key - attrib_offset + + # if self.cur_val - attrib_offset > 0: + # self.tile.num_vals = self.cur_val - attrib_offset + + def addFeatures(self, features, coord, extents, layer_name=""): + self.layer = self.tile.layers.add() + self.layer.name = layer_name + self.layer.version = 2 + self.layer.extent = extents + for feature in features: + self.addFeature(feature, coord) + + def addFeature(self, feature, coord): + geom = self.geomencoder + + f = self.layer.features.add() + self.feature_count += 1 + f.id = self.feature_count + + self._handle_attr(self.layer, f, feature[1]) + + geom.parseGeometry(feature[0]) + + if geom.isPoint: + f.type = self.tile.Point + else: + # # empty geometry + # if len(geom.index) == 0: + # logging.debug('empty geom: %s %s' % feature[1]) + # return + + if geom.isPoly: + f.type = self.tile.Polygon + else: + f.type = self.tile.LineString + + # add coordinate index list (coordinates per geometry) + # feature.indices.extend(geom.index) + + # add indice count (number of geometries) + # if len(feature.indices) > 1: + # feature.num_indices = len(feature.indices) + + # add coordinates + for coordinate in geom.coordinates: + logging.info(coordinate) + if coordinate <= 4096 and coordinate > 0: + f.geometry.append(coordinate) + + def _handle_attr(self, layer, feature, props): + for k,v in props.items(): + if k not in self.keys: + layer.keys.append(k) + self.keys.append(k) + idx = self.keys.index(k) + feature.tags.append(idx) + else: + idx = self.keys.index(k) + feature.tags.append(idx) + if v not in self.values: + if (isinstance(v,bool)): + val = layer.values.add() + val.bool_value = v + elif (isinstance(v,str)) or (isinstance(v,unicode)): + val = layer.values.add() + val.string_value = v + elif (isinstance(v,int)): + val = layer.values.add() + val.int_value = v + elif (isinstance(v,float)): + val = layer.values.add() + val.double_value = v + # else: + # raise Exception("Unknown value type: '%s'" % type(v)) + self.values.append(v) + feature.tags.append(self.values.index(v)) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index c7f90c5d..ae7af3f9 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -22,7 +22,7 @@ def connect(*args, **kwargs): raise err -from . import mvt, geojson, topojson +from . import mvt, geojson, topojson, mapbox from ...Geography import SphericalMercator from ModestMaps.Core import Point @@ -155,7 +155,7 @@ def renderTile(self, width, height, srs, coord): tolerance = self.simplify * tolerances[coord.zoom] if coord.zoom < self.simplify_until else None - return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip) + return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name()) def getTypeByExtension(self, extension): ''' Get mime-type and format by file extension, one of "mvt", "json" or "topojson". @@ -169,6 +169,9 @@ def getTypeByExtension(self, extension): elif extension.lower() == 'topojson': return 'application/json', 'TopoJSON' + elif extension.lower() == 'mapbox': + return 'image/png', 'Mapbox' + else: raise ValueError(extension) @@ -210,6 +213,9 @@ def getTypeByExtension(self, extension): elif extension.lower() == 'topojson': return 'application/json', 'TopoJSON' + elif extension.lower() == 'mapbox': + return 'image/png', 'Mapbox' + else: raise ValueError(extension) @@ -232,7 +238,7 @@ def __exit__(self, type, value, traceback): class Response: ''' ''' - def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip): + def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip, coord, layer_name): ''' Create a new response object with Postgres connection info and a query. bounds argument is a 4-tuple with (xmin, ymin, xmax, ymax). @@ -241,33 +247,19 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli self.bounds = bounds self.zoom = zoom self.clip = clip + self.coord= coord + self.layer_name = layer_name bbox = 'ST_MakeBox2D(ST_MakePoint(%.2f, %.2f), ST_MakePoint(%.2f, %.2f))' % bounds geo_query = build_query(srid, subquery, columns, bbox, tolerance, True, clip) merc_query = build_query(srid, subquery, columns, bbox, tolerance, False, clip) - self.query = dict(TopoJSON=geo_query, JSON=geo_query, MVT=merc_query) - + mapbox_query = build_query(srid, subquery, columns, bbox, tolerance, False, clip, mapbox.extents) + self.query = dict(TopoJSON=geo_query, JSON=geo_query, MVT=merc_query, Mapbox=mapbox_query) + def save(self, out, format): ''' ''' - with Connection(self.dbinfo) as db: - db.execute(self.query[format]) - - features = [] - - for row in db.fetchall(): - if row['__geometry__'] is None: - continue - - wkb = bytes(row['__geometry__']) - prop = dict([(k, v) for (k, v) in row.items() - if k not in ('__geometry__', '__id__')]) - - if '__id__' in row: - features.append((wkb, prop, row['__id__'])) - - else: - features.append((wkb, prop)) + features = get_features(self.dbinfo, self.query[format]) if format == 'MVT': mvt.encode(out, features) @@ -279,7 +271,10 @@ def save(self, out, format): ll = SphericalMercator().projLocation(Point(*self.bounds[0:2])) ur = SphericalMercator().projLocation(Point(*self.bounds[2:4])) topojson.encode(out, features, (ll.lon, ll.lat, ur.lon, ur.lat), self.clip) - + + elif format == 'Mapbox': + mapbox.encode(out, features, self.coord, self.layer_name) + else: raise ValueError(format) @@ -302,7 +297,10 @@ def save(self, out, format): ll = SphericalMercator().projLocation(Point(*self.bounds[0:2])) ur = SphericalMercator().projLocation(Point(*self.bounds[2:4])) topojson.encode(out, [], (ll.lon, ll.lat, ur.lon, ur.lat), False) - + + elif format == 'Mapbox': + mapbox.encode(out, [], None) + else: raise ValueError(format) @@ -325,6 +323,16 @@ def save(self, out, format): elif format == 'JSON': geojson.merge(out, self.names, self.config, self.coord) + elif format == 'Mapbox': + features = [] + layers = [self.config.layers[name] for name in self.names] + for layer in layers: + width, height = layer.dim, layer.dim + tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) + if isinstance(tile,EmptyResponse): continue + features.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["Mapbox"])}) + mapbox.merge(out, features, self.coord) + else: raise ValueError(format) @@ -357,8 +365,30 @@ def query_columns(dbinfo, srid, subquery, bounds): column_names = set(row.keys()) return column_names + +def get_features(dbinfo, query): + with Connection(dbinfo) as db: + db.execute(query) + + features = [] -def build_query(srid, subquery, subcolumns, bbox, tolerance, is_geo, is_clipped): + for row in db.fetchall(): + if row['__geometry__'] is None: + continue + + wkb = bytes(row['__geometry__']) + prop = dict([(k, v) for (k, v) in row.items() + if k not in ('__geometry__', '__id__')]) + + if '__id__' in row: + features.append((wkb, prop, row['__id__'])) + + else: + features.append((wkb, prop)) + + return features + +def build_query(srid, subquery, subcolumns, bbox, tolerance, is_geo, is_clipped, scale=None): ''' Build and return an PostGIS query. ''' bbox = 'ST_SetSRID(%s, %d)' % (bbox, srid) @@ -372,6 +402,9 @@ def build_query(srid, subquery, subcolumns, bbox, tolerance, is_geo, is_clipped) if is_geo: geom = 'ST_Transform(%s, 4326)' % geom + + if scale: + geom = 'ST_TransScale(%s, -ST_XMin(%s), -ST_YMin(%s), (%d / (ST_XMax(%s) - ST_XMin(%s))), (%d / (ST_YMax(%s) - ST_YMin(%s))))' % (geom, bbox, bbox, scale, bbox, bbox, scale, bbox, bbox) subquery = subquery.replace('!bbox!', bbox) columns = ['q."%s"' % c for c in subcolumns if c not in ('__geometry__', )] diff --git a/setup.py b/setup.py index 8004e5db..2be53354 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,8 @@ def is_installed(name): 'TileStache.Goodies', 'TileStache.Goodies.Caches', 'TileStache.Goodies.Providers', - 'TileStache.Goodies.VecTiles'], + 'TileStache.Goodies.VecTiles', + 'TileStache.Goodies.VecTiles/Mapbox'], scripts=['scripts/tilestache-compose.py', 'scripts/tilestache-seed.py', 'scripts/tilestache-clean.py', 'scripts/tilestache-server.py', 'scripts/tilestache-render.py', 'scripts/tilestache-list.py'], data_files=[('share/tilestache', ['TileStache/Goodies/Providers/DejaVuSansMono-alphanumeric.ttf'])], download_url='http://tilestache.org/download/TileStache-%(version)s.tar.gz' % locals(), From a1d6f11bb15ce57cd47e33b567986e2b3af7a2d1 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Mon, 28 Apr 2014 17:45:01 -0400 Subject: [PATCH 021/344] cleaning up - naming and removing commented lines out --- TileStache/Goodies/VecTiles/mapbox.py | 20 +++----------------- TileStache/Goodies/VecTiles/server.py | 6 +++--- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index 8ede2559..3c7ef48e 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -23,27 +23,21 @@ def encode(file, features, coord, layer_name): tile.addFeatures(features, coord, extents, layer_name) - # tile.complete() - data = tile.tile.SerializeToString() - logging.info(tile.tile) file.write(struct.pack(">I", len(data))) file.write(data) -def merge(file, features, coord): +def merge(file, feature_layers, coord): ''' Retrieve a list of GeoJSON tile responses and merge them into one. get_tiles() retrieves data and performs basic integrity checks. ''' tile = VectorTile(extents) - for feature in features: - name = feature['name'] - feats= feature['features'] - tile.addFeatures(feats, coord, extents, name) + for layer in feature_layers: + tile.addFeatures(layer['features'], coord, extents, layer['name']) data = tile.tile.SerializeToString() - logging.info(tile.tile) file.write(struct.pack(">I", len(data))) file.write(data) @@ -60,13 +54,6 @@ def __init__(self, extents, layer_name=""): self.pixels = [] - # def complete(self): - # if self.cur_key - attrib_offset > 0: - # self.tile.num_keys = self.cur_key - attrib_offset - - # if self.cur_val - attrib_offset > 0: - # self.tile.num_vals = self.cur_val - attrib_offset - def addFeatures(self, features, coord, extents, layer_name=""): self.layer = self.tile.layers.add() self.layer.name = layer_name @@ -108,7 +95,6 @@ def addFeature(self, feature, coord): # add coordinates for coordinate in geom.coordinates: - logging.info(coordinate) if coordinate <= 4096 and coordinate > 0: f.geometry.append(coordinate) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index ae7af3f9..459192cc 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -324,14 +324,14 @@ def save(self, out, format): geojson.merge(out, self.names, self.config, self.coord) elif format == 'Mapbox': - features = [] + feature_layers = [] layers = [self.config.layers[name] for name in self.names] for layer in layers: width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - features.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["Mapbox"])}) - mapbox.merge(out, features, self.coord) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["Mapbox"])}) + mapbox.merge(out, feature_layers, self.coord) else: raise ValueError(format) From e85474b1e6f1211e99e5d9cabbdf6cfe7085ca78 Mon Sep 17 00:00:00 2001 From: Ragi Yaser Burhum Date: Tue, 29 Apr 2014 19:36:34 -0700 Subject: [PATCH 022/344] Fix Travis build A few fixes: - Removed UbuntuGIS-unstable since Travis now ships with PostGIS and PostgreSQL - The latest version of pip is more strict and since ModestMaps is not hosted by Pypi, it needs the --allow-external ModestMaps --allow-unverified ModestMaps flags - The recommended way to setup databases in Travis is to use the before_script section. --- .travis.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f5165075..c4c1571d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,20 +10,22 @@ virtualenv: system_site_packages: true before_install: - - sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable +# - sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable - sudo apt-get update -qq - - sudo apt-get install -qq postgis gdal-bin libgdal-dev libgdal1 libgdal1-dev libpq-dev memcached python-pip + - sudo apt-get install -qq gdal-bin memcached python-pip - sudo apt-get install -qq python-nose python-imaging python-memcache python-gdal - sudo apt-get install -qq python-coverage python-werkzeug python-psycopg2 - ogrinfo --version - ogrinfo --formats + +before_script: - sudo -u postgres psql -c "drop database if exists test_tilestache" - sudo -u postgres psql -c "create database test_tilestache" - sudo -u postgres psql -c "create extension postgis" -d test_tilestache - sudo -u postgres ogr2ogr -nlt MULTIPOLYGON -f "PostgreSQL" PG:"user=postgres dbname=test_tilestache" ./examples/sample_data/world_merc.shp install: - - sudo pip install -r requirements.txt --use-mirrors + - sudo pip install -r requirements.txt --use-mirrors --allow-external ModestMaps --allow-unverified ModestMaps script: - nosetests -v --with-coverage --cover-package TileStache From 624318a4997bcb0c7082eac14784da1a90bd3a69 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Thu, 1 May 2014 11:13:38 -0400 Subject: [PATCH 023/344] Manual zig zag encoding + geometry encoding (first pass) --- TileStache/Goodies/VecTiles/mapbox.py | 96 +++++++++++++++++++-------- 1 file changed, 68 insertions(+), 28 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index 3c7ef48e..e6054b37 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -3,6 +3,8 @@ from Mapbox.GeomEncoder import GeomEncoder from shapely.wkb import loads +from math import floor, fabs + from TileStache.Core import KnownUnknown import re import logging @@ -10,6 +12,9 @@ # coordindates are scaled to this range within tile extents = 4096 +cmd_bits = 3 +path_multiplier = 16 +tolerance = 0 def decode(file): ''' Stub function to decode a mapbox vector tile file into a list of features. @@ -46,57 +51,92 @@ class VectorTile: """ def __init__(self, extents, layer_name=""): self.geomencoder = GeomEncoder(extents) - - self.tile = vector_tile_pb2.tile() - self.feature_count = 0 - self.keys = [] - self.values = [] - self.pixels = [] - + self.tile = vector_tile_pb2.tile() def addFeatures(self, features, coord, extents, layer_name=""): self.layer = self.tile.layers.add() self.layer.name = layer_name self.layer.version = 2 self.layer.extent = extents + self.feature_count = 0 + self.keys = [] + self.values = [] + self.pixels = [] for feature in features: self.addFeature(feature, coord) def addFeature(self, feature, coord): geom = self.geomencoder - + x_ = coord.column + y_ = coord.row + cmd= 1 + skipped_last = False + f = self.layer.features.add() self.feature_count += 1 f.id = self.feature_count - + f.type = self.tile.Point if geom.isPoint else (self.tile.Polygon if geom.isPoly else self.tile.LineString) + self._handle_attr(self.layer, f, feature[1]) geom.parseGeometry(feature[0]) + coordinates = [] + for coords in self._chunker(geom.coordinates,2): + coordinates.append((coords[0], coords[1])) + length = geom.num_points + + f.geometry.append(self._encode_cmd_length(1, length)) + + it = 0 + cmd= 1 if geom.isPoint else 2 # TODO: figure out if cmd can change within a feature geom + + for coordinate in coordinates: + x,y = coordinate[0],coordinate[1] + + cur_x = int(floor((x * path_multiplier) + 0.5)) + cur_y = int(floor((y * path_multiplier) + 0.5)) - if geom.isPoint: - f.type = self.tile.Point - else: - # # empty geometry - # if len(geom.index) == 0: - # logging.debug('empty geom: %s %s' % feature[1]) - # return + if skipped_last and cmd == 1: + self._handle_skipped_last(f, cur_x, cur_y, x_, y_) - if geom.isPoly: - f.type = self.tile.Polygon + dx = cur_x - x + dy = cur_y - y + + sharp_turn_ahead = False + + if (it+2 <= len(coordinates)): + next_coord = coordinates[it+1] + next_x, next_y = next_coord[0], next_coord[1] + next_dx = fabs(cur_x - int(floor((next_x * path_multiplier) + 0.5))) + next_dy = fabs(cur_y - int(floor((next_y * path_multiplier) + 0.5))) + if ((next_dx == 0 and next_dy >= tolerance) or (next_dy == 0 and next_dx >= tolerance)): + sharp_turn_ahead = True + + + if (sharp_turn_ahead or fabs(dx) >= tolerance or fabs(dy) >= tolerance): + f.geometry.append((dx << 1) ^ (dx >> 31)) + f.geometry.append((dy << 1) ^ (dy >> 31)) + x_ = cur_x + y_ = cur_y + skipped_last = False else: - f.type = self.tile.LineString + skipped_last = True + it = it+1 + + f.geometry.append(self._encode_cmd_length(7, length)) - # add coordinate index list (coordinates per geometry) - # feature.indices.extend(geom.index) + def _encode_cmd_length(self, cmd, length): + # cmd: 1 (MOVE_TO) + # cmd: 2 (LINE_TO) + # cmd: 7 (CLOSE_PATH) + return (length << cmd_bits) | (cmd & ((1 << cmd_bits) - 1)) - # add indice count (number of geometries) - # if len(feature.indices) > 1: - # feature.num_indices = len(feature.indices) + def _chunker(self, seq, size): + return (seq[pos:pos + size] for pos in xrange(0, len(seq), size)) - # add coordinates - for coordinate in geom.coordinates: - if coordinate <= 4096 and coordinate > 0: - f.geometry.append(coordinate) + def _handle_skipped_last(self, f, cur_x, cur_y, x_, y_): + # TODO + return True def _handle_attr(self, layer, feature, props): for k,v in props.items(): From 93d345970f347577d5fda3d898e29fa7dfc588c9 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Fri, 2 May 2014 11:58:39 -0400 Subject: [PATCH 024/344] re-writing the encoding with while loops. handle unicode encoding str values. --- TileStache/Goodies/VecTiles/mapbox.py | 155 ++++++++++++++++++-------- 1 file changed, 111 insertions(+), 44 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index e6054b37..bd2c97d0 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -16,6 +16,10 @@ path_multiplier = 16 tolerance = 0 +CMD_MOVE_TO = 1 +CMD_LINE_TO = 2 +CMD_SEG_END = 7 + def decode(file): ''' Stub function to decode a mapbox vector tile file into a list of features. @@ -69,8 +73,15 @@ def addFeature(self, feature, coord): geom = self.geomencoder x_ = coord.column y_ = coord.row - cmd= 1 + cmd= -1 + cmd_idx = -1 + vtx_cmd = -1 + prev_cmd= -1 + + skipped_index = -1 skipped_last = False + cur_x = 0 + cur_y = 0 f = self.layer.features.add() self.feature_count += 1 @@ -83,47 +94,95 @@ def addFeature(self, feature, coord): coordinates = [] for coords in self._chunker(geom.coordinates,2): coordinates.append((coords[0], coords[1])) - length = geom.num_points - - f.geometry.append(self._encode_cmd_length(1, length)) - + it = 0 - cmd= 1 if geom.isPoint else 2 # TODO: figure out if cmd can change within a feature geom - - for coordinate in coordinates: - x,y = coordinate[0],coordinate[1] - - cur_x = int(floor((x * path_multiplier) + 0.5)) - cur_y = int(floor((y * path_multiplier) + 0.5)) - - if skipped_last and cmd == 1: - self._handle_skipped_last(f, cur_x, cur_y, x_, y_) - - dx = cur_x - x - dy = cur_y - y - - sharp_turn_ahead = False - - if (it+2 <= len(coordinates)): - next_coord = coordinates[it+1] - next_x, next_y = next_coord[0], next_coord[1] - next_dx = fabs(cur_x - int(floor((next_x * path_multiplier) + 0.5))) - next_dy = fabs(cur_y - int(floor((next_y * path_multiplier) + 0.5))) - if ((next_dx == 0 and next_dy >= tolerance) or (next_dy == 0 and next_dx >= tolerance)): - sharp_turn_ahead = True - - - if (sharp_turn_ahead or fabs(dx) >= tolerance or fabs(dy) >= tolerance): - f.geometry.append((dx << 1) ^ (dx >> 31)) - f.geometry.append((dy << 1) ^ (dy >> 31)) - x_ = cur_x - y_ = cur_y - skipped_last = False + length = 0 + + while (True): + if it >= len(coordinates): + break; + + x,y = coordinates[it][0],coordinates[it][1] + + vtx_cmd= self._get_cmd_type(f.type, geom.num_points, it) + + if vtx_cmd != cmd: + if (cmd_idx >= 0): + f.geometry.__setitem__(cmd_idx, self._encode_cmd_length(cmd, length)) + + cmd = vtx_cmd + length = 0 + cmd_idx = len(f.geometry) + f.geometry.append(0) #placeholder added in first pass + + if (vtx_cmd == CMD_MOVE_TO or vtx_cmd == CMD_LINE_TO): + if cmd == CMD_MOVE_TO and skipped_last and skipped_index >1: + self._handle_skipped_last(f, skipped_index, cur_x, cur_y, x_, y_) + + # Compute delta to the previous coordinate. + cur_x = int(floor((x * path_multiplier) + 0.5)) + cur_y = int(floor((y * path_multiplier) + 0.5)) + + dx = cur_x - x_ + dy = cur_y - y_ + + sharp_turn_ahead = False + + if (it+2 <= len(coordinates)): + next_coord = coordinates[it+1] + if self._get_cmd_type(f.type, geom.num_points, it) == CMD_LINE_TO: + next_x, next_y = next_coord[0], next_coord[1] + next_dx = fabs(cur_x - int(floor((next_x * path_multiplier) + 0.5))) + next_dy = fabs(cur_y - int(floor((next_y * path_multiplier) + 0.5))) + if ((next_dx == 0 and next_dy >= tolerance) or (next_dy == 0 and next_dx >= tolerance)): + sharp_turn_ahead = True + + # Keep all move_to commands, but omit other movements that are + # not >= the tolerance threshold and should be considered no-ops. + # NOTE: length == 0 indicates the command has changed and will + # preserve any non duplicate move_to or line_to + if length == 0 or sharp_turn_ahead or fabs(dx) >= tolerance or fabs(dy) >= tolerance: + # Manual zigzag encoding. + f.geometry.append((dx << 1) ^ (dx >> 31)) + f.geometry.append((dy << 1) ^ (dy >> 31)) + x_ = cur_x + y_ = cur_y + skipped_last = False + length = length + 1 + else: + skipped_last = True + skipped_index = len(f.geometry) + elif vtx_cmd == CMD_SEG_END: + if prev_cmd != CMD_SEG_END: + length = length + 1 else: - skipped_last = True - it = it+1 - - f.geometry.append(self._encode_cmd_length(7, length)) + raise Exception("Unknown command type: '%s'" % vtx_cmd) + + it = it + 1 + prev_cmd = cmd + + # at least one vertex + cmd/length + if (skipped_last and skipped_index > 1): + # if we skipped previous vertex we just update it to the last one here. + handle_skipped_last(f, skipped_index, cur_x, cur_y, x_, y_) + + # Update the last length/command value. + if (cmd_idx >= 0): + f.geometry.__setitem__(cmd_idx, self._encode_cmd_length(cmd, length)) + + + # TODO: figure out if cmd can change within a feature geom + def _get_cmd_type(self, gtype, gpoints, glen): + cmd_type = -1 + if gtype == self.tile.Point: + cmd_type = CMD_MOVE_TO + elif gtype == self.tile.Polygon or gtype == self.tile.LineString: + cmd_type = CMD_LINE_TO + if glen==0: + cmd_type = CMD_MOVE_TO + if gtype == self.tile.Polygon and glen>=gpoints: + cmd_type = CMD_SEG_END + return cmd_type def _encode_cmd_length(self, cmd, length): # cmd: 1 (MOVE_TO) @@ -134,9 +193,17 @@ def _encode_cmd_length(self, cmd, length): def _chunker(self, seq, size): return (seq[pos:pos + size] for pos in xrange(0, len(seq), size)) - def _handle_skipped_last(self, f, cur_x, cur_y, x_, y_): - # TODO - return True + def _handle_skipped_last(self, f, skipped_index, cur_x, cur_y, x_, y_): + last_x = f.geometry[skipped_index - 2] + last_y = f.geometry[skipped_index - 1] + last_dx = ((last_x >> 1) ^ (-(last_x & 1))) + last_dy = ((last_y >> 1) ^ (-(last_y & 1))) + dx = cur_x - x_ + last_dx + dy = cur_y - y_ + last_dy + x_ = cur_x + y_ = cur_y + f.geometry.__setitem__(skipped_index - 2, ((dx << 1) ^ (dx >> 31))) + f.geometry.__setitem__(skipped_index - 1, ((dy << 1) ^ (dy >> 31))) def _handle_attr(self, layer, feature, props): for k,v in props.items(): @@ -154,7 +221,7 @@ def _handle_attr(self, layer, feature, props): val.bool_value = v elif (isinstance(v,str)) or (isinstance(v,unicode)): val = layer.values.add() - val.string_value = v + val.string_value = unicode(v,'utf8') elif (isinstance(v,int)): val = layer.values.add() val.int_value = v From a21d6bdd6fd2647116ae6c77abf8046877881088 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Fri, 2 May 2014 14:54:17 -0400 Subject: [PATCH 025/344] making coordinates an array of objects that have vertex2d info (x,y,cmd) --- TileStache/Goodies/VecTiles/mapbox.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index bd2c97d0..8d7567ab 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -92,8 +92,12 @@ def addFeature(self, feature, coord): geom.parseGeometry(feature[0]) coordinates = [] - for coords in self._chunker(geom.coordinates,2): - coordinates.append((coords[0], coords[1])) + points = self._chunker(geom.coordinates,2) # x,y corodinates grouped as one point. + for i, coords in enumerate(points): + coordinates.append({ + 'x': coords[0], + 'y': coords[1], + 'cmd': self._get_cmd_type(f.type, i, len(points))}) it = 0 length = 0 @@ -102,9 +106,7 @@ def addFeature(self, feature, coord): if it >= len(coordinates): break; - x,y = coordinates[it][0],coordinates[it][1] - - vtx_cmd= self._get_cmd_type(f.type, geom.num_points, it) + x,y,vtx_cmd = coordinates[it]['x'],coordinates[it]['y'],coordinates[it]['cmd'] if vtx_cmd != cmd: if (cmd_idx >= 0): @@ -130,8 +132,8 @@ def addFeature(self, feature, coord): if (it+2 <= len(coordinates)): next_coord = coordinates[it+1] - if self._get_cmd_type(f.type, geom.num_points, it) == CMD_LINE_TO: - next_x, next_y = next_coord[0], next_coord[1] + if next_coord['cmd'] == CMD_LINE_TO: + next_x, next_y = next_coord['x'], next_coord['y'] next_dx = fabs(cur_x - int(floor((next_x * path_multiplier) + 0.5))) next_dy = fabs(cur_y - int(floor((next_y * path_multiplier) + 0.5))) if ((next_dx == 0 and next_dy >= tolerance) or (next_dy == 0 and next_dx >= tolerance)): @@ -155,6 +157,7 @@ def addFeature(self, feature, coord): elif vtx_cmd == CMD_SEG_END: if prev_cmd != CMD_SEG_END: length = length + 1 + break; else: raise Exception("Unknown command type: '%s'" % vtx_cmd) @@ -172,15 +175,15 @@ def addFeature(self, feature, coord): # TODO: figure out if cmd can change within a feature geom - def _get_cmd_type(self, gtype, gpoints, glen): + def _get_cmd_type(self, gtype, i, points): cmd_type = -1 if gtype == self.tile.Point: cmd_type = CMD_MOVE_TO elif gtype == self.tile.Polygon or gtype == self.tile.LineString: cmd_type = CMD_LINE_TO - if glen==0: + if i==0: cmd_type = CMD_MOVE_TO - if gtype == self.tile.Polygon and glen>=gpoints: + if gtype == self.tile.Polygon and i+1==points: cmd_type = CMD_SEG_END return cmd_type @@ -191,7 +194,7 @@ def _encode_cmd_length(self, cmd, length): return (length << cmd_bits) | (cmd & ((1 << cmd_bits) - 1)) def _chunker(self, seq, size): - return (seq[pos:pos + size] for pos in xrange(0, len(seq), size)) + return [seq[pos:pos + size] for pos in xrange(0, len(seq), size)] def _handle_skipped_last(self, f, skipped_index, cur_x, cur_y, x_, y_): last_x = f.geometry[skipped_index - 2] From 039f954d05ed937f9278b90a9c3a91b27b48b2b4 Mon Sep 17 00:00:00 2001 From: Ragi Yaser Burhum Date: Fri, 2 May 2014 17:52:42 -0700 Subject: [PATCH 026/344] Fix import errors for case-insensitive filesystems If you have a case-insensitive filesystem (or even a VM running on top of a case-insensitive filesystem) you will get import errors. For example, TileStache's Mapnik.py has an "import mapnik" call. Since the filesystem is not distinguishing mapnik.py from Mapnik.py it re-imports itself and then fails. We can fix this by enabling absolute imports. --- TileStache/Mapnik.py | 6 ++++++ TileStache/Memcache.py | 6 ++++++ TileStache/Redis.py | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/TileStache/Mapnik.py b/TileStache/Mapnik.py index 24120a85..de148f3b 100644 --- a/TileStache/Mapnik.py +++ b/TileStache/Mapnik.py @@ -4,6 +4,7 @@ known as "mapnik grid". Both require Mapnik to be installed; Grid requires Mapnik 2.0.0 and above. """ +from __future__ import absolute_import from time import time from os.path import exists from thread import allocate_lock @@ -17,6 +18,11 @@ import logging import json +# We enabled absolute_import because case insensitive filesystems +# cause this file to be loaded twice (the name of this file +# conflicts with the name of the module we want to import). +# Forcing absolute imports fixes the issue. + try: import mapnik except ImportError: diff --git a/TileStache/Memcache.py b/TileStache/Memcache.py index 1a1f3322..43f54942 100644 --- a/TileStache/Memcache.py +++ b/TileStache/Memcache.py @@ -31,8 +31,14 @@ """ +from __future__ import absolute_import from time import time as _time, sleep as _sleep +# We enabled absolute_import because case insensitive filesystems +# cause this file to be loaded twice (the name of this file +# conflicts with the name of the module we want to import). +# Forcing absolute imports fixes the issue. + try: from memcache import Client except ImportError: diff --git a/TileStache/Redis.py b/TileStache/Redis.py index e7c5baa2..a8a74836 100644 --- a/TileStache/Redis.py +++ b/TileStache/Redis.py @@ -39,8 +39,13 @@ """ +from __future__ import absolute_import from time import time as _time, sleep as _sleep +# We enabled absolute_import because case insensitive filesystems +# cause this file to be loaded twice (the name of this file +# conflicts with the name of the module we want to import). +# Forcing absolute imports fixes the issue. try: import redis From c17e81d1380c3ed9a9e36ff002051a69cf245602 Mon Sep 17 00:00:00 2001 From: Ragi Yaser Burhum Date: Fri, 2 May 2014 19:51:10 -0700 Subject: [PATCH 027/344] Remove 'host' from test conn strings --- tests/vectiles_tests.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/vectiles_tests.py b/tests/vectiles_tests.py index f21a66b1..fdcf7b3d 100644 --- a/tests/vectiles_tests.py +++ b/tests/vectiles_tests.py @@ -88,7 +88,6 @@ def setUp(self): "clip": false, "dbinfo": { - "host": "localhost", "user": "postgres", "password": "", "database": "test_tilestache" @@ -109,7 +108,6 @@ def setUp(self): { "dbinfo": { - "host": "localhost", "user": "postgres", "password": "", "database": "test_tilestache" From f287203a472643ea029b980b09fbf3a4a677181a Mon Sep 17 00:00:00 2001 From: Ragi Yaser Burhum Date: Mon, 5 May 2014 15:13:59 -0700 Subject: [PATCH 028/344] Add TileStache Vagrant Machine Install Vagrant ( http://www.vagrantup.com/ ). If you execute: vagrant up You will get an Ubuntu 14.04 LTS with all the dependencies installed. To run the Tilestache tests: cd /srv/tilestache ./runtests.sh --- Vagrant/setup.sh | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ Vagrantfile | 17 +++++++++++++ 2 files changed, 81 insertions(+) create mode 100755 Vagrant/setup.sh create mode 100644 Vagrantfile diff --git a/Vagrant/setup.sh b/Vagrant/setup.sh new file mode 100755 index 00000000..6601fccd --- /dev/null +++ b/Vagrant/setup.sh @@ -0,0 +1,64 @@ +#!/bin/bash -e + +if [ -f ~/.bootstrap_complete ]; then + exit 0 +fi + +set -x + +whoami +sudo apt-get -q update +sudo apt-get -q install python-software-properties +sudo add-apt-repository ppa:mapnik/nightly-2.3 -y +sudo apt-get -q update +sudo apt-get -q install libmapnik-dev mapnik-utils python-mapnik virtualenvwrapper python-dev -y +sudo apt-get -q install gdal-bin libgdal-dev -y + +# needed to build gdal bindings separately +sudo apt-get install build-essential -y + +# create a python virtualenv +virtualenv -q ~/.virtualenvs/tilestache +source ~/.virtualenvs/tilestache/bin/activate + +# make sure it gets activated the next time we log in +echo "source ~/.virtualenvs/tilestache/bin/activate" >> ~/.bashrc + +# add system mapnik to virtualenv +ln -s /usr/lib/pymodules/python2.7/mapnik ~/.virtualenvs/tilestache/lib/python2.7/site-packages/mapnik + +# for tests +sudo apt-get -q install postgresql-9.3-postgis-2.1 memcached -y +~/.virtualenvs/tilestache/bin/pip install nose coverage python-memcached psycopg2 werkzeug +~/.virtualenvs/tilestache/bin/pip install pil --allow-external pil --allow-unverified pil + +# install basic TileStache requirements +cd /srv/tilestache/ +~/.virtualenvs/tilestache/bin/pip install -r requirements.txt --allow-external ModestMaps --allow-unverified ModestMaps + +# workaround for gdal bindings +~/.virtualenvs/tilestache/bin/pip install --no-install GDAL +cd ~/.virtualenvs/tilestache/build/GDAL +~/.virtualenvs/tilestache/bin/python setup.py build_ext --include-dirs=/usr/include/gdal/ +~/.virtualenvs/tilestache/bin/pip install --no-download GDAL + +# allow any user to connect as postgres to this test data. DO NOT USE IN PRODUCTION +sudo sed -i '1i local test_tilestache postgres trust' /etc/postgresql/9.3/main/pg_hba.conf + +sudo /etc/init.d/postgresql restart + +# add some test data +sudo -u postgres psql -c "drop database if exists test_tilestache" +sudo -u postgres psql -c "create database test_tilestache" +sudo -u postgres psql -c "create extension postgis" -d test_tilestache +sudo -u postgres ogr2ogr -nlt MULTIPOLYGON -f "PostgreSQL" PG:"user=postgres dbname=test_tilestache" ./examples/sample_data/world_merc.shp + +set +x +echo " +**************************************************************** +* Warning: your postgres security settings (pg_hba.conf) +* are not setup for production (i.e. have been set insecurely). +****************************************************************" + +# we did it. let's mark the script as complete +touch ~/.bootstrap_complete diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 00000000..caca279c --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,17 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + + config.vm.box = "Trusty64Daily" + config.vm.box_url = "http://cloud-images.ubuntu.com/vagrant/trusty/20140501/trusty-server-cloudimg-amd64-vagrant-disk1.box" + + #config.vm.network :private_network, ip: "192.168.33.10" + + config.vm.synced_folder ".", "/srv/tilestache" + + config.vm.provision :shell, :privileged => false, :inline => "sh /srv/tilestache/Vagrant/setup.sh" + +end From 61411c3c91ae602f2ee713d019a9ba0db7f74b07 Mon Sep 17 00:00:00 2001 From: Ragi Yaser Burhum Date: Mon, 5 May 2014 16:03:52 -0700 Subject: [PATCH 029/344] Fix travis memcache travis tests The memcached-related tests were failing because they could not connect to memcached. Travis has its own way of starting certain services - memcached being one of them. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index c4c1571d..d0c0d29a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,9 @@ python: virtualenv: system_site_packages: true +services: + - memcached + before_install: # - sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable - sudo apt-get update -qq From 0f33b2508330c461b8c9392d5ce4c9c80c627193 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Tue, 6 May 2014 13:08:57 -0400 Subject: [PATCH 030/344] One step closer - flipping y upside down correctly, starting the tile with 0,0 moving coords out of addFeature(), not multiplying x,y coords with path_multiplier becuase the query takes care of it. --- .../Goodies/VecTiles/Mapbox/GeomEncoder.py | 19 +++++++--------- TileStache/Goodies/VecTiles/mapbox.py | 22 ++++++++----------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/TileStache/Goodies/VecTiles/Mapbox/GeomEncoder.py b/TileStache/Goodies/VecTiles/Mapbox/GeomEncoder.py index 0bc143f0..c7a08126 100644 --- a/TileStache/Goodies/VecTiles/Mapbox/GeomEncoder.py +++ b/TileStache/Goodies/VecTiles/Mapbox/GeomEncoder.py @@ -281,12 +281,9 @@ def parsePoint(self, reader, dimensions): # flip upside down yy = self.tileSize - int(round(y)) - if self.first or xx - self.lastX != 0 or yy - self.lastY != 0: - self.coordinates.append(xx - self.lastX) - self.coordinates.append(yy - self.lastY) - self.num_points += 1 - else: - self.dropped += 1; + self.coordinates.append(xx) + self.coordinates.append(yy) + self.num_points += 1 self.first = False self.lastX = xx @@ -319,13 +316,13 @@ def parseLinearRing(self, reader, dimensions): self.num_points = 0; - # skip the last point - for _ in xrange(0,num_points-1): + # dont skip the last point + for _ in xrange(0,num_points): self.parsePoint(reader,dimensions) - # skip the last point - reader.unpack_double() - reader.unpack_double() + # dont skip the last point + # reader.unpack_double() + # reader.unpack_double() if dimensions == 3: reader.unpack_double() diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index 8d7567ab..fe052534 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -13,7 +13,6 @@ # coordindates are scaled to this range within tile extents = 4096 cmd_bits = 3 -path_multiplier = 16 tolerance = 0 CMD_MOVE_TO = 1 @@ -33,7 +32,6 @@ def encode(file, features, coord, layer_name): tile.addFeatures(features, coord, extents, layer_name) data = tile.tile.SerializeToString() - file.write(struct.pack(">I", len(data))) file.write(data) def merge(file, feature_layers, coord): @@ -47,7 +45,6 @@ def merge(file, feature_layers, coord): tile.addFeatures(layer['features'], coord, extents, layer['name']) data = tile.tile.SerializeToString() - file.write(struct.pack(">I", len(data))) file.write(data) class VectorTile: @@ -67,12 +64,12 @@ def addFeatures(self, features, coord, extents, layer_name=""): self.values = [] self.pixels = [] for feature in features: - self.addFeature(feature, coord) + self.addFeature(feature) - def addFeature(self, feature, coord): + def addFeature(self, feature): geom = self.geomencoder - x_ = coord.column - y_ = coord.row + x_, y_ = 0, 0 + cmd= -1 cmd_idx = -1 vtx_cmd = -1 @@ -122,8 +119,8 @@ def addFeature(self, feature, coord): self._handle_skipped_last(f, skipped_index, cur_x, cur_y, x_, y_) # Compute delta to the previous coordinate. - cur_x = int(floor((x * path_multiplier) + 0.5)) - cur_y = int(floor((y * path_multiplier) + 0.5)) + cur_x = int(x) + cur_y = int(y) dx = cur_x - x_ dy = cur_y - y_ @@ -134,8 +131,8 @@ def addFeature(self, feature, coord): next_coord = coordinates[it+1] if next_coord['cmd'] == CMD_LINE_TO: next_x, next_y = next_coord['x'], next_coord['y'] - next_dx = fabs(cur_x - int(floor((next_x * path_multiplier) + 0.5))) - next_dy = fabs(cur_y - int(floor((next_y * path_multiplier) + 0.5))) + next_dx = fabs(cur_x - int(next_x)) + next_dy = fabs(cur_y - int(next_y)) if ((next_dx == 0 and next_dy >= tolerance) or (next_dy == 0 and next_dx >= tolerance)): sharp_turn_ahead = True @@ -157,7 +154,6 @@ def addFeature(self, feature, coord): elif vtx_cmd == CMD_SEG_END: if prev_cmd != CMD_SEG_END: length = length + 1 - break; else: raise Exception("Unknown command type: '%s'" % vtx_cmd) @@ -168,7 +164,7 @@ def addFeature(self, feature, coord): if (skipped_last and skipped_index > 1): # if we skipped previous vertex we just update it to the last one here. handle_skipped_last(f, skipped_index, cur_x, cur_y, x_, y_) - + # Update the last length/command value. if (cmd_idx >= 0): f.geometry.__setitem__(cmd_idx, self._encode_cmd_length(cmd, length)) From f6055af30b25864377afd7b008d5582548d334bb Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Wed, 7 May 2014 14:38:49 -0400 Subject: [PATCH 031/344] removing GeomEncoder - using shapely.wkb's loads instead for parsing purposes. cleaning up mapbox - splitting into more meaninglful functional blocks --- .../Goodies/VecTiles/Mapbox/GeomEncoder.py | 340 ------------------ TileStache/Goodies/VecTiles/mapbox.py | 239 +++++++----- 2 files changed, 150 insertions(+), 429 deletions(-) delete mode 100644 TileStache/Goodies/VecTiles/Mapbox/GeomEncoder.py diff --git a/TileStache/Goodies/VecTiles/Mapbox/GeomEncoder.py b/TileStache/Goodies/VecTiles/Mapbox/GeomEncoder.py deleted file mode 100644 index c7a08126..00000000 --- a/TileStache/Goodies/VecTiles/Mapbox/GeomEncoder.py +++ /dev/null @@ -1,340 +0,0 @@ -""" -A parser for the Well Text Binary format of OpenGIS types. -""" - -import sys, traceback, struct - - -# based on xdrlib.Unpacker -class _ExtendedUnPacker: - """ - A simple binary struct parser, only implements the types that are need for the WKB format. - """ - - def __init__(self,data): - self.reset(data) - self.setEndianness('XDR') - - def reset(self, data): - self.__buf = data - self.__pos = 0 - - def get_position(self): - return self.__pos - - def set_position(self, position): - self.__pos = position - - def get_buffer(self): - return self.__buf - - def done(self): - if self.__pos < len(self.__buf): - raise ExceptionWKBParser('unextracted data remains') - - def setEndianness(self,endianness): - if endianness == 'XDR': - self._endflag = '>' - elif endianness == 'NDR': - self._endflag = '<' - else: - raise ExceptionWKBParser('Attempt to set unknown endianness in ExtendedUnPacker') - - def unpack_byte(self): - i = self.__pos - self.__pos = j = i+1 - data = self.__buf[i:j] - if len(data) < 1: - raise EOFError - byte = struct.unpack('%sB' % self._endflag, data)[0] - return byte - - def unpack_uint32(self): - i = self.__pos - self.__pos = j = i+4 - data = self.__buf[i:j] - if len(data) < 4: - raise EOFError - uint32 = struct.unpack('%si' % self._endflag, data)[0] - return uint32 - - def unpack_short(self): - i = self.__pos - self.__pos = j = i+2 - data = self.__buf[i:j] - if len(data) < 2: - raise EOFError - short = struct.unpack('%sH' % self._endflag, data)[0] - return short - - def unpack_double(self): - i = self.__pos - self.__pos = j = i+8 - data = self.__buf[i:j] - if len(data) < 8: - raise EOFError - return struct.unpack('%sd' % self._endflag, data)[0] - -class ExceptionWKBParser(Exception): - '''This is the WKB Parser Exception class.''' - def __init__(self, value): - self.value = value - def __str__(self): - return `self.value` - -class GeomEncoder: - - _count = 0 - - def __init__(self, tileSize): - """ - Initialise a new WKBParser. - - """ - - self._typemap = {1: self.parsePoint, - 2: self.parseLineString, - 3: self.parsePolygon, - 4: self.parseMultiPoint, - 5: self.parseMultiLineString, - 6: self.parseMultiPolygon, - 7: self.parseGeometryCollection} - self.coordinates = [] - self.index = [] - self.position = 0 - self.lastX = 0 - self.lastY = 0 - self.dropped = 0 - self.num_points = 0 - self.isPoint = True - self.tileSize = tileSize - 1 - self.first = True - - def parseGeometry(self, geometry): - - - """ - A factory method for creating objects of the correct OpenGIS type. - """ - - self.coordinates = [] - self.index = [] - self.position = 0 - self.lastX = 0 - self.lastY = 0 - self.isPoly = False - self.isPoint = True; - self.dropped = 0; - self.first = True - # Used for exception strings - self._current_string = geometry - - reader = _ExtendedUnPacker(geometry) - - # Start the parsing - self._dispatchNextType(reader) - - - def _dispatchNextType(self,reader): - """ - Read a type id from the binary stream (reader) and call the correct method to parse it. - """ - - # Need to check endianess here! - endianness = reader.unpack_byte() - if endianness == 0: - reader.setEndianness('XDR') - elif endianness == 1: - reader.setEndianness('NDR') - else: - raise ExceptionWKBParser("Invalid endianness in WKB format.\n"\ - "The parser can only cope with XDR/big endian WKB format.\n"\ - "To force the WKB format to be in XDR use AsBinary(,'XDR'") - - - geotype = reader.unpack_uint32() - - mask = geotype & 0x80000000 # This is used to mask of the dimension flag. - - srid = geotype & 0x20000000 - # ignore srid ... - if srid != 0: - reader.unpack_uint32() - - dimensions = 2 - if mask == 0: - dimensions = 2 - else: - dimensions = 3 - - geotype = geotype & 0x1FFFFFFF - # Despatch to a method on the type id. - if self._typemap.has_key(geotype): - self._typemap[geotype](reader, dimensions) - else: - raise ExceptionWKBParser('Error type to dispatch with geotype = %s \n'\ - 'Invalid geometry in WKB string: %s' % (str(geotype), - str(self._current_string),)) - - def parseGeometryCollection(self, reader, dimension): - try: - num_geoms = reader.unpack_uint32() - - for _ in xrange(0,num_geoms): - self._dispatchNextType(reader) - - except: - _, value, tb = sys.exc_info()[:3] - error = ("%s , %s \n" % (type, value)) - for bits in traceback.format_exception(type,value,tb): - error = error + bits + '\n' - del tb - raise ExceptionWKBParser("Caught unhandled exception parsing GeometryCollection: %s \n"\ - "Traceback: %s\n" % (str(self._current_string),error)) - - - def parseLineString(self, reader, dimensions): - self.isPoint = False; - try: - num_points = reader.unpack_uint32() - - self.num_points = 0; - - for _ in xrange(0,num_points): - self.parsePoint(reader,dimensions) - - self.index.append(self.num_points) - #self.lastX = 0 - #self.lastY = 0 - self.first = True - - except: - _, value, tb = sys.exc_info()[:3] - error = ("%s , %s \n" % (type, value)) - for bits in traceback.format_exception(type,value,tb): - error = error + bits + '\n' - del tb - print error - raise ExceptionWKBParser("Caught unhandled exception parsing Linestring: %s \n"\ - "Traceback: %s\n" % (str(self._current_string),error)) - - - def parseMultiLineString(self, reader, dimensions): - try: - num_linestrings = reader.unpack_uint32() - - for _ in xrange(0,num_linestrings): - self._dispatchNextType(reader) - - except: - _, value, tb = sys.exc_info()[:3] - error = ("%s , %s \n" % (type, value)) - for bits in traceback.format_exception(type,value,tb): - error = error + bits + '\n' - del tb - raise ExceptionWKBParser("Caught unhandled exception parsing MultiLineString: %s \n"\ - "Traceback: %s\n" % (str(self._current_string),error)) - - - def parseMultiPoint(self, reader, dimensions): - try: - num_points = reader.unpack_uint32() - - for _ in xrange(0,num_points): - self._dispatchNextType(reader) - except: - _, value, tb = sys.exc_info()[:3] - error = ("%s , %s \n" % (type, value)) - for bits in traceback.format_exception(type,value,tb): - error = error + bits + '\n' - del tb - raise ExceptionWKBParser("Caught unhandled exception parsing MultiPoint: %s \n"\ - "Traceback: %s\n" % (str(self._current_string),error)) - - - def parseMultiPolygon(self, reader, dimensions): - try: - num_polygons = reader.unpack_uint32() - for n in xrange(0,num_polygons): - if n > 0: - self.index.append(0); - - self._dispatchNextType(reader) - except: - _, value, tb = sys.exc_info()[:3] - error = ("%s , %s \n" % (type, value)) - for bits in traceback.format_exception(type,value,tb): - error = error + bits + '\n' - del tb - raise ExceptionWKBParser("Caught unhandled exception parsing MultiPolygon: %s \n"\ - "Traceback: %s\n" % (str(self._current_string),error)) - - - def parsePoint(self, reader, dimensions): - x = reader.unpack_double() - y = reader.unpack_double() - - if dimensions == 3: - reader.unpack_double() - - xx = int(round(x)) - # flip upside down - yy = self.tileSize - int(round(y)) - - self.coordinates.append(xx) - self.coordinates.append(yy) - self.num_points += 1 - - self.first = False - self.lastX = xx - self.lastY = yy - - - def parsePolygon(self, reader, dimensions): - self.isPoint = False; - try: - num_rings = reader.unpack_uint32() - - for _ in xrange(0,num_rings): - self.parseLinearRing(reader,dimensions) - - self.isPoly = True - - except: - _, value, tb = sys.exc_info()[:3] - error = ("%s , %s \n" % (type, value)) - for bits in traceback.format_exception(type,value,tb): - error = error + bits + '\n' - del tb - raise ExceptionWKBParser("Caught unhandled exception parsing Polygon: %s \n"\ - "Traceback: %s\n" % (str(self._current_string),error)) - - def parseLinearRing(self, reader, dimensions): - self.isPoint = False; - try: - num_points = reader.unpack_uint32() - - self.num_points = 0; - - # dont skip the last point - for _ in xrange(0,num_points): - self.parsePoint(reader,dimensions) - - # dont skip the last point - # reader.unpack_double() - # reader.unpack_double() - if dimensions == 3: - reader.unpack_double() - - self.index.append(self.num_points) - - self.first = True - - except: - _, value, tb = sys.exc_info()[:3] - error = ("%s , %s \n" % (type, value)) - for bits in traceback.format_exception(type,value,tb): - error = error + bits + '\n' - del tb - raise ExceptionWKBParser("Caught unhandled exception parsing LinearRing: %s \n"\ - "Traceback: %s\n" % (str(self._current_string),error)) \ No newline at end of file diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index fe052534..898c8708 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -1,6 +1,5 @@ import types from Mapbox import vector_tile_pb2 -from Mapbox.GeomEncoder import GeomEncoder from shapely.wkb import loads from math import floor, fabs @@ -8,7 +7,6 @@ from TileStache.Core import KnownUnknown import re import logging -import struct # coordindates are scaled to this range within tile extents = 4096 @@ -42,7 +40,7 @@ def merge(file, feature_layers, coord): tile = VectorTile(extents) for layer in feature_layers: - tile.addFeatures(layer['features'], coord, extents, layer['name']) + tile.addFeatures(layer['features'], coord, layer['name']) data = tile.tile.SerializeToString() file.write(data) @@ -51,53 +49,179 @@ class VectorTile: """ """ def __init__(self, extents, layer_name=""): - self.geomencoder = GeomEncoder(extents) self.tile = vector_tile_pb2.tile() + self.extents = extents - def addFeatures(self, features, coord, extents, layer_name=""): + def addFeatures(self, features, coord, layer_name=""): self.layer = self.tile.layers.add() self.layer.name = layer_name self.layer.version = 2 - self.layer.extent = extents + self.layer.extent = self.extents self.feature_count = 0 self.keys = [] self.values = [] - self.pixels = [] + for feature in features: self.addFeature(feature) def addFeature(self, feature): - geom = self.geomencoder - x_, y_ = 0, 0 - - cmd= -1 - cmd_idx = -1 - vtx_cmd = -1 - prev_cmd= -1 - - skipped_index = -1 - skipped_last = False - cur_x = 0 - cur_y = 0 - f = self.layer.features.add() self.feature_count += 1 f.id = self.feature_count - f.type = self.tile.Point if geom.isPoint else (self.tile.Polygon if geom.isPoly else self.tile.LineString) + + # osm_id or the hash is passed in as a feature tag 'uid' + if len(feature) >= 2: + feature[1].update(uid=feature[2]) + # properties self._handle_attr(self.layer, f, feature[1]) - geom.parseGeometry(feature[0]) + # geometry + shape = loads(feature[0]) + f.type = self._get_feature_type(shape) + self._geo_encode(f, shape) + + + def _get_cmd_type(self, gtype, i, num_points): + cmd_type = -1 + if gtype == self.tile.Point: + cmd_type = CMD_MOVE_TO + elif gtype == self.tile.Polygon or gtype == self.tile.LineString: + cmd_type = CMD_LINE_TO + if i==0: + cmd_type = CMD_MOVE_TO + if gtype == self.tile.Polygon and i+1==num_points: + cmd_type = CMD_SEG_END + return cmd_type + + def _get_feature_type(self, shape): + if shape.type == 'Point' or shape.type == 'MultiPoint': + return self.tile.Point + elif shape.type == 'LineString' or shape.type == 'MultiLineString': + return self.tile.LineString + elif shape.type == 'Polygon' or shape.type == 'MultiPolygon': + return self.tile.Polygon + + def _encode_cmd_length(self, cmd, length): + return (length << cmd_bits) | (cmd & ((1 << cmd_bits) - 1)) + + def _chunker(self, seq, size): + return [seq[pos:pos + size] for pos in xrange(0, len(seq), size)] + + def _handle_attr(self, layer, feature, props): + for k,v in props.items(): + if k not in self.keys: + layer.keys.append(k) + self.keys.append(k) + idx = self.keys.index(k) + feature.tags.append(idx) + else: + idx = self.keys.index(k) + feature.tags.append(idx) + if v not in self.values: + if (isinstance(v,bool)): + val = layer.values.add() + val.bool_value = v + elif (isinstance(v,str)) or (isinstance(v,unicode)): + val = layer.values.add() + val.string_value = unicode(v,'utf8') + elif (isinstance(v,int)): + val = layer.values.add() + val.int_value = v + elif (isinstance(v,float)) or (isinstance(v,long)): + val = layer.values.add() + val.double_value = v + # else: + # # do nothing because we know kind is sometimes + # logging.info("Unknown value type: '%s' for key: '%s'", type(v), k) + # raise Exception("Unknown value type: '%s'" % type(v)) + self.values.append(v) + feature.tags.append(self.values.index(v)) + + def _handle_skipped_last(self, f, skipped_index, cur_x, cur_y, x_, y_): + last_x = f.geometry[skipped_index - 2] + last_y = f.geometry[skipped_index - 1] + last_dx = ((last_x >> 1) ^ (-(last_x & 1))) + last_dy = ((last_y >> 1) ^ (-(last_y & 1))) + dx = cur_x - x_ + last_dx + dy = cur_y - y_ + last_dy + x_ = cur_x + y_ = cur_y + f.geometry.__setitem__(skipped_index - 2, ((dx << 1) ^ (dx >> 31))) + f.geometry.__setitem__(skipped_index - 1, ((dy << 1) ^ (dy >> 31))) + + def _process_points(self, geom_type, geom_coordinates): coordinates = [] - points = self._chunker(geom.coordinates,2) # x,y corodinates grouped as one point. + points = self._chunker(geom_coordinates,2) # x,y coordinates grouped as one point. for i, coords in enumerate(points): coordinates.append({ 'x': coords[0], 'y': coords[1], - 'cmd': self._get_cmd_type(f.type, i, len(points))}) + 'cmd': self._get_cmd_type(geom_type, i, len(points))}) + return coordinates + + def _parseGeometry(self, shape): + coordinates = [] + + def _get_point_obj(x, y): + coordinates.append(x) + coordinates.append(self.extents - y) + + def _get_arc_obj(line): + [ _get_point_obj(x,y) for (x,y) in line.coords ] + + if shape.type == 'GeometryCollection': + # do nothing + coordinates = [] + + elif shape.type == 'Point': + _get_point_obj(shape.x,shape.y) + + elif shape.type == 'LineString': + _get_arc_obj(shape) + + elif shape.type == 'Polygon': + rings = [shape.exterior] + list(shape.interiors) + for ring in rings: + _get_arc_obj(ring) + + elif shape.type == 'MultiPoint': + for point in shape.geoms: + _get_point_obj(point.x, point.y) + + elif shape.type == 'MultiLineString': + for line in shape.geoms: + _get_arc_obj(line) + + elif shape.type == 'MultiPolygon': + for polygon in shape.geoms: + rings = [polygon.exterior] + list(polygon.interiors) + for ring in rings: + _get_arc_obj(ring) + + else: + raise NotImplementedError("Can't do %s geometries" % shape.type) + + return coordinates + + def _geo_encode(self, f, shape): + x_, y_ = 0, 0 + + cmd= -1 + cmd_idx = -1 + vtx_cmd = -1 + prev_cmd= -1 + skipped_index = -1 + skipped_last = False + cur_x = 0 + cur_y = 0 + it = 0 length = 0 + + geom_coordinates = self._parseGeometry(shape) + coordinates = self._process_points(f.type, geom_coordinates) while (True): if it >= len(coordinates): @@ -163,71 +287,8 @@ def addFeature(self, feature): # at least one vertex + cmd/length if (skipped_last and skipped_index > 1): # if we skipped previous vertex we just update it to the last one here. - handle_skipped_last(f, skipped_index, cur_x, cur_y, x_, y_) - + self._handle_skipped_last(f, skipped_index, cur_x, cur_y, x_, y_) + # Update the last length/command value. if (cmd_idx >= 0): f.geometry.__setitem__(cmd_idx, self._encode_cmd_length(cmd, length)) - - - # TODO: figure out if cmd can change within a feature geom - def _get_cmd_type(self, gtype, i, points): - cmd_type = -1 - if gtype == self.tile.Point: - cmd_type = CMD_MOVE_TO - elif gtype == self.tile.Polygon or gtype == self.tile.LineString: - cmd_type = CMD_LINE_TO - if i==0: - cmd_type = CMD_MOVE_TO - if gtype == self.tile.Polygon and i+1==points: - cmd_type = CMD_SEG_END - return cmd_type - - def _encode_cmd_length(self, cmd, length): - # cmd: 1 (MOVE_TO) - # cmd: 2 (LINE_TO) - # cmd: 7 (CLOSE_PATH) - return (length << cmd_bits) | (cmd & ((1 << cmd_bits) - 1)) - - def _chunker(self, seq, size): - return [seq[pos:pos + size] for pos in xrange(0, len(seq), size)] - - def _handle_skipped_last(self, f, skipped_index, cur_x, cur_y, x_, y_): - last_x = f.geometry[skipped_index - 2] - last_y = f.geometry[skipped_index - 1] - last_dx = ((last_x >> 1) ^ (-(last_x & 1))) - last_dy = ((last_y >> 1) ^ (-(last_y & 1))) - dx = cur_x - x_ + last_dx - dy = cur_y - y_ + last_dy - x_ = cur_x - y_ = cur_y - f.geometry.__setitem__(skipped_index - 2, ((dx << 1) ^ (dx >> 31))) - f.geometry.__setitem__(skipped_index - 1, ((dy << 1) ^ (dy >> 31))) - - def _handle_attr(self, layer, feature, props): - for k,v in props.items(): - if k not in self.keys: - layer.keys.append(k) - self.keys.append(k) - idx = self.keys.index(k) - feature.tags.append(idx) - else: - idx = self.keys.index(k) - feature.tags.append(idx) - if v not in self.values: - if (isinstance(v,bool)): - val = layer.values.add() - val.bool_value = v - elif (isinstance(v,str)) or (isinstance(v,unicode)): - val = layer.values.add() - val.string_value = unicode(v,'utf8') - elif (isinstance(v,int)): - val = layer.values.add() - val.int_value = v - elif (isinstance(v,float)): - val = layer.values.add() - val.double_value = v - # else: - # raise Exception("Unknown value type: '%s'" % type(v)) - self.values.append(v) - feature.tags.append(self.values.index(v)) From af53e87589594056aaba3e0a25b682e232f7d1e9 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Wed, 7 May 2014 15:32:48 -0400 Subject: [PATCH 032/344] ignoring key,value pairs where value is of NoneType. (ex: when natural, landuse, kind is None, nothing will be sent) --- TileStache/Goodies/VecTiles/mapbox.py | 51 +++++++++++++-------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index 898c8708..aaa6f55f 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -110,33 +110,30 @@ def _chunker(self, seq, size): def _handle_attr(self, layer, feature, props): for k,v in props.items(): - if k not in self.keys: - layer.keys.append(k) - self.keys.append(k) - idx = self.keys.index(k) - feature.tags.append(idx) - else: - idx = self.keys.index(k) - feature.tags.append(idx) - if v not in self.values: - if (isinstance(v,bool)): - val = layer.values.add() - val.bool_value = v - elif (isinstance(v,str)) or (isinstance(v,unicode)): - val = layer.values.add() - val.string_value = unicode(v,'utf8') - elif (isinstance(v,int)): - val = layer.values.add() - val.int_value = v - elif (isinstance(v,float)) or (isinstance(v,long)): - val = layer.values.add() - val.double_value = v - # else: - # # do nothing because we know kind is sometimes - # logging.info("Unknown value type: '%s' for key: '%s'", type(v), k) - # raise Exception("Unknown value type: '%s'" % type(v)) - self.values.append(v) - feature.tags.append(self.values.index(v)) + if v is not None: + if k not in self.keys: + layer.keys.append(k) + self.keys.append(k) + feature.tags.append(self.keys.index(k)) + if v not in self.values: + self.values.append(v) + if (isinstance(v,bool)): + val = layer.values.add() + val.bool_value = v + elif (isinstance(v,str)) or (isinstance(v,unicode)): + val = layer.values.add() + val.string_value = unicode(v,'utf8') + elif (isinstance(v,int)) or (isinstance(v,long)): + val = layer.values.add() + val.int_value = v + elif (isinstance(v,float)): + val = layer.values.add() + val.double_value = v + # else: + # # do nothing because we know kind is sometimes + # logging.info("Unknown value type: '%s' for key: '%s'", type(v), k) + # raise Exception("Unknown value type: '%s'" % type(v)) + feature.tags.append(self.values.index(v)) def _handle_skipped_last(self, f, skipped_index, cur_x, cur_y, x_, y_): last_x = f.geometry[skipped_index - 2] From 7cec9a149647097223eb78f242b4f8409f8bb042 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Thu, 8 May 2014 14:41:43 -0400 Subject: [PATCH 033/344] encoding exterior and interior polygons correctly by treating each ring/arc seperated with their MOVE_TO and SEG_ENDs. This simplifies things because the cmd logic is now part of the parser/builder that builds the cordinate array - we no longer need _process_points and _get_cmd_type functions --- TileStache/Goodies/VecTiles/mapbox.py | 71 +++++++++++++-------------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index aaa6f55f..520ee5df 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -80,19 +80,6 @@ def addFeature(self, feature): shape = loads(feature[0]) f.type = self._get_feature_type(shape) self._geo_encode(f, shape) - - - def _get_cmd_type(self, gtype, i, num_points): - cmd_type = -1 - if gtype == self.tile.Point: - cmd_type = CMD_MOVE_TO - elif gtype == self.tile.Polygon or gtype == self.tile.LineString: - cmd_type = CMD_LINE_TO - if i==0: - cmd_type = CMD_MOVE_TO - if gtype == self.tile.Polygon and i+1==num_points: - cmd_type = CMD_SEG_END - return cmd_type def _get_feature_type(self, shape): if shape.type == 'Point' or shape.type == 'MultiPoint': @@ -128,7 +115,7 @@ def _handle_attr(self, layer, feature, props): val.int_value = v elif (isinstance(v,float)): val = layer.values.add() - val.double_value = v + val.int_value = int(v) # this is a hack! TODO: Fix it # else: # # do nothing because we know kind is sometimes # logging.info("Unknown value type: '%s' for key: '%s'", type(v), k) @@ -147,25 +134,34 @@ def _handle_skipped_last(self, f, skipped_index, cur_x, cur_y, x_, y_): f.geometry.__setitem__(skipped_index - 2, ((dx << 1) ^ (dx >> 31))) f.geometry.__setitem__(skipped_index - 1, ((dy << 1) ^ (dy >> 31))) - def _process_points(self, geom_type, geom_coordinates): - coordinates = [] - points = self._chunker(geom_coordinates,2) # x,y coordinates grouped as one point. - for i, coords in enumerate(points): - coordinates.append({ - 'x': coords[0], - 'y': coords[1], - 'cmd': self._get_cmd_type(geom_type, i, len(points))}) - return coordinates - def _parseGeometry(self, shape): coordinates = [] - - def _get_point_obj(x, y): - coordinates.append(x) - coordinates.append(self.extents - y) - - def _get_arc_obj(line): - [ _get_point_obj(x,y) for (x,y) in line.coords ] + line = "line" + polygon = "polygon" + + def _get_point_obj(x, y, cmd=CMD_MOVE_TO): + coordinate = { + 'x' : x, + 'y' : self.extents - y, + 'cmd': cmd + } + coordinates.append(coordinate) + + def _get_arc_obj(arc, type): + length = len(arc.coords) + iterator=0 + cmd = CMD_MOVE_TO + while (iterator < length): + x = arc.coords[iterator][0] + y = arc.coords[iterator][1] + if iterator == 0: + cmd = CMD_MOVE_TO + elif iterator == length-1 and type == polygon: + cmd = CMD_SEG_END + else: + cmd = CMD_LINE_TO + _get_point_obj(x, y, cmd) + iterator = iterator + 1 if shape.type == 'GeometryCollection': # do nothing @@ -175,26 +171,26 @@ def _get_arc_obj(line): _get_point_obj(shape.x,shape.y) elif shape.type == 'LineString': - _get_arc_obj(shape) + _get_arc_obj(shape, line) elif shape.type == 'Polygon': rings = [shape.exterior] + list(shape.interiors) for ring in rings: - _get_arc_obj(ring) + _get_arc_obj(ring, polygon) elif shape.type == 'MultiPoint': for point in shape.geoms: _get_point_obj(point.x, point.y) elif shape.type == 'MultiLineString': - for line in shape.geoms: - _get_arc_obj(line) + for arc in shape.geoms: + _get_arc_obj(arc, line) elif shape.type == 'MultiPolygon': for polygon in shape.geoms: rings = [polygon.exterior] + list(polygon.interiors) for ring in rings: - _get_arc_obj(ring) + _get_arc_obj(ring, polygon) else: raise NotImplementedError("Can't do %s geometries" % shape.type) @@ -217,8 +213,7 @@ def _geo_encode(self, f, shape): it = 0 length = 0 - geom_coordinates = self._parseGeometry(shape) - coordinates = self._process_points(f.type, geom_coordinates) + coordinates = self._parseGeometry(shape) while (True): if it >= len(coordinates): From 2a83e05f044367ac1df61a3bfce4ac4be5da8141 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Thu, 8 May 2014 17:51:15 -0400 Subject: [PATCH 034/344] getting the right byte order is important. double array() + byteswap trick to get what google pbf wants (big endian) --- TileStache/Goodies/VecTiles/mapbox.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index 520ee5df..1a1bca82 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -3,6 +3,7 @@ from shapely.wkb import loads from math import floor, fabs +from array import array from TileStache.Core import KnownUnknown import re @@ -115,7 +116,10 @@ def _handle_attr(self, layer, feature, props): val.int_value = v elif (isinstance(v,float)): val = layer.values.add() - val.int_value = int(v) # this is a hack! TODO: Fix it + d_arr = array('d', [v]) + # google pbf expects big endian by default + d_arr.byteswap() + val.double_value = d_arr[0] # else: # # do nothing because we know kind is sometimes # logging.info("Unknown value type: '%s' for key: '%s'", type(v), k) From b500d1229bc122085ec723812254d947298c1b10 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 10 May 2014 11:53:57 -0700 Subject: [PATCH 035/344] Updated location of version information and live package, bumped to 1.49.9 --- CHANGELOG | 3 +++ Makefile | 61 +++--------------------------------------- TileStache/VERSION | 1 + TileStache/__init__.py | 4 ++- VERSION | 1 - setup.py | 4 +-- 6 files changed, 13 insertions(+), 61 deletions(-) create mode 100644 TileStache/VERSION delete mode 100644 VERSION diff --git a/CHANGELOG b/CHANGELOG index 9bfef885..9a5ad946 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +2014-05-10: 1.49.9 +- Moved everything to PyPI and fixed VERSION kerfuffle. + 2013-07-02: 1.49.8 - Dropped Proxy provider srs=900913 check. - Updated most JSON mime-types from text/json to application/json. diff --git a/Makefile b/Makefile index 11926d12..7b7626eb 100644 --- a/Makefile +++ b/Makefile @@ -1,61 +1,8 @@ -VERSION:=$(shell cat VERSION) -PACKAGE=TileStache-$(VERSION) -TARBALL=$(PACKAGE).tar.gz DOCROOT=tilestache.org:public_html/tilestache/www -all: $(TARBALL) - # - -live: $(TARBALL) doc - scp $(TARBALL) $(DOCROOT)/download/ +live: doc rsync -Cr doc/ $(DOCROOT)/doc/ - python setup.py register - -$(TARBALL): doc - mkdir $(PACKAGE) - ln setup.py $(PACKAGE)/ - ln README.md $(PACKAGE)/ - ln VERSION $(PACKAGE)/ - ln tilestache.cfg $(PACKAGE)/ - ln tilestache.cgi $(PACKAGE)/ - - mkdir $(PACKAGE)/TileStache - ln TileStache/*.py $(PACKAGE)/TileStache/ - - rm $(PACKAGE)/TileStache/__init__.py - cp TileStache/__init__.py $(PACKAGE)/TileStache/__init__.py - perl -pi -e 's#\bN\.N\.N\b#$(VERSION)#' $(PACKAGE)/TileStache/__init__.py - - mkdir $(PACKAGE)/TileStache/Vector - ln TileStache/Vector/*.py $(PACKAGE)/TileStache/Vector/ - - mkdir $(PACKAGE)/TileStache/Goodies - ln TileStache/Goodies/*.py $(PACKAGE)/TileStache/Goodies/ - - mkdir $(PACKAGE)/TileStache/Goodies/Caches - ln TileStache/Goodies/Caches/*.py $(PACKAGE)/TileStache/Goodies/Caches/ - - mkdir $(PACKAGE)/TileStache/Goodies/Providers - ln TileStache/Goodies/Providers/*.py $(PACKAGE)/TileStache/Goodies/Providers/ - ln TileStache/Goodies/Providers/*.ttf $(PACKAGE)/TileStache/Goodies/Providers/ - - mkdir $(PACKAGE)/TileStache/Goodies/VecTiles - ln TileStache/Goodies/VecTiles/*.py $(PACKAGE)/TileStache/Goodies/VecTiles/ - - mkdir $(PACKAGE)/scripts - ln scripts/*.py $(PACKAGE)/scripts/ - - mkdir $(PACKAGE)/examples - #ln examples/*.py $(PACKAGE)/examples/ - - mkdir $(PACKAGE)/doc - ln doc/*.html $(PACKAGE)/doc/ - - mkdir $(PACKAGE)/man - ln man/*.1 $(PACKAGE)/man/ - - tar -czf $(TARBALL) $(PACKAGE) - rm -rf $(PACKAGE) + python setup.py sdist upload doc: mkdir doc @@ -115,8 +62,8 @@ doc: cp API.html doc/index.html perl -pi -e 's#http://tilestache.org/doc/##' doc/index.html - perl -pi -e 's#\bN\.N\.N\b#$(VERSION)#' doc/index.html + perl -pi -e 's#\bN\.N\.N\b#$(TileStache/VERSION)#' doc/index.html clean: find TileStache -name '*.pyc' -delete - rm -rf $(TARBALL) doc + rm -rf doc diff --git a/TileStache/VERSION b/TileStache/VERSION new file mode 100644 index 00000000..077f4d09 --- /dev/null +++ b/TileStache/VERSION @@ -0,0 +1 @@ +1.49.9 diff --git a/TileStache/__init__.py b/TileStache/__init__.py index ba117951..b1a519bd 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -8,7 +8,9 @@ Documentation available at http://tilestache.org/doc/ """ -__version__ = 'N.N.N' +import os.path + +__version__ = open(os.path.join(os.path.dirname(__file__), 'VERSION')).read().strip() import re diff --git a/VERSION b/VERSION deleted file mode 100644 index 7e564d13..00000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -1.49.8 diff --git a/setup.py b/setup.py index 8004e5db..963c7da8 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ import sys -version = open('VERSION', 'r').read().strip() +version = open('TileStache/VERSION', 'r').read().strip() def is_installed(name): @@ -40,5 +40,5 @@ def is_installed(name): 'TileStache.Goodies.VecTiles'], scripts=['scripts/tilestache-compose.py', 'scripts/tilestache-seed.py', 'scripts/tilestache-clean.py', 'scripts/tilestache-server.py', 'scripts/tilestache-render.py', 'scripts/tilestache-list.py'], data_files=[('share/tilestache', ['TileStache/Goodies/Providers/DejaVuSansMono-alphanumeric.ttf'])], - download_url='http://tilestache.org/download/TileStache-%(version)s.tar.gz' % locals(), + package_data={'TileStache': ['VERSION', '../doc/*.html']}, license='BSD') From 9f616edaf92d6f5fed71c4a837c5e6e91a69082d Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 10 May 2014 12:16:19 -0700 Subject: [PATCH 036/344] Replaced pydoc with python -m pydoc so the docs will build in a virtualenv --- Makefile | 90 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index 7b7626eb..a2f6b3df 100644 --- a/Makefile +++ b/Makefile @@ -7,52 +7,52 @@ live: doc doc: mkdir doc - pydoc -w TileStache - pydoc -w TileStache.Core - pydoc -w TileStache.Caches - pydoc -w TileStache.Memcache - pydoc -w TileStache.Redis - pydoc -w TileStache.S3 - pydoc -w TileStache.Config - pydoc -w TileStache.Vector - pydoc -w TileStache.Vector.Arc - pydoc -w TileStache.Geography - pydoc -w TileStache.Providers - pydoc -w TileStache.Mapnik - pydoc -w TileStache.MBTiles - pydoc -w TileStache.Sandwich - pydoc -w TileStache.Pixels - pydoc -w TileStache.Goodies - pydoc -w TileStache.Goodies.Caches - pydoc -w TileStache.Goodies.Caches.LimitedDisk - pydoc -w TileStache.Goodies.Caches.GoogleCloud - pydoc -w TileStache.Goodies.Providers - pydoc -w TileStache.Goodies.Providers.Composite - pydoc -w TileStache.Goodies.Providers.Cascadenik - pydoc -w TileStache.Goodies.Providers.PostGeoJSON - pydoc -w TileStache.Goodies.Providers.SolrGeoJSON - pydoc -w TileStache.Goodies.Providers.MapnikGrid - pydoc -w TileStache.Goodies.Providers.MirrorOSM - pydoc -w TileStache.Goodies.Providers.Monkeycache - pydoc -w TileStache.Goodies.Providers.UtfGridComposite - pydoc -w TileStache.Goodies.Providers.UtfGridCompositeOverlap - pydoc -w TileStache.Goodies.Providers.TileDataOSM - pydoc -w TileStache.Goodies.Providers.Grid - pydoc -w TileStache.Goodies.Providers.GDAL - pydoc -w TileStache.Goodies.AreaServer - pydoc -w TileStache.Goodies.StatusServer - pydoc -w TileStache.Goodies.Proj4Projection - pydoc -w TileStache.Goodies.ExternalConfigServer - pydoc -w TileStache.Goodies.VecTiles - pydoc -w TileStache.Goodies.VecTiles.server - pydoc -w TileStache.Goodies.VecTiles.client - pydoc -w TileStache.Goodies.VecTiles.geojson - pydoc -w TileStache.Goodies.VecTiles.topojson - pydoc -w TileStache.Goodies.VecTiles.mvt - pydoc -w TileStache.Goodies.VecTiles.wkb - pydoc -w TileStache.Goodies.VecTiles.ops + python -m pydoc -w TileStache + python -m pydoc -w TileStache.Core + python -m pydoc -w TileStache.Caches + python -m pydoc -w TileStache.Memcache + python -m pydoc -w TileStache.Redis + python -m pydoc -w TileStache.S3 + python -m pydoc -w TileStache.Config + python -m pydoc -w TileStache.Vector + python -m pydoc -w TileStache.Vector.Arc + python -m pydoc -w TileStache.Geography + python -m pydoc -w TileStache.Providers + python -m pydoc -w TileStache.Mapnik + python -m pydoc -w TileStache.MBTiles + python -m pydoc -w TileStache.Sandwich + python -m pydoc -w TileStache.Pixels + python -m pydoc -w TileStache.Goodies + python -m pydoc -w TileStache.Goodies.Caches + python -m pydoc -w TileStache.Goodies.Caches.LimitedDisk + python -m pydoc -w TileStache.Goodies.Caches.GoogleCloud + python -m pydoc -w TileStache.Goodies.Providers + python -m pydoc -w TileStache.Goodies.Providers.Composite + python -m pydoc -w TileStache.Goodies.Providers.Cascadenik + python -m pydoc -w TileStache.Goodies.Providers.PostGeoJSON + python -m pydoc -w TileStache.Goodies.Providers.SolrGeoJSON + python -m pydoc -w TileStache.Goodies.Providers.MapnikGrid + python -m pydoc -w TileStache.Goodies.Providers.MirrorOSM + python -m pydoc -w TileStache.Goodies.Providers.Monkeycache + python -m pydoc -w TileStache.Goodies.Providers.UtfGridComposite + python -m pydoc -w TileStache.Goodies.Providers.UtfGridCompositeOverlap + python -m pydoc -w TileStache.Goodies.Providers.TileDataOSM + python -m pydoc -w TileStache.Goodies.Providers.Grid + python -m pydoc -w TileStache.Goodies.Providers.GDAL + python -m pydoc -w TileStache.Goodies.AreaServer + python -m pydoc -w TileStache.Goodies.StatusServer + python -m pydoc -w TileStache.Goodies.Proj4Projection + python -m pydoc -w TileStache.Goodies.ExternalConfigServer + python -m pydoc -w TileStache.Goodies.VecTiles + python -m pydoc -w TileStache.Goodies.VecTiles.server + python -m pydoc -w TileStache.Goodies.VecTiles.client + python -m pydoc -w TileStache.Goodies.VecTiles.geojson + python -m pydoc -w TileStache.Goodies.VecTiles.topojson + python -m pydoc -w TileStache.Goodies.VecTiles.mvt + python -m pydoc -w TileStache.Goodies.VecTiles.wkb + python -m pydoc -w TileStache.Goodies.VecTiles.ops - pydoc -w scripts/tilestache-*.py + python -m pydoc -w scripts/tilestache-*.py mv TileStache.html doc/ mv TileStache.*.html doc/ From 438f64980d144a01c788a77278a424956e5a9a3b Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 10 May 2014 12:16:36 -0700 Subject: [PATCH 037/344] Made Blit import failure pass so the docs can be built --- TileStache/Sandwich.py | 44 +++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/TileStache/Sandwich.py b/TileStache/Sandwich.py index c0ac7201..ee77f4db 100644 --- a/TileStache/Sandwich.py +++ b/TileStache/Sandwich.py @@ -125,23 +125,35 @@ from . import Core -import Image -import Blit - -blend_modes = { - 'screen': Blit.blends.screen, - 'add': Blit.blends.add, - 'multiply': Blit.blends.multiply, - 'subtract': Blit.blends.subtract, - 'linear light': Blit.blends.linear_light, - 'hard light': Blit.blends.hard_light - } +try: + import Image +except ImportError: + try: + from Pillow import Image + except ImportError: + from PIL import Image + +try: + import Blit + + blend_modes = { + 'screen': Blit.blends.screen, + 'add': Blit.blends.add, + 'multiply': Blit.blends.multiply, + 'subtract': Blit.blends.subtract, + 'linear light': Blit.blends.linear_light, + 'hard light': Blit.blends.hard_light + } -adjustment_names = { - 'threshold': Blit.adjustments.threshold, - 'curves': Blit.adjustments.curves, - 'curves2': Blit.adjustments.curves2 - } + adjustment_names = { + 'threshold': Blit.adjustments.threshold, + 'curves': Blit.adjustments.curves, + 'curves2': Blit.adjustments.curves2 + } + +except ImportError: + # Well, this will not work. + pass class Provider: """ Sandwich Provider. From c535e16c00978b65a384099d413784f633756429 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 10 May 2014 12:21:10 -0700 Subject: [PATCH 038/344] oops --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a2f6b3df..75d97120 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ +VERSION:=$(shell cat TileStache/VERSION) DOCROOT=tilestache.org:public_html/tilestache/www live: doc @@ -62,7 +63,7 @@ doc: cp API.html doc/index.html perl -pi -e 's#http://tilestache.org/doc/##' doc/index.html - perl -pi -e 's#\bN\.N\.N\b#$(TileStache/VERSION)#' doc/index.html + perl -pi -e 's#\bN\.N\.N\b#$(VERSION)#' doc/index.html clean: find TileStache -name '*.pyc' -delete From cb696071e00e6544b0e9e588448c646a86061132 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 10 May 2014 12:30:55 -0700 Subject: [PATCH 039/344] Bumped to 1.49.10 with testing fixes from Amigocloud --- CHANGELOG | 6 ++++++ TileStache/VERSION | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 9a5ad946..0b6a4326 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +2014-05-10: 1.49.10 +- Fixed Travis build. +- Fixed import errors for case-insensitive filesystems. +- Added TileStache Vagrant machine configuration. +- Fixed some memcache testing problems. + 2014-05-10: 1.49.9 - Moved everything to PyPI and fixed VERSION kerfuffle. diff --git a/TileStache/VERSION b/TileStache/VERSION index 077f4d09..300ca1e9 100644 --- a/TileStache/VERSION +++ b/TileStache/VERSION @@ -1 +1 @@ -1.49.9 +1.49.10 From 9ee2965f4d39dc8fb666797016497fb99d2027bd Mon Sep 17 00:00:00 2001 From: Jesse Crocker Date: Fri, 14 Mar 2014 10:27:54 -0700 Subject: [PATCH 040/344] fix allow-origin header getting over-written --- TileStache/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/TileStache/__init__.py b/TileStache/__init__.py index b1a519bd..2375b4ba 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -224,9 +224,6 @@ def requestHandler2(config_hint, path_info, query_string=None, script_name=''): except KeyError: callback = None - if layer.allowed_origin: - headers.setdefault('Access-Control-Allow-Origin', layer.allowed_origin) - # # Special case for index page. # @@ -255,7 +252,10 @@ def requestHandler2(config_hint, path_info, query_string=None, script_name=''): else: status_code, headers, content = layer.getTileResponse(coord, extension) - + + if layer.allowed_origin: + headers.setdefault('Access-Control-Allow-Origin', layer.allowed_origin) + if callback and 'json' in headers['Content-Type']: headers['Content-Type'] = 'application/javascript; charset=utf-8' content = '%s(%s)' % (callback, content) From e9b725f78dee41485dbe24de75cd16e26d5c6bfc Mon Sep 17 00:00:00 2001 From: Jesse Crocker Date: Thu, 16 Jan 2014 09:41:38 -0800 Subject: [PATCH 041/344] Add "source projection" parameter to url template provider to support WMS servers that want WGS84 coordinates --- API.html | 4 ++++ TileStache/Providers.py | 20 +++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/API.html b/API.html index b0c032ac..e8f883eb 100644 --- a/API.html +++ b/API.html @@ -1312,6 +1312,10 @@

URL Template Some WMS servers use the Referer request header to authenticate requests; this parameter provides one. +
source projection
+ Names a geographic projection, explained in Projections, that + coordinates should be transformed to for requests. +

diff --git a/TileStache/Providers.py b/TileStache/Providers.py index 8cc7b6d8..1092a981 100644 --- a/TileStache/Providers.py +++ b/TileStache/Providers.py @@ -278,11 +278,14 @@ class UrlTemplate: - referer (optional) String to use in the "Referer" header when making HTTP requests. + - source projection (optional) + Projection to transform coordinates into before making request + More on string substitutions: - http://docs.python.org/library/string.html#template-strings """ - def __init__(self, layer, template, referer=None): + def __init__(self, layer, template, referer=None, source_projection=None): """ Initialize a UrlTemplate provider with layer and template string. http://docs.python.org/library/string.html#template-strings @@ -290,6 +293,7 @@ def __init__(self, layer, template, referer=None): self.layer = layer self.template = Template(template) self.referer = referer + self.source_projection = source_projection @staticmethod def prepareKeywordArgs(config_dict): @@ -300,6 +304,9 @@ def prepareKeywordArgs(config_dict): if 'referer' in config_dict: kwargs['referer'] = config_dict['referer'] + if 'source projection' in config_dict: + kwargs['source_projection'] = Geography.getProjectionByName(config_dict['source projection']) + return kwargs def renderArea(self, width, height, srs, xmin, ymin, xmax, ymax, zoom): @@ -307,6 +314,17 @@ def renderArea(self, width, height, srs, xmin, ymin, xmax, ymax, zoom): Each argument (width, height, etc.) is substituted into the template. """ + if self.source_projection is not None: + ne_location = self.layer.projection.projLocation(Point(xmax, ymax)) + ne_point = self.source_projection.locationProj(ne_location) + ymax = ne_point.y + xmax = ne_point.x + sw_location = self.layer.projection.projLocation(Point(xmin, ymin)) + sw_point = self.source_projection.locationProj(sw_location) + ymin = sw_point.y + xmin = sw_point.x + srs = self.source_projection.srs + mapping = {'width': width, 'height': height, 'srs': srs, 'zoom': zoom} mapping.update({'xmin': xmin, 'ymin': ymin, 'xmax': xmax, 'ymax': ymax}) From 606e2c6f9f585991e07518b6522e857c843760b3 Mon Sep 17 00:00:00 2001 From: Ragi Yaser Burhum Date: Tue, 20 May 2014 10:38:25 -0700 Subject: [PATCH 042/344] Update TileStache build link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 38e07a80..5795f042 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ _a stylish alternative for caching your map tiles_ -[![Build Status](https://travis-ci.org/migurski/TileStache.png)](https://travis-ci.org/migurski/TileStache) +[![Build Status](https://travis-ci.org/TileStache/TileStache.png)](https://travis-ci.org/TileStache/TileStache) **TileStache** is a Python-based server application that can serve up map tiles based on rendered geographic data. You might be familiar with [TileCache](http://tilecache.org), From 7f5a81ab060243690f454660a0fc4e5491043358 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Tue, 20 May 2014 17:50:52 -0400 Subject: [PATCH 043/344] adding layer_name to featuers in the opensciencemap vtm tile --- TileStache/Goodies/VecTiles/oscimap.py | 28 +++++++++++++++++++++++--- TileStache/Goodies/VecTiles/server.py | 8 ++++---- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/TileStache/Goodies/VecTiles/oscimap.py b/TileStache/Goodies/VecTiles/oscimap.py index 8fc53f83..7f2c523f 100644 --- a/TileStache/Goodies/VecTiles/oscimap.py +++ b/TileStache/Goodies/VecTiles/oscimap.py @@ -18,11 +18,11 @@ # coordindates are scaled to this range within tile extents = 4096 -def encode(file, features, coord): +def encode(file, features, coord, layer_name): tile = VectorTile(extents) for feature in features: - tile.addFeature(feature, coord) + tile.addFeature(feature, coord, layer_name) tile.complete() @@ -30,6 +30,22 @@ def encode(file, features, coord): file.write(struct.pack(">I", len(data))) file.write(data) +def merge(file, feature_layers, coord): + ''' Retrieve a list of OSciMap4 tile responses and merge them into one. + + get_tiles() retrieves data and performs basic integrity checks. + ''' + tile = VectorTile(extents) + + for layer in feature_layers: + tile.addFeatures(layer['features'], coord, layer['name']) + + tile.complete() + + data = tile.out.SerializeToString() + file.write(struct.pack(">I", len(data))) + file.write(data) + class VectorTile: """ """ @@ -62,12 +78,18 @@ def complete(self): if self.cur_val - attrib_offset > 0: self.out.num_vals = self.cur_val - attrib_offset - def addFeature(self, row, coord): + def addFeatures(self, features, coord, this_layer): + for feature in features: + self.addFeature(feature, coord, this_layer) + + def addFeature(self, row, coord, this_layer): geom = self.geomencoder tags = [] #height = None layer = None + # add layer tag + tags.append(self.getTagId(('layer_name', this_layer))) for tag in row[1].iteritems(): if tag[1] is None: diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 700456c0..a6aabfa2 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -284,7 +284,7 @@ def save(self, out, format): topojson.encode(out, features, (ll.lon, ll.lat, ur.lon, ur.lat), self.clip) elif format == 'OpenScienceMap': - oscimap.encode(out, features, self.coord) + oscimap.encode(out, features, self.coord, self.layer_name) elif format == 'Mapbox': mapbox.encode(out, features, self.coord, self.layer_name) @@ -340,14 +340,14 @@ def save(self, out, format): geojson.merge(out, self.names, self.get_tiles(format), self.config, self.coord) elif format == 'OpenScienceMap': - features = [] + feature_layers = [] layers = [self.config.layers[name] for name in self.names] for layer in layers: width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - features.extend(get_features(tile.dbinfo, tile.query["OpenScienceMap"])) - oscimap.encode(out, features, self.coord) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["OpenScienceMap"])}) + oscimap.merge(out, feature_layers, self.coord) elif format == 'Mapbox': feature_layers = [] From a227e6629981b5af4fd0da9a15ed6ea78634a528 Mon Sep 17 00:00:00 2001 From: Julio M Alegria Date: Tue, 20 May 2014 17:29:40 -0500 Subject: [PATCH 044/344] Timeouts for Proxy and UrlTemplate providers Added an optional parameter "timeout" for Proxy and UrlTemplate providers: - timeout (optional) Defines a timeout in seconds for the request (if not defined, the global default timeout setting will be used). Example configuration: { "name": "proxy", "url": "http://tile.openstreetmap.org/{Z}/{X}/{Y}.png", "timeout": 10 } --- API.html | 8 ++++++++ TileStache/Providers.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/API.html b/API.html index e8f883eb..075dc7f0 100644 --- a/API.html +++ b/API.html @@ -1082,6 +1082,10 @@

Proxy . +
timeout
+ Defines a timeout in seconds for the request. + If not defined, the global default timeout setting will be used. +

@@ -1316,6 +1320,10 @@

URL Template Names a geographic projection, explained in Projections, that coordinates should be transformed to for requests. +
timeout
+ Defines a timeout in seconds for the request. + If not defined, the global default timeout setting will be used. +

diff --git a/TileStache/Providers.py b/TileStache/Providers.py index 1092a981..0d309e74 100644 --- a/TileStache/Providers.py +++ b/TileStache/Providers.py @@ -200,8 +200,12 @@ class Proxy: Provider name string from Modest Maps built-ins. See ModestMaps.builtinProviders.keys() for a list. Example: "OPENSTREETMAP". + - timeout (optional) + Defines a timeout in seconds for the request. + If not defined, the global default timeout setting will be used. - One of the above is required. When both are present, url wins. + + Either url or provider is required. When both are present, url wins. Example configuration: @@ -210,7 +214,7 @@ class Proxy: "url": "http://tile.openstreetmap.org/{Z}/{X}/{Y}.png" } """ - def __init__(self, layer, url=None, provider_name=None): + def __init__(self, layer, url=None, provider_name=None, timeout=None): """ Initialize Proxy provider with layer and url. """ if url: @@ -225,6 +229,8 @@ def __init__(self, layer, url=None, provider_name=None): else: raise Exception('Missing required url or provider parameter to Proxy provider') + self.timeout = timeout + @staticmethod def prepareKeywordArgs(config_dict): """ Convert configured parameters to keyword args for __init__(). @@ -237,6 +243,9 @@ def prepareKeywordArgs(config_dict): if 'provider' in config_dict: kwargs['provider_name'] = config_dict['provider'] + if 'timeout' in config_dict: + kwargs['timeout'] = config_dict['timeout'] + return kwargs def renderTile(self, width, height, srs, coord): @@ -245,8 +254,12 @@ def renderTile(self, width, height, srs, coord): img = None urls = self.provider.getTileUrls(coord) + # Explicitly tell urllib2 to get no proxies + proxy_support = urllib2.ProxyHandler({}) + url_opener = urllib2.build_opener(proxy_support) + for url in urls: - body = urllib.urlopen(url).read() + body = url_opener.open(url, timeout=self.timeout).read() tile = Verbatim(body) if len(urls) == 1: @@ -280,12 +293,16 @@ class UrlTemplate: - source projection (optional) Projection to transform coordinates into before making request + - timeout (optional) + Defines a timeout in seconds for the request. + If not defined, the global default timeout setting will be used. More on string substitutions: - http://docs.python.org/library/string.html#template-strings """ - def __init__(self, layer, template, referer=None, source_projection=None): + def __init__(self, layer, template, referer=None, source_projection=None, + timeout=None): """ Initialize a UrlTemplate provider with layer and template string. http://docs.python.org/library/string.html#template-strings @@ -294,6 +311,7 @@ def __init__(self, layer, template, referer=None, source_projection=None): self.template = Template(template) self.referer = referer self.source_projection = source_projection + self.timeout = timeout @staticmethod def prepareKeywordArgs(config_dict): @@ -307,6 +325,9 @@ def prepareKeywordArgs(config_dict): if 'source projection' in config_dict: kwargs['source_projection'] = Geography.getProjectionByName(config_dict['source projection']) + if 'timeout' in config_dict: + kwargs['timeout'] = config_dict['timeout'] + return kwargs def renderArea(self, width, height, srs, xmin, ymin, xmax, ymax, zoom): @@ -334,7 +355,7 @@ def renderArea(self, width, height, srs, xmin, ymin, xmax, ymax, zoom): if self.referer: req.add_header('Referer', self.referer) - body = urllib2.urlopen(req).read() + body = urllib2.urlopen(req, timeout=self.timeout).read() tile = Verbatim(body) return tile From 1ca673a245855fdea14222042e0e9e1b74c64300 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Wed, 21 May 2014 16:11:04 -0400 Subject: [PATCH 045/344] deleting an unnecessary logging statement --- TileStache/Goodies/VecTiles/oscimap.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/oscimap.py b/TileStache/Goodies/VecTiles/oscimap.py index 7f2c523f..11992634 100644 --- a/TileStache/Goodies/VecTiles/oscimap.py +++ b/TileStache/Goodies/VecTiles/oscimap.py @@ -90,7 +90,6 @@ def addFeature(self, row, coord, this_layer): layer = None # add layer tag tags.append(self.getTagId(('layer_name', this_layer))) - for tag in row[1].iteritems(): if tag[1] is None: continue @@ -102,7 +101,6 @@ def addFeature(self, row, coord, this_layer): continue tag = fixTag(tag, coord.zoom) - logging.info(tag) if tag is None: continue From bb2c4eca489198fa0a16124e6eef47e37bfa91e2 Mon Sep 17 00:00:00 2001 From: Peter Richardson Date: Thu, 22 May 2014 15:33:29 -0400 Subject: [PATCH 046/344] added -L to example curl command --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5795f042..376b9646 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Install the pure python modules with pip: Install pip (http://www.pip-installer.org/) like: - curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py + curl -O -L https://raw.github.com/pypa/pip/master/contrib/get-pip.py sudo python get-pip.py Install Mapnik via instructions at: From c38b6e700484825e5c42b8d9cf14f9ad6e9d596f Mon Sep 17 00:00:00 2001 From: Brett Camper Date: Sun, 25 May 2014 22:43:35 -0400 Subject: [PATCH 047/344] VecTiles: constrain output geometry to the same type as the input N.B.: adding this because of cases where line + tile bounding box intersections return points (instead of lines), which can be difficult to handle client-side; cleaner to make the assumption that a query for lines will always return lines --- TileStache/Goodies/VecTiles/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 700456c0..cfaff563 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -472,5 +472,6 @@ def build_query(srid, subquery, subcolumns, bbox, tolerance, is_geo, is_clipped, ) AS q WHERE ST_IsValid(q.__geometry__) AND q.__geometry__ && %(bbox)s - AND ST_Intersects(q.__geometry__, %(bbox)s)''' \ + AND ST_Intersects(q.__geometry__, %(bbox)s) + AND GeometryType(q.__geometry__) = GeometryType(%(geom)s)''' \ % locals() From 7c2bd2d1a939a79d6bca859ad8f1c998ae9c9b96 Mon Sep 17 00:00:00 2001 From: Brett Camper Date: Mon, 26 May 2014 16:33:37 -0400 Subject: [PATCH 048/344] it's OK to mix 'multi' geometry of the same type (e.g. MULTIPOLYGON source geoms outputting to POLYGON) --- TileStache/Goodies/VecTiles/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index cfaff563..9ad25420 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -473,5 +473,5 @@ def build_query(srid, subquery, subcolumns, bbox, tolerance, is_geo, is_clipped, WHERE ST_IsValid(q.__geometry__) AND q.__geometry__ && %(bbox)s AND ST_Intersects(q.__geometry__, %(bbox)s) - AND GeometryType(q.__geometry__) = GeometryType(%(geom)s)''' \ + AND REPLACE(GeometryType(q.__geometry__), 'MULTI', '') = REPLACE(GeometryType(%(geom)s), 'MULTI', '')''' \ % locals() From 33af74ee528897675500e7a544573ab6fbd5ecef Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Tue, 27 May 2014 17:51:25 -0400 Subject: [PATCH 049/344] drop null valued properties. --- TileStache/Goodies/VecTiles/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 700456c0..efa1ad1e 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -426,7 +426,7 @@ def get_features(dbinfo, query): wkb = bytes(row['__geometry__']) prop = dict([(k, v) for (k, v) in row.items() - if k not in ('__geometry__', '__id__')]) + if (k not in ('__geometry__', '__id__') and v is not None)]) if '__id__' in row: features.append((wkb, prop, row['__id__'])) From a55714322ad2257946d5b4a92924636d76687f21 Mon Sep 17 00:00:00 2001 From: Evan Babb Date: Wed, 28 May 2014 21:48:37 +0000 Subject: [PATCH 050/344] more informative KnownUnknown when layer.GetSpatialRef() returns None --- TileStache/Vector/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Vector/__init__.py b/TileStache/Vector/__init__.py index 9ed0d491..6ab0b080 100644 --- a/TileStache/Vector/__init__.py +++ b/TileStache/Vector/__init__.py @@ -450,7 +450,7 @@ def _open_layer(driver_name, parameters, dirpath): layer = datasource.GetLayer(0) if layer.GetSpatialRef() is None and driver_name != 'SQLite': - raise KnownUnknown('Couldn\'t get a layer from data source %s' % source_name) + raise KnownUnknown('The layer has no spatial reference: %s' % source_name) # # Return the layer and the datasource. From c473ea46f38d18ec79982f3a2709714d585bf7d5 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Tue, 10 Jun 2014 15:04:50 -0400 Subject: [PATCH 051/344] changing the layername of the layer from the default config.custom_layer_name to the layername combination (better caching key uniqueness for a given z/x/y) --- TileStache/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/TileStache/__init__.py b/TileStache/__init__.py index 0a3bf867..cf7f4e8a 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -197,9 +197,11 @@ def requestLayer(config, path_info): custom_layer = layername.find(_delimiter)!=-1 if custom_layer: - config.layers[config.custom_layer_name].provider(config.layers[config.custom_layer_name], **{'names': layername.split(_delimiter)}) - - return config.layers[layername] if not custom_layer else config.layers[config.custom_layer_name] + config.layers[layername] = config.layers[config.custom_layer_name] + config.layers[layername].provider(config.layers[layername], **{'names': layername.split(_delimiter)}) + del config.layers[config.custom_layer_name] + + return config.layers[layername] def requestHandler(config_hint, path_info, query_string=None): """ Generate a mime-type and response body for a given request. From f43db7e9c846c5d372ad0cf9f075a2016f8b8cd1 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Tue, 17 Jun 2014 14:09:07 -0400 Subject: [PATCH 052/344] fixing the bug addFeatures() takes at most 4 arguments (5 given) --- TileStache/Goodies/VecTiles/mapbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index 1a1bca82..fd719106 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -28,7 +28,7 @@ def decode(file): def encode(file, features, coord, layer_name): tile = VectorTile(extents) - tile.addFeatures(features, coord, extents, layer_name) + tile.addFeatures(features, coord, layer_name) data = tile.tile.SerializeToString() file.write(data) From c322a27c5fc66dc70c27ef271afc570430a2d404 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 18 Jul 2014 13:52:16 -0400 Subject: [PATCH 053/344] update strategy for getting column names * if there were ever no results, we would fail later on because we wouldn't have any column names * this should be faster too, because we never have to issue repeat queries with alternate bounds --- TileStache/Goodies/VecTiles/server.py | 34 +++++++-------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 818ca783..55468635 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -391,31 +391,15 @@ def query_columns(dbinfo, srid, subquery, bounds): ''' Get information about the columns returned for a subquery. ''' with Connection(dbinfo) as db: - # - # While bounds covers less than the full planet, look for just one feature. - # - while (abs(bounds[2] - bounds[0]) * abs(bounds[2] - bounds[0])) < 1.61e15: - bbox = 'ST_MakeBox2D(ST_MakePoint(%f, %f), ST_MakePoint(%f, %f))' % bounds - bbox = 'ST_SetSRID(%s, %d)' % (bbox, srid) - - query = subquery.replace('!bbox!', bbox) - - db.execute(query + '\n LIMIT 1') # newline is important here, to break out of comments. - row = db.fetchone() - - if row is None: - # - # Try zooming out three levels (8x) to look for features. - # - bounds = (bounds[0] - (bounds[2] - bounds[0]) * 3.5, - bounds[1] - (bounds[3] - bounds[1]) * 3.5, - bounds[2] + (bounds[2] - bounds[0]) * 3.5, - bounds[3] + (bounds[3] - bounds[1]) * 3.5) - - continue - - column_names = set(row.keys()) - return column_names + bbox = 'ST_MakeBox2D(ST_MakePoint(%f, %f), ST_MakePoint(%f, %f))' % bounds + bbox = 'ST_SetSRID(%s, %d)' % (bbox, srid) + + query = subquery.replace('!bbox!', bbox) + + # newline is important here, to break out of comments. + db.execute(query + '\n LIMIT 0') + column_names = set(x.name for x in db.description) + return column_names def get_features(dbinfo, query): with Connection(dbinfo) as db: From b6a8857fdd901dafbe34a16f9953c22137cf4db5 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 12 Aug 2014 11:18:04 -0400 Subject: [PATCH 054/344] default to an empty string for mapbox layer names --- TileStache/Goodies/VecTiles/mapbox.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index d87a6186..8b210e3c 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -29,10 +29,10 @@ def decode(file): ''' raise NotImplementedError('mapbox.decode() not yet written') -def encode(file, features, coord, layer_name): +def encode(file, features, coord, layer_name=''): tile = VectorTile(extents) - tile.addFeatures(features, coord, layer_name) + tile.addFeatures(features, coord, layer_name or '') data = tile.tile.SerializeToString() file.write(data) From c843a5037e2ba55aa5be89c598ac98aa6c7a551c Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 12 Aug 2014 11:49:17 -0400 Subject: [PATCH 055/344] default to empty string for oscimap too --- TileStache/Goodies/VecTiles/oscimap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/oscimap.py b/TileStache/Goodies/VecTiles/oscimap.py index fb9f585b..c14fed5e 100644 --- a/TileStache/Goodies/VecTiles/oscimap.py +++ b/TileStache/Goodies/VecTiles/oscimap.py @@ -21,7 +21,8 @@ # tiles are padded by this number of pixels for the current zoom level (OSciMap uses this to cover up seams between tiles) padding = 5 -def encode(file, features, coord, layer_name): +def encode(file, features, coord, layer_name=''): + layer_name = layer_name or '' tile = VectorTile(extents) for feature in features: From 7a6763b4c1517d9de26b82b00e84fa02b667364f Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 23 Jul 2014 17:48:00 -0400 Subject: [PATCH 056/344] remove redundant bbox intersection clause Postgis performs a bounding box check as part of its ST_Intersects implementation. http://lists.osgeo.org/pipermail/postgis-users/2010-September/027675.html --- TileStache/Goodies/VecTiles/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 55468635..8a89f12f 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -459,7 +459,6 @@ def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clippe %(subquery)s ) AS q WHERE ST_IsValid(q.__geometry__) - AND q.__geometry__ && %(bbox)s AND ST_Intersects(q.__geometry__, %(bbox)s) AND REPLACE(GeometryType(q.__geometry__), 'MULTI', '') = REPLACE(GeometryType(%(geom)s), 'MULTI', '')''' \ % locals() From 54555698e38983a25c3e1b4469c19c0c216f02de Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 31 Jul 2014 14:54:55 -0400 Subject: [PATCH 057/344] suppress simplification at particular zooms levels --- TileStache/Goodies/VecTiles/server.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 8a89f12f..15f4875d 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -76,7 +76,11 @@ class Provider: simplify_until: Optional integer specifying a zoom level where no more geometry simplification should occur. Default 16. - + + suppress_simplification: + Optional list of zoom levels where no dynamic simplification should + occur. + Sample configuration, for a layer with no results at zooms 0-9, basic selection of lines with names and highway tags for zoom 10, a remote URL containing a query for zoom 11, and a local file for zooms 12+: @@ -104,7 +108,7 @@ class Provider: } } ''' - def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, simplify_until=16): + def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, simplify_until=16, suppress_simplification=()): ''' ''' self.layer = layer @@ -116,7 +120,8 @@ def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, self.srid = int(srid) self.simplify = float(simplify) self.simplify_until = int(simplify_until) - + self.suppress_simplification = set(suppress_simplification) + self.queries = [] self.columns = {} @@ -157,7 +162,10 @@ def renderTile(self, width, height, srs, coord): if query not in self.columns: self.columns[query] = query_columns(self.dbinfo, self.srid, query, bounds) - tolerance = self.simplify * tolerances[coord.zoom] if coord.zoom < self.simplify_until else None + if coord.zoom in self.suppress_simplification: + tolerance = None + else: + tolerance = self.simplify * tolerances[coord.zoom] if coord.zoom < self.simplify_until else None return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name()) From d01e7f0955ead790228f1c8d8cb55eebb127db73 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 31 Jul 2014 14:23:09 -0400 Subject: [PATCH 058/344] specify allowed geometry types per layer Configuring the layers in advance prevents the query from having to perform a check to ensure that the geometry type returned is consistent. --- TileStache/Goodies/VecTiles/server.py | 58 ++++++++++++++++----------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 15f4875d..b3351d72 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -11,6 +11,7 @@ from urlparse import urljoin, urlparse from urllib import urlopen from os.path import exists +from shapely.wkb import loads import json from ... import getTile @@ -81,6 +82,10 @@ class Provider: Optional list of zoom levels where no dynamic simplification should occur. + geometry_types: + Optional list of geometry types that constrains the results of what + kind of features are returned. + Sample configuration, for a layer with no results at zooms 0-9, basic selection of lines with names and highway tags for zoom 10, a remote URL containing a query for zoom 11, and a local file for zooms 12+: @@ -108,7 +113,7 @@ class Provider: } } ''' - def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, simplify_until=16, suppress_simplification=()): + def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, simplify_until=16, suppress_simplification=(), geometry_types=None): ''' ''' self.layer = layer @@ -121,6 +126,7 @@ def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, self.simplify = float(simplify) self.simplify_until = int(simplify_until) self.suppress_simplification = set(suppress_simplification) + self.geometry_types = None if geometry_types is None else set(geometry_types) self.queries = [] self.columns = {} @@ -167,7 +173,7 @@ def renderTile(self, width, height, srs, coord): else: tolerance = self.simplify * tolerances[coord.zoom] if coord.zoom < self.simplify_until else None - return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name()) + return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name(), self.geometry_types) def getTypeByExtension(self, extension): ''' Get mime-type and format by file extension, one of "mvt", "json" or "topojson". @@ -260,7 +266,7 @@ def __exit__(self, type, value, traceback): class Response: ''' ''' - def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip, coord, layer_name): + def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip, coord, layer_name, geometry_types): ''' Create a new response object with Postgres connection info and a query. bounds argument is a 4-tuple with (xmin, ymin, xmax, ymax). @@ -271,6 +277,7 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli self.clip = clip self.coord= coord self.layer_name = layer_name + self.geometry_types = geometry_types geo_query = build_query(srid, subquery, columns, bounds, tolerance, True, clip) merc_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip) @@ -281,7 +288,7 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli def save(self, out, format): ''' ''' - features = get_features(self.dbinfo, self.query[format]) + features = get_features(self.dbinfo, self.query[format], self.geometry_types) if format == 'MVT': mvt.encode(out, features) @@ -341,6 +348,7 @@ def __init__(self, config, names, coord): self.config = config self.names = names self.coord = coord + def save(self, out, format): ''' ''' @@ -357,7 +365,7 @@ def save(self, out, format): width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["OpenScienceMap"])}) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["OpenScienceMap"], layer.provider.geometry_types)}) oscimap.merge(out, feature_layers, self.coord) elif format == 'Mapbox': @@ -367,7 +375,7 @@ def save(self, out, format): width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["Mapbox"])}) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["Mapbox"], layer.provider.geometry_types)}) mapbox.merge(out, feature_layers, self.coord) else: @@ -409,25 +417,28 @@ def query_columns(dbinfo, srid, subquery, bounds): column_names = set(x.name for x in db.description) return column_names -def get_features(dbinfo, query): +def get_features(dbinfo, query, geometry_types): + features = [] + with Connection(dbinfo) as db: db.execute(query) - - features = [] - for row in db.fetchall(): - if row['__geometry__'] is None: - continue - - wkb = bytes(row['__geometry__']) - prop = dict([(k, v) for (k, v) in row.items() - if (k not in ('__geometry__', '__id__') and v is not None)]) - - if '__id__' in row: - features.append((wkb, prop, row['__id__'])) - - else: - features.append((wkb, prop)) + assert '__geometry__' in row, 'Missing __geometry__ in feature result' + assert '__id__' in row, 'Missing __id__ in feature result' + + wkb = bytes(row.pop('__geometry__')) + id = row.pop('__id__') + + if geometry_types is not None: + shape = loads(wkb) + geom_type = shape.__geo_interface__['type'] + if geom_type not in geometry_types: + #print 'found %s which is not in: %s' % (geom_type, geometry_types) + continue + + props = dict((k, v) for k, v in row.items() if v is not None) + features.append((wkb, props, id)) + return features def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clipped, padding=0, scale=None): @@ -467,6 +478,5 @@ def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clippe %(subquery)s ) AS q WHERE ST_IsValid(q.__geometry__) - AND ST_Intersects(q.__geometry__, %(bbox)s) - AND REPLACE(GeometryType(q.__geometry__), 'MULTI', '') = REPLACE(GeometryType(%(geom)s), 'MULTI', '')''' \ + AND ST_Intersects(q.__geometry__, %(bbox)s)''' \ % locals() From 8259630d0e15135aff79eff7a0c4392c1a000c55 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Sun, 31 Aug 2014 10:58:11 -0400 Subject: [PATCH 059/344] sort by id to ensure stable results ordering --- TileStache/Goodies/VecTiles/server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index b3351d72..c8ec3d59 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -14,6 +14,7 @@ from shapely.wkb import loads import json +import operator from ... import getTile from ...Core import KnownUnknown @@ -439,6 +440,9 @@ def get_features(dbinfo, query, geometry_types): props = dict((k, v) for k, v in row.items() if v is not None) features.append((wkb, props, id)) + # sort features by id to ensure stable ordering + features.sort(key=operator.itemgetter(2)) + return features def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clipped, padding=0, scale=None): From 4820e89f463c98fa70e69ec52f95d40cdc6740de Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 3 Sep 2014 15:38:02 -0400 Subject: [PATCH 060/344] vtm formatter expects integer heights, * 100 --- TileStache/Goodies/VecTiles/oscimap.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/oscimap.py b/TileStache/Goodies/VecTiles/oscimap.py index c14fed5e..1314a7f7 100644 --- a/TileStache/Goodies/VecTiles/oscimap.py +++ b/TileStache/Goodies/VecTiles/oscimap.py @@ -94,10 +94,19 @@ def addFeature(self, row, coord, this_layer): layer = None # add layer tag tags.append(self.getTagId(('layer_name', this_layer))) - for tag in row[1].iteritems(): - if tag[1] is None: + for k, v in row[1].iteritems(): + if v is None: continue - tag = (str(tag[0]), str(tag[1])) + + # the vtm stylesheet expects the heights to be an integer, + # multiplied by 100 + if this_layer == 'buildings' and k in ('height', 'min_height'): + try: + v = int(v * 100) + except ValueError: + logging.warning('vtm: Invalid %s value: %s' % (k, v)) + + tag = str(k), str(v) # use unsigned int for layer. i.e. map to 0..10 if "layer" == tag[0]: From 2ec4b87af960dc3dc7fd3c90837a448d577e1d3c Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 22 Oct 2014 16:02:27 -0400 Subject: [PATCH 061/344] Remove feature sort by id The queries will now all have their own order by, which allows them to sort on whatever makes more sense. --- TileStache/Goodies/VecTiles/server.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index c8ec3d59..b3351d72 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -14,7 +14,6 @@ from shapely.wkb import loads import json -import operator from ... import getTile from ...Core import KnownUnknown @@ -440,9 +439,6 @@ def get_features(dbinfo, query, geometry_types): props = dict((k, v) for k, v in row.items() if v is not None) features.append((wkb, props, id)) - # sort features by id to ensure stable ordering - features.sort(key=operator.itemgetter(2)) - return features def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clipped, padding=0, scale=None): From 3c013ceac0666f3e1ecc005901f4baa488720ba1 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 7 Nov 2014 11:46:56 -0500 Subject: [PATCH 062/344] Resolve error on multiple layer requests In certain deployment models, the configuration is shared between requests. The custom layer should not be removed or subsequent requests will result in errors. --- TileStache/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/TileStache/__init__.py b/TileStache/__init__.py index 69bc926b..4373bcea 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -206,7 +206,6 @@ def requestLayer(config, path_info): if custom_layer: config.layers[layername] = config.layers[config.custom_layer_name] config.layers[layername].provider(config.layers[layername], **{'names': layername.split(_delimiter)}) - del config.layers[config.custom_layer_name] return config.layers[layername] From 112d84f82fc58da642a4a028b58753d0e70a0d38 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 7 Nov 2014 17:51:23 -0500 Subject: [PATCH 063/344] Copy layer instead of referring to it Making an explicit copy prevents getting the wrong layer name. --- TileStache/__init__.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/TileStache/__init__.py b/TileStache/__init__.py index 4373bcea..ddbf7b32 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -204,11 +204,40 @@ def requestLayer(config, path_info): custom_layer = layername.find(_delimiter)!=-1 if custom_layer: - config.layers[layername] = config.layers[config.custom_layer_name] - config.layers[layername].provider(config.layers[layername], **{'names': layername.split(_delimiter)}) + # we can't just assign references, because we get identity problems + # when tilestache tries to look up the layer's name, which won't match + # the list of names in the provider + provider_names = layername.split(_delimiter) + custom_layer_obj = config.layers[config.custom_layer_name] + config.layers[layername] = clone_layer(custom_layer_obj, provider_names) return config.layers[layername] + +def clone_layer(layer, provider_names): + from TileStache.Core import Layer + copy = Layer( + layer.config, + layer.projection, + layer.metatile, + layer.stale_lock_timeout, + layer.cache_lifespan, + layer.write_cache, + layer.allowed_origin, + layer.max_cache_age, + layer.redirects, + layer.preview_lat, + layer.preview_lon, + layer.preview_zoom, + layer.preview_ext, + layer.bounds, + layer.dim, + ) + copy.provider = layer.provider + copy.provider(copy, provider_names) + return copy + + def requestHandler(config_hint, path_info, query_string=None): """ Generate a mime-type and response body for a given request. From 3bd1225a5e19b3f33cf21967b0c86c1fa5d1a38a Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 20 Nov 2014 11:29:09 -0500 Subject: [PATCH 064/344] Update mapbox content type response header --- TileStache/Goodies/VecTiles/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index b3351d72..691b9e99 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -191,7 +191,7 @@ def getTypeByExtension(self, extension): return 'image/png', 'OpenScienceMap' # TODO: make this proper stream type, app only seems to work with png elif extension.lower() == 'mapbox': - return 'image/png', 'Mapbox' + return 'application/x-protobuf', 'Mapbox' else: raise ValueError(extension + " is not a valid extension") @@ -242,7 +242,7 @@ def getTypeByExtension(self, extension): return 'image/png', 'OpenScienceMap' # TODO: make this proper stream type, app only seems to work with png elif extension.lower() == 'mapbox': - return 'image/png', 'Mapbox' + return 'application/x-protobuf', 'Mapbox' else: raise ValueError(extension + " is not a valid extension for responses with multiple layers") From d6a951237196acc04fb181a47cea42743edeea99 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Tue, 2 Dec 2014 18:52:00 -0500 Subject: [PATCH 065/344] using mapbox-vector-tile pip package --- TileStache/Goodies/VecTiles/mapbox.py | 297 +++----------------------- 1 file changed, 28 insertions(+), 269 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index 8b210e3c..634b4c82 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -1,294 +1,53 @@ -import types -from Mapbox import vector_tile_pb2 -from shapely.wkb import loads - -from math import floor, fabs -from array import array - from TileStache.Core import KnownUnknown import re import logging +import mapbox_vector_tile + # coordindates are scaled to this range within tile extents = 4096 # tiles are padded by this number of pixels for the current zoom level padding = 0 -cmd_bits = 3 -tolerance = 0 - -CMD_MOVE_TO = 1 -CMD_LINE_TO = 2 -CMD_SEG_END = 7 - def decode(file): - ''' Stub function to decode a mapbox vector tile file into a list of features. - - Not currently implemented, modeled on geojson.decode(). - ''' - raise NotImplementedError('mapbox.decode() not yet written') + tile = file.read() + data = mapbox_vector_tile.decode(tile) + return data # print data or write to file? def encode(file, features, coord, layer_name=''): - tile = VectorTile(extents) + layers = [] - tile.addFeatures(features, coord, layer_name or '') - - data = tile.tile.SerializeToString() - file.write(data) + layers.append(get_feature_layer(layer_name, features)) + + data = mapbox_vector_tile.encode(layers) + file.write(data) def merge(file, feature_layers, coord): - ''' Retrieve a list of GeoJSON tile responses and merge them into one. + ''' Retrieve a list of mapbox tile responses and merge them into one. get_tiles() retrieves data and performs basic integrity checks. ''' - tile = VectorTile(extents) - + layers = [] + for layer in feature_layers: - tile.addFeatures(layer['features'], coord, layer['name']) - - data = tile.tile.SerializeToString() + layers.append(get_feature_layer(layer['name'], layer['features'])) + + data = mapbox_vector_tile.encode(layers) file.write(data) -class VectorTile: - """ - """ - def __init__(self, extents, layer_name=""): - self.tile = vector_tile_pb2.tile() - self.extents = extents - - def addFeatures(self, features, coord, layer_name=""): - self.layer = self.tile.layers.add() - self.layer.name = layer_name - self.layer.version = 2 - self.layer.extent = self.extents - self.feature_count = 0 - self.keys = [] - self.values = [] - - for feature in features: - self.addFeature(feature) +def get_feature_layer(name, features): + features_ = [] - def addFeature(self, feature): - f = self.layer.features.add() - self.feature_count += 1 - f.id = self.feature_count - - # osm_id or the hash is passed in as a feature tag 'uid' + for feature in features: if len(feature) >= 2: feature[1].update(uid=feature[2]) - - # properties - self._handle_attr(self.layer, f, feature[1]) - - # geometry - shape = loads(feature[0]) - f.type = self._get_feature_type(shape) - self._geo_encode(f, shape) - - def _get_feature_type(self, shape): - if shape.type == 'Point' or shape.type == 'MultiPoint': - return self.tile.Point - elif shape.type == 'LineString' or shape.type == 'MultiLineString': - return self.tile.LineString - elif shape.type == 'Polygon' or shape.type == 'MultiPolygon': - return self.tile.Polygon - - def _encode_cmd_length(self, cmd, length): - return (length << cmd_bits) | (cmd & ((1 << cmd_bits) - 1)) - - def _chunker(self, seq, size): - return [seq[pos:pos + size] for pos in xrange(0, len(seq), size)] - - def _handle_attr(self, layer, feature, props): - for k,v in props.items(): - if v is not None: - if k not in self.keys: - layer.keys.append(k) - self.keys.append(k) - feature.tags.append(self.keys.index(k)) - if v not in self.values: - self.values.append(v) - if (isinstance(v,bool)): - val = layer.values.add() - val.bool_value = v - elif (isinstance(v,str)) or (isinstance(v,unicode)): - val = layer.values.add() - val.string_value = unicode(v,'utf8') - elif (isinstance(v,int)) or (isinstance(v,long)): - val = layer.values.add() - val.int_value = v - elif (isinstance(v,float)): - val = layer.values.add() - d_arr = array('d', [v]) - # google pbf expects big endian by default - d_arr.byteswap() - val.double_value = d_arr[0] - # else: - # # do nothing because we know kind is sometimes - # logging.info("Unknown value type: '%s' for key: '%s'", type(v), k) - # raise Exception("Unknown value type: '%s'" % type(v)) - feature.tags.append(self.values.index(v)) - - def _handle_skipped_last(self, f, skipped_index, cur_x, cur_y, x_, y_): - last_x = f.geometry[skipped_index - 2] - last_y = f.geometry[skipped_index - 1] - last_dx = ((last_x >> 1) ^ (-(last_x & 1))) - last_dy = ((last_y >> 1) ^ (-(last_y & 1))) - dx = cur_x - x_ + last_dx - dy = cur_y - y_ + last_dy - x_ = cur_x - y_ = cur_y - f.geometry.__setitem__(skipped_index - 2, ((dx << 1) ^ (dx >> 31))) - f.geometry.__setitem__(skipped_index - 1, ((dy << 1) ^ (dy >> 31))) - - def _parseGeometry(self, shape): - coordinates = [] - line = "line" - polygon = "polygon" - - def _get_point_obj(x, y, cmd=CMD_MOVE_TO): - coordinate = { - 'x' : x, - 'y' : self.extents - y, - 'cmd': cmd - } - coordinates.append(coordinate) - - def _get_arc_obj(arc, type): - length = len(arc.coords) - iterator=0 - cmd = CMD_MOVE_TO - while (iterator < length): - x = arc.coords[iterator][0] - y = arc.coords[iterator][1] - if iterator == 0: - cmd = CMD_MOVE_TO - elif iterator == length-1 and type == polygon: - cmd = CMD_SEG_END - else: - cmd = CMD_LINE_TO - _get_point_obj(x, y, cmd) - iterator = iterator + 1 - - if shape.type == 'GeometryCollection': - # do nothing - coordinates = [] - - elif shape.type == 'Point': - _get_point_obj(shape.x,shape.y) - - elif shape.type == 'LineString': - _get_arc_obj(shape, line) - - elif shape.type == 'Polygon': - rings = [shape.exterior] + list(shape.interiors) - for ring in rings: - _get_arc_obj(ring, polygon) - - elif shape.type == 'MultiPoint': - for point in shape.geoms: - _get_point_obj(point.x, point.y) - - elif shape.type == 'MultiLineString': - for arc in shape.geoms: - _get_arc_obj(arc, line) - - elif shape.type == 'MultiPolygon': - for polygon in shape.geoms: - rings = [polygon.exterior] + list(polygon.interiors) - for ring in rings: - _get_arc_obj(ring, polygon) - - else: - raise NotImplementedError("Can't do %s geometries" % shape.type) - - return coordinates - - def _geo_encode(self, f, shape): - x_, y_ = 0, 0 - - cmd= -1 - cmd_idx = -1 - vtx_cmd = -1 - prev_cmd= -1 - - skipped_index = -1 - skipped_last = False - cur_x = 0 - cur_y = 0 - - it = 0 - length = 0 - - coordinates = self._parseGeometry(shape) - - while (True): - if it >= len(coordinates): - break; - - x,y,vtx_cmd = coordinates[it]['x'],coordinates[it]['y'],coordinates[it]['cmd'] - - if vtx_cmd != cmd: - if (cmd_idx >= 0): - f.geometry.__setitem__(cmd_idx, self._encode_cmd_length(cmd, length)) - - cmd = vtx_cmd - length = 0 - cmd_idx = len(f.geometry) - f.geometry.append(0) #placeholder added in first pass - - if (vtx_cmd == CMD_MOVE_TO or vtx_cmd == CMD_LINE_TO): - if cmd == CMD_MOVE_TO and skipped_last and skipped_index >1: - self._handle_skipped_last(f, skipped_index, cur_x, cur_y, x_, y_) - - # Compute delta to the previous coordinate. - cur_x = int(x) - cur_y = int(y) - - dx = cur_x - x_ - dy = cur_y - y_ - - sharp_turn_ahead = False - - if (it+2 <= len(coordinates)): - next_coord = coordinates[it+1] - if next_coord['cmd'] == CMD_LINE_TO: - next_x, next_y = next_coord['x'], next_coord['y'] - next_dx = fabs(cur_x - int(next_x)) - next_dy = fabs(cur_y - int(next_y)) - if ((next_dx == 0 and next_dy >= tolerance) or (next_dy == 0 and next_dx >= tolerance)): - sharp_turn_ahead = True - - # Keep all move_to commands, but omit other movements that are - # not >= the tolerance threshold and should be considered no-ops. - # NOTE: length == 0 indicates the command has changed and will - # preserve any non duplicate move_to or line_to - if length == 0 or sharp_turn_ahead or fabs(dx) >= tolerance or fabs(dy) >= tolerance: - # Manual zigzag encoding. - f.geometry.append((dx << 1) ^ (dx >> 31)) - f.geometry.append((dy << 1) ^ (dy >> 31)) - x_ = cur_x - y_ = cur_y - skipped_last = False - length = length + 1 - else: - skipped_last = True - skipped_index = len(f.geometry) - elif vtx_cmd == CMD_SEG_END: - if prev_cmd != CMD_SEG_END: - length = length + 1 - else: - raise Exception("Unknown command type: '%s'" % vtx_cmd) - - it = it + 1 - prev_cmd = cmd - - # at least one vertex + cmd/length - if (skipped_last and skipped_index > 1): - # if we skipped previous vertex we just update it to the last one here. - self._handle_skipped_last(f, skipped_index, cur_x, cur_y, x_, y_) - - # Update the last length/command value. - if (cmd_idx >= 0): - f.geometry.__setitem__(cmd_idx, self._encode_cmd_length(cmd, length)) + features_.append({ + 'geometry': feature[0], + 'properties': feature[1] + }) + + return { + 'name': name or '', + 'features': features_ + } \ No newline at end of file From 4c0499f03e0e338fc87522724a329e491145f00d Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Tue, 2 Dec 2014 18:55:08 -0500 Subject: [PATCH 066/344] removing mapbox package dependency from setup.py and deleting mapbox module --- .../Goodies/VecTiles/Mapbox/__init__.py | 0 .../Goodies/VecTiles/Mapbox/vector_tile.proto | 92 ------ .../VecTiles/Mapbox/vector_tile_pb2.py | 298 ------------------ setup.py | 3 +- 4 files changed, 1 insertion(+), 392 deletions(-) delete mode 100644 TileStache/Goodies/VecTiles/Mapbox/__init__.py delete mode 100644 TileStache/Goodies/VecTiles/Mapbox/vector_tile.proto delete mode 100644 TileStache/Goodies/VecTiles/Mapbox/vector_tile_pb2.py diff --git a/TileStache/Goodies/VecTiles/Mapbox/__init__.py b/TileStache/Goodies/VecTiles/Mapbox/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/TileStache/Goodies/VecTiles/Mapbox/vector_tile.proto b/TileStache/Goodies/VecTiles/Mapbox/vector_tile.proto deleted file mode 100644 index 110f37c3..00000000 --- a/TileStache/Goodies/VecTiles/Mapbox/vector_tile.proto +++ /dev/null @@ -1,92 +0,0 @@ -// Protocol Version 1 - -package mapnik.vector; - -option optimize_for = LITE_RUNTIME; - -message tile { - enum GeomType { - Unknown = 0; - Point = 1; - LineString = 2; - Polygon = 3; - } - - // Variant type encoding - message value { - // Exactly one of these values may be present in a valid message - optional string string_value = 1; - optional float float_value = 2; - optional double double_value = 3; - optional int64 int_value = 4; - optional uint64 uint_value = 5; - optional sint64 sint_value = 6; - optional bool bool_value = 7; - - extensions 8 to max; - } - - message feature { - optional uint64 id = 1; - - // Tags of this feature. Even numbered values refer to the nth - // value in the keys list on the tile message, odd numbered - // values refer to the nth value in the values list on the tile - // message. - repeated uint32 tags = 2 [ packed = true ]; - - // The type of geometry stored in this feature. - optional GeomType type = 3 [ default = Unknown ]; - - // Contains a stream of commands and parameters (vertices). The - // repeat count is shifted to the left by 3 bits. This means - // that the command has 3 bits (0-7). The repeat count - // indicates how often this command is to be repeated. Defined - // commands are: - // - MoveTo: 1 (2 parameters follow) - // - LineTo: 2 (2 parameters follow) - // - ClosePath: 7 (no parameters follow) - // - // Ex.: MoveTo(3, 6), LineTo(8, 12), LineTo(20, 34), ClosePath - // Encoded as: [ 9 3 6 18 5 6 12 22 15 ] - // == command type 7 (ClosePath), length 1 - // ===== relative LineTo(+12, +22) == LineTo(20, 34) - // === relative LineTo(+5, +6) == LineTo(8, 12) - // == [00010 010] = command type 2 (LineTo), length 2 - // === relative MoveTo(+3, +6) - // == [00001 001] = command type 1 (MoveTo), length 1 - // Commands are encoded as uint32 varints, vertex parameters are - // encoded as sint32 varints (zigzag). Vertex parameters are - // also encoded as deltas to the previous position. The original - // position is (0,0) - repeated uint32 geometry = 4 [ packed = true ]; - } - - message layer { - // Any compliant implementation must first read the version - // number encoded in this message and choose the correct - // implementation for this version number before proceeding to - // decode other parts of this message. - required uint32 version = 15 [ default = 1 ]; - - required string name = 1; - - // The actual features in this tile. - repeated feature features = 2; - - // Dictionary encoding for keys - repeated string keys = 3; - - // Dictionary encoding for values - repeated value values = 4; - - // The bounding box in this tile spans from 0..4095 units - optional uint32 extent = 5 [ default = 4096 ]; - - extensions 16 to max; - } - - repeated layer layers = 3; - - extensions 16 to 8191; -} diff --git a/TileStache/Goodies/VecTiles/Mapbox/vector_tile_pb2.py b/TileStache/Goodies/VecTiles/Mapbox/vector_tile_pb2.py deleted file mode 100644 index 6c1bee2f..00000000 --- a/TileStache/Goodies/VecTiles/Mapbox/vector_tile_pb2.py +++ /dev/null @@ -1,298 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: vector_tile.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='vector_tile.proto', - package='mapnik.vector', - serialized_pb='\n\x11vector_tile.proto\x12\rmapnik.vector\"\xc5\x04\n\x04tile\x12)\n\x06layers\x18\x03 \x03(\x0b\x32\x19.mapnik.vector.tile.layer\x1a\xa1\x01\n\x05value\x12\x14\n\x0cstring_value\x18\x01 \x01(\t\x12\x13\n\x0b\x66loat_value\x18\x02 \x01(\x02\x12\x14\n\x0c\x64ouble_value\x18\x03 \x01(\x01\x12\x11\n\tint_value\x18\x04 \x01(\x03\x12\x12\n\nuint_value\x18\x05 \x01(\x04\x12\x12\n\nsint_value\x18\x06 \x01(\x12\x12\x12\n\nbool_value\x18\x07 \x01(\x08*\x08\x08\x08\x10\x80\x80\x80\x80\x02\x1ar\n\x07\x66\x65\x61ture\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x10\n\x04tags\x18\x02 \x03(\rB\x02\x10\x01\x12\x33\n\x04type\x18\x03 \x01(\x0e\x32\x1c.mapnik.vector.tile.GeomType:\x07Unknown\x12\x14\n\x08geometry\x18\x04 \x03(\rB\x02\x10\x01\x1a\xb1\x01\n\x05layer\x12\x12\n\x07version\x18\x0f \x02(\r:\x01\x31\x12\x0c\n\x04name\x18\x01 \x02(\t\x12-\n\x08\x66\x65\x61tures\x18\x02 \x03(\x0b\x32\x1b.mapnik.vector.tile.feature\x12\x0c\n\x04keys\x18\x03 \x03(\t\x12)\n\x06values\x18\x04 \x03(\x0b\x32\x19.mapnik.vector.tile.value\x12\x14\n\x06\x65xtent\x18\x05 \x01(\r:\x04\x34\x30\x39\x36*\x08\x08\x10\x10\x80\x80\x80\x80\x02\"?\n\x08GeomType\x12\x0b\n\x07Unknown\x10\x00\x12\t\n\x05Point\x10\x01\x12\x0e\n\nLineString\x10\x02\x12\x0b\n\x07Polygon\x10\x03*\x05\x08\x10\x10\x80@B\x02H\x03') - - - -_TILE_GEOMTYPE = _descriptor.EnumDescriptor( - name='GeomType', - full_name='mapnik.vector.tile.GeomType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='Unknown', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Point', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LineString', index=2, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Polygon', index=3, number=3, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=548, - serialized_end=611, -) - - -_TILE_VALUE = _descriptor.Descriptor( - name='value', - full_name='mapnik.vector.tile.value', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='string_value', full_name='mapnik.vector.tile.value.string_value', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='float_value', full_name='mapnik.vector.tile.value.float_value', index=1, - number=2, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='double_value', full_name='mapnik.vector.tile.value.double_value', index=2, - number=3, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='int_value', full_name='mapnik.vector.tile.value.int_value', index=3, - number=4, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='uint_value', full_name='mapnik.vector.tile.value.uint_value', index=4, - number=5, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='sint_value', full_name='mapnik.vector.tile.value.sint_value', index=5, - number=6, type=18, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bool_value', full_name='mapnik.vector.tile.value.bool_value', index=6, - number=7, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=True, - extension_ranges=[(8, 536870912), ], - serialized_start=89, - serialized_end=250, -) - -_TILE_FEATURE = _descriptor.Descriptor( - name='feature', - full_name='mapnik.vector.tile.feature', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='id', full_name='mapnik.vector.tile.feature.id', index=0, - number=1, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='tags', full_name='mapnik.vector.tile.feature.tags', index=1, - number=2, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), - _descriptor.FieldDescriptor( - name='type', full_name='mapnik.vector.tile.feature.type', index=2, - number=3, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='geometry', full_name='mapnik.vector.tile.feature.geometry', index=3, - number=4, type=13, cpp_type=3, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001')), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=252, - serialized_end=366, -) - -_TILE_LAYER = _descriptor.Descriptor( - name='layer', - full_name='mapnik.vector.tile.layer', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='version', full_name='mapnik.vector.tile.layer.version', index=0, - number=15, type=13, cpp_type=3, label=2, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='name', full_name='mapnik.vector.tile.layer.name', index=1, - number=1, type=9, cpp_type=9, label=2, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='features', full_name='mapnik.vector.tile.layer.features', index=2, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='keys', full_name='mapnik.vector.tile.layer.keys', index=3, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='values', full_name='mapnik.vector.tile.layer.values', index=4, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='extent', full_name='mapnik.vector.tile.layer.extent', index=5, - number=5, type=13, cpp_type=3, label=1, - has_default_value=True, default_value=4096, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=True, - extension_ranges=[(16, 536870912), ], - serialized_start=369, - serialized_end=546, -) - -_TILE = _descriptor.Descriptor( - name='tile', - full_name='mapnik.vector.tile', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='layers', full_name='mapnik.vector.tile.layers', index=0, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_TILE_VALUE, _TILE_FEATURE, _TILE_LAYER, ], - enum_types=[ - _TILE_GEOMTYPE, - ], - options=None, - is_extendable=True, - extension_ranges=[(16, 8192), ], - serialized_start=37, - serialized_end=618, -) - -_TILE_VALUE.containing_type = _TILE; -_TILE_FEATURE.fields_by_name['type'].enum_type = _TILE_GEOMTYPE -_TILE_FEATURE.containing_type = _TILE; -_TILE_LAYER.fields_by_name['features'].message_type = _TILE_FEATURE -_TILE_LAYER.fields_by_name['values'].message_type = _TILE_VALUE -_TILE_LAYER.containing_type = _TILE; -_TILE.fields_by_name['layers'].message_type = _TILE_LAYER -_TILE_GEOMTYPE.containing_type = _TILE; -DESCRIPTOR.message_types_by_name['tile'] = _TILE - -class tile(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - - class value(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _TILE_VALUE - - # @@protoc_insertion_point(class_scope:mapnik.vector.tile.value) - - class feature(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _TILE_FEATURE - - # @@protoc_insertion_point(class_scope:mapnik.vector.tile.feature) - - class layer(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _TILE_LAYER - - # @@protoc_insertion_point(class_scope:mapnik.vector.tile.layer) - DESCRIPTOR = _TILE - - # @@protoc_insertion_point(class_scope:mapnik.vector.tile) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), 'H\003') -_TILE_FEATURE.fields_by_name['tags'].has_options = True -_TILE_FEATURE.fields_by_name['tags']._options = _descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001') -_TILE_FEATURE.fields_by_name['geometry'].has_options = True -_TILE_FEATURE.fields_by_name['geometry']._options = _descriptor._ParseOptions(descriptor_pb2.FieldOptions(), '\020\001') -# @@protoc_insertion_point(module_scope) diff --git a/setup.py b/setup.py index 247942dd..39d94ac4 100644 --- a/setup.py +++ b/setup.py @@ -41,8 +41,7 @@ def is_installed(name): 'TileStache.Goodies.VecTiles/OSciMap4/StaticKeys', 'TileStache.Goodies.VecTiles/OSciMap4/StaticVals', 'TileStache.Goodies.VecTiles/OSciMap4/TagRewrite', - 'TileStache.Goodies.VecTiles/OSciMap4', - 'TileStache.Goodies.VecTiles/Mapbox'], + 'TileStache.Goodies.VecTiles/OSciMap4'], scripts=['scripts/tilestache-compose.py', 'scripts/tilestache-seed.py', 'scripts/tilestache-clean.py', 'scripts/tilestache-server.py', 'scripts/tilestache-render.py', 'scripts/tilestache-list.py'], data_files=[('share/tilestache', ['TileStache/Goodies/Providers/DejaVuSansMono-alphanumeric.ttf'])], package_data={'TileStache': ['VERSION', '../doc/*.html']}, From d024d8159bf1a6259037cc9ce9e040d9d027bd58 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Wed, 3 Dec 2014 10:53:31 -0500 Subject: [PATCH 067/344] adding mapbox-vector-tile to requires --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 39d94ac4..95b401da 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ def is_installed(name): return False -requires = ['ModestMaps >=1.3.0','simplejson', 'Werkzeug'] +requires = ['ModestMaps >=1.3.0','simplejson', 'Werkzeug', 'mapbox-vector-tile'] # Soft dependency on PIL or Pillow if is_installed('Pillow') or sys.platform == 'win32': From 61df676e5d1377216bf0e76e1c8131ff02215879 Mon Sep 17 00:00:00 2001 From: Harish Krishna Date: Wed, 3 Dec 2014 10:54:19 -0500 Subject: [PATCH 068/344] codestyle _features --- TileStache/Goodies/VecTiles/mapbox.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index 634b4c82..f314045e 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -37,17 +37,17 @@ def merge(file, feature_layers, coord): file.write(data) def get_feature_layer(name, features): - features_ = [] + _features = [] for feature in features: if len(feature) >= 2: feature[1].update(uid=feature[2]) - features_.append({ + _features.append({ 'geometry': feature[0], 'properties': feature[1] }) return { 'name': name or '', - 'features': features_ + 'features': _features } \ No newline at end of file From 107993f9d0b863393c7a10da9d8fbc4da83594fe Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 10 Dec 2014 17:20:09 -0500 Subject: [PATCH 069/344] Update precision on transscale operation --- TileStache/Goodies/VecTiles/server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 691b9e99..b82f2b98 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -459,7 +459,10 @@ def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clippe if scale: # scale applies to the un-padded bounds, e.g. geometry in the padding area "spills over" past the scale range - geom = 'ST_TransScale(%s, %.2f, %.2f, (%.2f / (%.2f - %.2f)), (%.2f / (%.2f - %.2f)))' % (geom, -bounds[0], -bounds[1], scale, bounds[2], bounds[0], scale, bounds[3], bounds[1]) + geom = ('ST_TransScale(%s, %s, %s, %s, %s)' + % (geom, -bounds[0], -bounds[1], + scale / (bounds[2] - bounds[0]), + scale / (bounds[3] - bounds[1]))) subquery = subquery.replace('!bbox!', bbox) columns = ['q."%s"' % c for c in subcolumns if c not in ('__geometry__', )] From 2e894c21b2293ccb844fddc0b9fbe1b49213be7e Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 7 Nov 2014 10:20:31 -0500 Subject: [PATCH 070/344] Support cache write suppression --- TileStache/Core.py | 9 +++++---- TileStache/Goodies/VecTiles/server.py | 13 ++++++++----- TileStache/__init__.py | 5 +++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/TileStache/Core.py b/TileStache/Core.py index c89eee7d..6013588c 100644 --- a/TileStache/Core.py +++ b/TileStache/Core.py @@ -347,13 +347,14 @@ def name(self): return None - def getTileResponse(self, coord, extension, ignore_cached=False): + def getTileResponse(self, coord, extension, ignore_cached=False, suppress_cache_write=False): """ Get status code, headers, and a tile binary for a given request layer tile. Arguments: - coord: one ModestMaps.Core.Coordinate corresponding to a single tile. - extension: filename extension to choose response type, e.g. "png" or "jpg". - ignore_cached: always re-render the tile, whether it's in the cache or not. + - suppress_cache_write: don't save the tile to the cache This is the main entry point, after site configuration has been loaded and individual tiles need to be rendered. @@ -393,7 +394,7 @@ def getTileResponse(self, coord, extension, ignore_cached=False): try: lockCoord = None - if self.write_cache: + if (not suppress_cache_write) and self.write_cache: # this is the coordinate that actually gets locked. lockCoord = self.metatile.firstCoord(coord) @@ -417,9 +418,9 @@ def getTileResponse(self, coord, extension, ignore_cached=False): tile = e.tile save = False - if not self.write_cache: + if suppress_cache_write or (not self.write_cache): save = False - + if format.lower() == 'jpeg': save_kwargs = self.jpeg_options elif format.lower() == 'png': diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index b82f2b98..3b67f853 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -216,18 +216,20 @@ class MultiProvider: } } ''' - def __init__(self, layer, names): + def __init__(self, layer, names, ignore_cached_sublayers=False): self.layer = layer self.names = names + self.ignore_cached_sublayers = ignore_cached_sublayers - def __call__(self, layer, names): + def __call__(self, layer, names, ignore_cached_sublayers=False): self.layer = layer self.names = names + self.ignore_cached_sublayers = ignore_cached_sublayers def renderTile(self, width, height, srs, coord): ''' Render a single tile, return a Response instance. ''' - return MultiResponse(self.layer.config, self.names, coord) + return MultiResponse(self.layer.config, self.names, coord, self.ignore_cached_sublayers) def getTypeByExtension(self, extension): ''' Get mime-type and format by file extension, "json" or "topojson" only. @@ -342,12 +344,13 @@ def save(self, out, format): class MultiResponse: ''' ''' - def __init__(self, config, names, coord): + def __init__(self, config, names, coord, ignore_cached_sublayers): ''' Create a new response object with TileStache config and layer names. ''' self.config = config self.names = names self.coord = coord + self.ignore_cached_sublayers = ignore_cached_sublayers def save(self, out, format): ''' @@ -388,7 +391,7 @@ def get_tiles(self, format): raise KnownUnknown("%s.get_tiles didn't recognize %s when trying to load %s." % (__name__, ', '.join(unknown_layers), ', '.join(self.names))) layers = [self.config.layers[name] for name in self.names] - mimes, bodies = zip(*[getTile(layer, self.coord, format.lower()) for layer in layers]) + mimes, bodies = zip(*[getTile(layer, self.coord, format.lower(), self.ignore_cached_sublayers, self.ignore_cached_sublayers) for layer in layers]) bad_mimes = [(name, mime) for (mime, name) in zip(mimes, self.names) if not mime.endswith('/json')] if bad_mimes: diff --git a/TileStache/__init__.py b/TileStache/__init__.py index ddbf7b32..c92453db 100644 --- a/TileStache/__init__.py +++ b/TileStache/__init__.py @@ -51,7 +51,7 @@ # symbol used to separate layers when specifying more than one layer _delimiter = ',' -def getTile(layer, coord, extension, ignore_cached=False): +def getTile(layer, coord, extension, ignore_cached=False, suppress_cache_write=False): ''' Get a type string and tile binary for a given request layer tile. This function is documented as part of TileStache's public API: @@ -62,11 +62,12 @@ def getTile(layer, coord, extension, ignore_cached=False): - coord: one ModestMaps.Core.Coordinate corresponding to a single tile. - extension: filename extension to choose response type, e.g. "png" or "jpg". - ignore_cached: always re-render the tile, whether it's in the cache or not. + - suppress_cache_write: don't save the tile to the cache This is the main entry point, after site configuration has been loaded and individual tiles need to be rendered. ''' - status_code, headers, body = layer.getTileResponse(coord, extension, ignore_cached) + status_code, headers, body = layer.getTileResponse(coord, extension, ignore_cached, suppress_cache_write) mime = headers.get('Content-Type') return mime, body From 81a449d2729b7dbdb929eca08c77ac927941382d Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 15 Jan 2015 17:04:56 -0500 Subject: [PATCH 071/344] Retry query on transaction rollback errors Retry executing the same query up to 5 times when receiving transaction rollback errors. --- TileStache/Goodies/VecTiles/server.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 3b67f853..c583f2ea 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -20,6 +20,7 @@ try: from psycopg2.extras import RealDictCursor from psycopg2 import connect + from psycopg2.extensions import TransactionRollbackError except ImportError, err: # Still possible to build the documentation without psycopg2 @@ -420,11 +421,19 @@ def query_columns(dbinfo, srid, subquery, bounds): column_names = set(x.name for x in db.description) return column_names -def get_features(dbinfo, query, geometry_types): +def get_features(dbinfo, query, geometry_types, n_try=1): features = [] with Connection(dbinfo) as db: - db.execute(query) + try: + db.execute(query) + except TransactionRollbackError: + if n_try >= 5: + print 'TransactionRollbackError occurred 5 times' + raise + else: + return get_features(dbinfo, query, geometry_types, + n_try=n_try + 1) for row in db.fetchall(): assert '__geometry__' in row, 'Missing __geometry__ in feature result' assert '__id__' in row, 'Missing __id__ in feature result' From 3fcf08da622d5864a8abdd397890466fccffa175 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 15 Jan 2015 16:21:38 -0500 Subject: [PATCH 072/344] Encode floats little endian --- TileStache/Goodies/VecTiles/mapbox.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index f314045e..89b28f76 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -20,7 +20,7 @@ def encode(file, features, coord, layer_name=''): layers.append(get_feature_layer(layer_name, features)) - data = mapbox_vector_tile.encode(layers) + data = mapbox_vector_tile.encode(layers, encode_floats_big_endian=False) file.write(data) def merge(file, feature_layers, coord): @@ -33,7 +33,7 @@ def merge(file, feature_layers, coord): for layer in feature_layers: layers.append(get_feature_layer(layer['name'], layer['features'])) - data = mapbox_vector_tile.encode(layers) + data = mapbox_vector_tile.encode(layers, encode_floats_big_endian=False) file.write(data) def get_feature_layer(name, features): @@ -50,4 +50,4 @@ def get_feature_layer(name, features): return { 'name': name or '', 'features': _features - } \ No newline at end of file + } From 30f5c3286e3ce2fe577bff2ed456e31eba991fce Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 16 Jan 2015 14:41:16 -0500 Subject: [PATCH 073/344] Remove encode_floats_big_endian option The latest version of mapbox_vector_tile no longer supports the encode_floats_big_endian option. --- TileStache/Goodies/VecTiles/mapbox.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index 89b28f76..6311e0b7 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -20,7 +20,7 @@ def encode(file, features, coord, layer_name=''): layers.append(get_feature_layer(layer_name, features)) - data = mapbox_vector_tile.encode(layers, encode_floats_big_endian=False) + data = mapbox_vector_tile.encode(layers) file.write(data) def merge(file, feature_layers, coord): @@ -33,7 +33,7 @@ def merge(file, feature_layers, coord): for layer in feature_layers: layers.append(get_feature_layer(layer['name'], layer['features'])) - data = mapbox_vector_tile.encode(layers, encode_floats_big_endian=False) + data = mapbox_vector_tile.encode(layers) file.write(data) def get_feature_layer(name, features): From b53058a9da70e9535ff41e53ae6083ef58a4d45e Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 20 Jan 2015 17:41:08 -0500 Subject: [PATCH 074/344] Use more precise values when issuing queries --- TileStache/Goodies/VecTiles/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 3b67f853..8cd9ce72 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -447,7 +447,7 @@ def get_features(dbinfo, query, geometry_types): def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clipped, padding=0, scale=None): ''' Build and return an PostGIS query. ''' - bbox = 'ST_MakeBox2D(ST_MakePoint(%.2f, %.2f), ST_MakePoint(%.2f, %.2f))' % (bounds[0] - padding, bounds[1] - padding, bounds[2] + padding, bounds[3] + padding) + bbox = 'ST_MakeBox2D(ST_MakePoint(%f, %f), ST_MakePoint(%f, %f))' % (bounds[0] - padding, bounds[1] - padding, bounds[2] + padding, bounds[3] + padding) bbox = 'ST_SetSRID(%s, %d)' % (bbox, srid) geom = 'q.__geometry__' @@ -455,7 +455,7 @@ def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clippe geom = 'ST_Intersection(%s, %s)' % (geom, bbox) if tolerance is not None: - geom = 'ST_SimplifyPreserveTopology(%s, %.2f)' % (geom, tolerance) + geom = 'ST_SimplifyPreserveTopology(%s, %f)' % (geom, tolerance) if is_geo: geom = 'ST_Transform(%s, 4326)' % geom From aef4f7765f4df283dd17c63cdf776ca259a78337 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 29 Jan 2015 18:02:33 -0500 Subject: [PATCH 075/344] Add geometry_type to vtm response --- TileStache/Goodies/VecTiles/oscimap.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/TileStache/Goodies/VecTiles/oscimap.py b/TileStache/Goodies/VecTiles/oscimap.py index 1314a7f7..69abdd5d 100644 --- a/TileStache/Goodies/VecTiles/oscimap.py +++ b/TileStache/Goodies/VecTiles/oscimap.py @@ -127,7 +127,9 @@ def addFeature(self, row, coord, this_layer): geom.parseGeometry(row[0]) feature = None; + geometry_type = None if geom.isPoint: + geometry_type = 'Point' feature = self.out.points.add() # add number of points (for multi-point) if len(geom.coordinates) > 2: @@ -140,8 +142,10 @@ def addFeature(self, row, coord, this_layer): return if geom.isPoly: + geometry_type = 'Polygon' feature = self.out.polygons.add() else: + geometry_type = 'LineString' feature = self.out.lines.add() # add coordinate index list (coordinates per geometry) @@ -154,6 +158,10 @@ def addFeature(self, row, coord, this_layer): # add coordinates feature.coordinates.extend(geom.coordinates) + # add geometry type to tags + geometry_type_tag = 'geometry_type', geometry_type + tags.append(self.getTagId(geometry_type_tag)) + # add tags feature.tags.extend(tags) if len(tags) > 1: From 15b19d976682fe19d7c8b35df9e2597bf2cc9850 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 3 Feb 2015 16:15:43 -0500 Subject: [PATCH 076/344] Increase precision when issuing queries --- TileStache/Goodies/VecTiles/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 8cd9ce72..ae2c45cc 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -447,7 +447,7 @@ def get_features(dbinfo, query, geometry_types): def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clipped, padding=0, scale=None): ''' Build and return an PostGIS query. ''' - bbox = 'ST_MakeBox2D(ST_MakePoint(%f, %f), ST_MakePoint(%f, %f))' % (bounds[0] - padding, bounds[1] - padding, bounds[2] + padding, bounds[3] + padding) + bbox = 'ST_MakeBox2D(ST_MakePoint(%.9f, %.9f), ST_MakePoint(%.9f, %.9f))' % (bounds[0] - padding, bounds[1] - padding, bounds[2] + padding, bounds[3] + padding) bbox = 'ST_SetSRID(%s, %d)' % (bbox, srid) geom = 'q.__geometry__' @@ -455,14 +455,14 @@ def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clippe geom = 'ST_Intersection(%s, %s)' % (geom, bbox) if tolerance is not None: - geom = 'ST_SimplifyPreserveTopology(%s, %f)' % (geom, tolerance) + geom = 'ST_SimplifyPreserveTopology(%s, %.9f)' % (geom, tolerance) if is_geo: geom = 'ST_Transform(%s, 4326)' % geom if scale: # scale applies to the un-padded bounds, e.g. geometry in the padding area "spills over" past the scale range - geom = ('ST_TransScale(%s, %s, %s, %s, %s)' + geom = ('ST_TransScale(%s, %.9f, %.9f, %.9f, %.9f)' % (geom, -bounds[0], -bounds[1], scale / (bounds[2] - bounds[0]), scale / (bounds[3] - bounds[1]))) From f500e693ff656110fc0294e8a57c71115dc8231b Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 4 Feb 2015 15:51:25 -0500 Subject: [PATCH 077/344] Even more precision for queries --- TileStache/Goodies/VecTiles/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index ae2c45cc..81d638fd 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -447,7 +447,7 @@ def get_features(dbinfo, query, geometry_types): def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clipped, padding=0, scale=None): ''' Build and return an PostGIS query. ''' - bbox = 'ST_MakeBox2D(ST_MakePoint(%.9f, %.9f), ST_MakePoint(%.9f, %.9f))' % (bounds[0] - padding, bounds[1] - padding, bounds[2] + padding, bounds[3] + padding) + bbox = 'ST_MakeBox2D(ST_MakePoint(%.12f, %.12f), ST_MakePoint(%.12f, %.12f))' % (bounds[0] - padding, bounds[1] - padding, bounds[2] + padding, bounds[3] + padding) bbox = 'ST_SetSRID(%s, %d)' % (bbox, srid) geom = 'q.__geometry__' @@ -455,14 +455,14 @@ def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clippe geom = 'ST_Intersection(%s, %s)' % (geom, bbox) if tolerance is not None: - geom = 'ST_SimplifyPreserveTopology(%s, %.9f)' % (geom, tolerance) + geom = 'ST_SimplifyPreserveTopology(%s, %.12f)' % (geom, tolerance) if is_geo: geom = 'ST_Transform(%s, 4326)' % geom if scale: # scale applies to the un-padded bounds, e.g. geometry in the padding area "spills over" past the scale range - geom = ('ST_TransScale(%s, %.9f, %.9f, %.9f, %.9f)' + geom = ('ST_TransScale(%s, %.12f, %.12f, %.12f, %.12f)' % (geom, -bounds[0], -bounds[1], scale / (bounds[2] - bounds[0]), scale / (bounds[3] - bounds[1]))) From 0aa94218bb82fd24ae84302cfc5cf620aaa7a2ce Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 12 Mar 2015 16:05:15 -0400 Subject: [PATCH 078/344] Update connection session details --- TileStache/Goodies/VecTiles/server.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 97c53c38..e7ab12a8 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -252,17 +252,19 @@ def getTypeByExtension(self, extension): class Connection: ''' Context manager for Postgres connections. - + See http://www.python.org/dev/peps/pep-0343/ and http://effbot.org/zone/python-with-statement.htm ''' def __init__(self, dbinfo): self.dbinfo = dbinfo - + def __enter__(self): - self.db = connect(**self.dbinfo).cursor(cursor_factory=RealDictCursor) + conn = connect(**self.dbinfo) + conn.set_session(readonly=True, autocommit=True) + self.db = conn.cursor(cursor_factory=RealDictCursor) return self.db - + def __exit__(self, type, value, traceback): self.db.connection.close() From 9d646686ef79c8ba60f4f5a721d12f03766c121f Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 16 Mar 2015 16:44:11 -0400 Subject: [PATCH 079/344] Support normalizing osm id Also add support for transformation functions that can be applied in the pipeline. --- TileStache/Goodies/VecTiles/server.py | 79 +++++++++++++++++------- TileStache/Goodies/VecTiles/transform.py | 18 ++++++ 2 files changed, 76 insertions(+), 21 deletions(-) create mode 100644 TileStache/Goodies/VecTiles/transform.py diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index e7ab12a8..fa650050 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -11,11 +11,13 @@ from urlparse import urljoin, urlparse from urllib import urlopen from os.path import exists +from shapely.wkb import dumps from shapely.wkb import loads import json from ... import getTile from ...Core import KnownUnknown +from TileStache.Config import loadClassPath try: from psycopg2.extras import RealDictCursor @@ -34,6 +36,24 @@ def connect(*args, **kwargs): tolerances = [6378137 * 2 * pi / (2 ** (zoom + 8)) for zoom in range(22)] + +def make_transform_fn(transform_fns): + if not transform_fns: + return None + + def transform_fn(shape, properties, fid): + for fn in transform_fns: + shape, properties, fid = fn(shape, properties, fid) + return shape, properties, fid + return transform_fn + + +def resolve_transform_fns(fn_dotted_names): + if not fn_dotted_names: + return None + return map(loadClassPath, fn_dotted_names) + + class Provider: ''' VecTiles provider for PostGIS data sources. @@ -87,10 +107,17 @@ class Provider: Optional list of geometry types that constrains the results of what kind of features are returned. + transform_fns: + Optional list of transformation functions. It will be + passed a shapely object, the properties dictionary, and + the feature id. The function should return a tuple + consisting of the new shapely object, properties + dictionary, and feature id for the feature. + Sample configuration, for a layer with no results at zooms 0-9, basic selection of lines with names and highway tags for zoom 10, a remote URL containing a query for zoom 11, and a local file for zooms 12+: - + "provider": { "class": "TileStache.Goodies.VecTiles:Provider", @@ -114,11 +141,11 @@ class Provider: } } ''' - def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, simplify_until=16, suppress_simplification=(), geometry_types=None): + def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, simplify_until=16, suppress_simplification=(), geometry_types=None, transform_fns=None): ''' ''' self.layer = layer - + keys = 'host', 'user', 'password', 'database', 'port', 'dbname' self.dbinfo = dict([(k, v) for (k, v) in dbinfo.items() if k in keys]) @@ -128,6 +155,7 @@ def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, self.simplify_until = int(simplify_until) self.suppress_simplification = set(suppress_simplification) self.geometry_types = None if geometry_types is None else set(geometry_types) + self.transform_fns = make_transform_fn(resolve_transform_fns(transform_fns)) self.queries = [] self.columns = {} @@ -174,7 +202,7 @@ def renderTile(self, width, height, srs, coord): else: tolerance = self.simplify * tolerances[coord.zoom] if coord.zoom < self.simplify_until else None - return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name(), self.geometry_types) + return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name(), self.geometry_types, self.transform_fns) def getTypeByExtension(self, extension): ''' Get mime-type and format by file extension, one of "mvt", "json" or "topojson". @@ -199,15 +227,19 @@ def getTypeByExtension(self, extension): class MultiProvider: ''' VecTiles provider to gather PostGIS tiles into a single multi-response. - + Returns a MultiResponse object for GeoJSON or TopoJSON requests. - + names: List of names of vector-generating layers from elsewhere in config. - + + ignore_cached_sublayers: + True if cache provider should not save intermediate layers + in cache. + Sample configuration, for a layer with combined data from water and land areas, both assumed to be vector-returning layers: - + "provider": { "class": "TileStache.Goodies.VecTiles:MultiProvider", @@ -221,7 +253,7 @@ def __init__(self, layer, names, ignore_cached_sublayers=False): self.layer = layer self.names = names self.ignore_cached_sublayers = ignore_cached_sublayers - + def __call__(self, layer, names, ignore_cached_sublayers=False): self.layer = layer self.names = names @@ -271,19 +303,20 @@ def __exit__(self, type, value, traceback): class Response: ''' ''' - def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip, coord, layer_name, geometry_types): + def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip, coord, layer_name, geometry_types, transform_fns): ''' Create a new response object with Postgres connection info and a query. - + bounds argument is a 4-tuple with (xmin, ymin, xmax, ymax). ''' self.dbinfo = dbinfo self.bounds = bounds self.zoom = zoom self.clip = clip - self.coord= coord + self.coord = coord self.layer_name = layer_name self.geometry_types = geometry_types - + self.transform_fns = transform_fns + geo_query = build_query(srid, subquery, columns, bounds, tolerance, True, clip) merc_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip) oscimap_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, oscimap.padding * tolerances[coord.zoom], oscimap.extents) @@ -293,7 +326,7 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli def save(self, out, format): ''' ''' - features = get_features(self.dbinfo, self.query[format], self.geometry_types) + features = get_features(self.dbinfo, self.query[format], self.geometry_types, self.transform_fns) if format == 'MVT': mvt.encode(out, features) @@ -371,7 +404,7 @@ def save(self, out, format): width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["OpenScienceMap"], layer.provider.geometry_types)}) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["OpenScienceMap"], layer.provider.geometry_types, layer.provider.transform_fns)}) oscimap.merge(out, feature_layers, self.coord) elif format == 'Mapbox': @@ -381,7 +414,7 @@ def save(self, out, format): width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["Mapbox"], layer.provider.geometry_types)}) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["Mapbox"], layer.provider.geometry_types, layer.provider.transform_fns)}) mapbox.merge(out, feature_layers, self.coord) else: @@ -423,7 +456,7 @@ def query_columns(dbinfo, srid, subquery, bounds): column_names = set(x.name for x in db.description) return column_names -def get_features(dbinfo, query, geometry_types, n_try=1): +def get_features(dbinfo, query, geometry_types, transform_fn, n_try=1): features = [] with Connection(dbinfo) as db: @@ -435,7 +468,7 @@ def get_features(dbinfo, query, geometry_types, n_try=1): raise else: return get_features(dbinfo, query, geometry_types, - n_try=n_try + 1) + transform_fn, n_try=n_try + 1) for row in db.fetchall(): assert '__geometry__' in row, 'Missing __geometry__ in feature result' assert '__id__' in row, 'Missing __id__ in feature result' @@ -443,14 +476,18 @@ def get_features(dbinfo, query, geometry_types, n_try=1): wkb = bytes(row.pop('__geometry__')) id = row.pop('__id__') + shape = loads(wkb) if geometry_types is not None: - shape = loads(wkb) - geom_type = shape.__geo_interface__['type'] - if geom_type not in geometry_types: + if shape.type not in geometry_types: #print 'found %s which is not in: %s' % (geom_type, geometry_types) continue props = dict((k, v) for k, v in row.items() if v is not None) + + if transform_fn: + shape, props, id = transform_fn(shape, props, id) + wkb = dumps(shape) + features.append((wkb, props, id)) return features diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py new file mode 100644 index 00000000..97e7d068 --- /dev/null +++ b/TileStache/Goodies/VecTiles/transform.py @@ -0,0 +1,18 @@ +# transformation functions to apply to features + + +def normalize_osm_id(shape, properties, fid): + osm_id = properties.get('osm_id') + if osm_id is None: + return shape, properties, fid + try: + int_osm_id = int(osm_id) + except ValueError: + return shape, properties, fid + else: + if int_osm_id < 0: + properties['osm_id'] = -int_osm_id + properties['osm_relation'] = True + else: + properties['osm_id'] = int_osm_id + return shape, properties, fid From caa12f580c81f98859d1c3d9d4657474007093d9 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 19 Mar 2015 16:40:55 -0400 Subject: [PATCH 080/344] Spell transform_fns singular where appropriate --- TileStache/Goodies/VecTiles/server.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index fa650050..7950f1d1 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -155,7 +155,7 @@ def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, self.simplify_until = int(simplify_until) self.suppress_simplification = set(suppress_simplification) self.geometry_types = None if geometry_types is None else set(geometry_types) - self.transform_fns = make_transform_fn(resolve_transform_fns(transform_fns)) + self.transform_fn = make_transform_fn(resolve_transform_fns(transform_fns)) self.queries = [] self.columns = {} @@ -202,7 +202,7 @@ def renderTile(self, width, height, srs, coord): else: tolerance = self.simplify * tolerances[coord.zoom] if coord.zoom < self.simplify_until else None - return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name(), self.geometry_types, self.transform_fns) + return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name(), self.geometry_types, self.transform_fn) def getTypeByExtension(self, extension): ''' Get mime-type and format by file extension, one of "mvt", "json" or "topojson". @@ -303,7 +303,7 @@ def __exit__(self, type, value, traceback): class Response: ''' ''' - def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip, coord, layer_name, geometry_types, transform_fns): + def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip, coord, layer_name, geometry_types, transform_fn): ''' Create a new response object with Postgres connection info and a query. bounds argument is a 4-tuple with (xmin, ymin, xmax, ymax). @@ -315,7 +315,7 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli self.coord = coord self.layer_name = layer_name self.geometry_types = geometry_types - self.transform_fns = transform_fns + self.transform_fn = transform_fn geo_query = build_query(srid, subquery, columns, bounds, tolerance, True, clip) merc_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip) @@ -326,7 +326,7 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli def save(self, out, format): ''' ''' - features = get_features(self.dbinfo, self.query[format], self.geometry_types, self.transform_fns) + features = get_features(self.dbinfo, self.query[format], self.geometry_types, self.transform_fn) if format == 'MVT': mvt.encode(out, features) @@ -404,7 +404,7 @@ def save(self, out, format): width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["OpenScienceMap"], layer.provider.geometry_types, layer.provider.transform_fns)}) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["OpenScienceMap"], layer.provider.geometry_types, layer.provider.transform_fn)}) oscimap.merge(out, feature_layers, self.coord) elif format == 'Mapbox': @@ -414,7 +414,7 @@ def save(self, out, format): width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["Mapbox"], layer.provider.geometry_types, layer.provider.transform_fns)}) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["Mapbox"], layer.provider.geometry_types, layer.provider.transform_fn)}) mapbox.merge(out, feature_layers, self.coord) else: From 3588f2dd00d4499dd7c66fb4fa430f7a614d95d6 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 1 Apr 2015 15:15:07 -0400 Subject: [PATCH 081/344] Add support for per layer custom sorting functions --- TileStache/Goodies/VecTiles/server.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 7950f1d1..50cbb328 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -114,6 +114,10 @@ class Provider: consisting of the new shapely object, properties dictionary, and feature id for the feature. + sort_fn: + Optional function that will be used to sort features + fetched from the database. + Sample configuration, for a layer with no results at zooms 0-9, basic selection of lines with names and highway tags for zoom 10, a remote URL containing a query for zoom 11, and a local file for zooms 12+: @@ -141,7 +145,7 @@ class Provider: } } ''' - def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, simplify_until=16, suppress_simplification=(), geometry_types=None, transform_fns=None): + def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, simplify_until=16, suppress_simplification=(), geometry_types=None, transform_fns=None, sort_fn=None): ''' ''' self.layer = layer @@ -156,6 +160,7 @@ def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, self.suppress_simplification = set(suppress_simplification) self.geometry_types = None if geometry_types is None else set(geometry_types) self.transform_fn = make_transform_fn(resolve_transform_fns(transform_fns)) + self.sort_fn = None if sort_fn is None else loadClassPath(sort_fn) self.queries = [] self.columns = {} @@ -202,7 +207,7 @@ def renderTile(self, width, height, srs, coord): else: tolerance = self.simplify * tolerances[coord.zoom] if coord.zoom < self.simplify_until else None - return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name(), self.geometry_types, self.transform_fn) + return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name(), self.geometry_types, self.transform_fn, self.sort_fn) def getTypeByExtension(self, extension): ''' Get mime-type and format by file extension, one of "mvt", "json" or "topojson". @@ -303,7 +308,7 @@ def __exit__(self, type, value, traceback): class Response: ''' ''' - def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip, coord, layer_name, geometry_types, transform_fn): + def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip, coord, layer_name, geometry_types, transform_fn, sort_fn): ''' Create a new response object with Postgres connection info and a query. bounds argument is a 4-tuple with (xmin, ymin, xmax, ymax). @@ -316,6 +321,7 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli self.layer_name = layer_name self.geometry_types = geometry_types self.transform_fn = transform_fn + self.sort_fn = sort_fn geo_query = build_query(srid, subquery, columns, bounds, tolerance, True, clip) merc_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip) @@ -326,7 +332,7 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli def save(self, out, format): ''' ''' - features = get_features(self.dbinfo, self.query[format], self.geometry_types, self.transform_fn) + features = get_features(self.dbinfo, self.query[format], self.geometry_types, self.transform_fn, self.sort_fn) if format == 'MVT': mvt.encode(out, features) @@ -404,7 +410,7 @@ def save(self, out, format): width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["OpenScienceMap"], layer.provider.geometry_types, layer.provider.transform_fn)}) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["OpenScienceMap"], layer.provider.geometry_types, layer.provider.transform_fn, layer.provider.sort_fn)}) oscimap.merge(out, feature_layers, self.coord) elif format == 'Mapbox': @@ -414,7 +420,7 @@ def save(self, out, format): width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["Mapbox"], layer.provider.geometry_types, layer.provider.transform_fn)}) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["Mapbox"], layer.provider.geometry_types, layer.provider.transform_fn, layer.provider.sort_fn)}) mapbox.merge(out, feature_layers, self.coord) else: @@ -456,7 +462,7 @@ def query_columns(dbinfo, srid, subquery, bounds): column_names = set(x.name for x in db.description) return column_names -def get_features(dbinfo, query, geometry_types, transform_fn, n_try=1): +def get_features(dbinfo, query, geometry_types, transform_fn, sort_fn, n_try=1): features = [] with Connection(dbinfo) as db: @@ -468,7 +474,7 @@ def get_features(dbinfo, query, geometry_types, transform_fn, n_try=1): raise else: return get_features(dbinfo, query, geometry_types, - transform_fn, n_try=n_try + 1) + transform_fn, sort_fn, n_try=n_try + 1) for row in db.fetchall(): assert '__geometry__' in row, 'Missing __geometry__ in feature result' assert '__id__' in row, 'Missing __id__ in feature result' @@ -490,6 +496,9 @@ def get_features(dbinfo, query, geometry_types, transform_fn, n_try=1): features.append((wkb, props, id)) + if sort_fn: + features = sort_fn(features) + return features def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clipped, padding=0, scale=None): From 52261f03593db7e93f92e4bf6c62588bcd13e4ad Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 1 Apr 2015 15:15:38 -0400 Subject: [PATCH 082/344] Add sorting functions --- TileStache/Goodies/VecTiles/sort.py | 66 +++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 TileStache/Goodies/VecTiles/sort.py diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py new file mode 100644 index 00000000..bb5dfd9e --- /dev/null +++ b/TileStache/Goodies/VecTiles/sort.py @@ -0,0 +1,66 @@ +# sort functions to apply to features + +from transform import _to_float + + +def _sort_features_by_key(features, key): + features.sort(key=key) + return features + + +def _place_key(feature): + wkb, properties, fid = feature + admin_level = properties.get('admin_level') + admin_level_float = _to_float(admin_level) + if admin_level_float is None: + return 1000.0 + return admin_level + + +def _by_feature_id(feature): + wkb, properties, fid = feature + return fid + + +def _by_area(feature): + wkb, properties, fid = feature + return properties.get('area') + + +def _sort_by_area_then_id(features): + features.sort(key=_by_feature_id) + features.sort(key=_by_area, reverse=True) + return features + + +def _road_key(feature): + wkb, properties, fid = feature + return properties.get('sort_key') + + +def buildings(features): + return _sort_by_area_then_id(features) + + +def earth(features): + return _sort_features_by_key(features, _by_feature_id) + + +def landuse(features): + return _sort_by_area_then_id(features) + + +def places(features): + return _sort_features_by_key(features, _place_key) + + +def pois(features): + return _sort_features_by_key(features, _by_feature_id) + + +def roads(features): + return _sort_features_by_key(features, _road_key) + + +def water(features): + return _sort_by_area_then_id(features) From 75c952f814218607d3ffbcbbee85d7986dafe7cc Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 1 Apr 2015 15:15:48 -0400 Subject: [PATCH 083/344] Add transformation functions --- TileStache/Goodies/VecTiles/transform.py | 245 ++++++++++++++++++++++- 1 file changed, 244 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 97e7d068..4362acda 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1,8 +1,133 @@ # transformation functions to apply to features +import re + + +def _to_float(x): + if x is None: + return None + # normalize punctuation + x = x.replace(';', '.').replace(',', '.') + try: + return float(x) + except ValueError: + return None + + +feet_pattern = re.compile('([+-]?[0-9.]+)\'(?: *([+-]?[0-9.]+)")?') +number_pattern = re.compile('([+-]?[0-9.]+)') + + +def _to_float_meters(x): + if x is None: + return None + + as_float = _to_float(x) + if as_float is not None: + return as_float + + # trim whitespace to simplify further matching + x = x.strip() + + # try explicit meters suffix + if x.endswith(' m'): + meters_as_float = _to_float(x[:-2]) + if meters_as_float is not None: + return meters_as_float + + # try if it looks like an expression in feet via ' " + feet_match = feet_pattern.match(x) + if feet_match is not None: + feet = feet_match.group(1) + inches = feet_match.group(2) + feet_as_float = _to_float(feet) + inches_as_float = _to_float(inches) + + total_inches = 0.0 + parsed_feet_or_inches = False + if feet_as_float is not None: + total_inches = feet_as_float * 12.0 + parsed_feet_or_inches = True + if inches_as_float is not None: + total_inches += inches_as_float + parsed_feet_or_inches = True + if parsed_feet_or_inches: + meters = total_inches * 0.02544 + return meters + + # try and match the first number that can be parsed + for number_match in number_pattern.finditer(x): + potential_number = number_match.group(1) + as_float = _to_float(potential_number) + if as_float is not None: + return as_float + + return None + + +def _coalesce(properties, *property_names): + for prop in property_names: + val = properties.get(prop) + if val: + return val + return None + + +def _remove_properties(properties, *property_names): + for prop in property_names: + properties.pop(prop, None) + return properties + + +def _building_calc_levels(levels): + levels = max(levels, 1) + levels = (levels * 3) + 2 + return levels + + +def _building_calc_min_levels(min_levels): + min_levels = max(min_levels, 0) + min_levels = min_levels * 3 + return min_levels + + +def _building_calc_height(height_val, levels_val, levels_calc_fn): + height = _to_float_meters(height_val) + if height is not None: + return height + levels = _to_float_meters(levels_val) + if levels is None: + return None + levels = levels_calc_fn(levels) + return levels + + +road_kind_highway = set(('motorway', 'motorway_link')) +road_kind_major_road = set(('trunk', 'trunk_link', 'primary', 'primary_link', + 'secondary', 'secondary_link', + 'tertiary', 'tertiary_link')) +road_kind_path = set(('footpath', 'track', 'footway', 'steps', 'pedestrian', + 'path', 'cycleway')) +road_kind_rail = set(('rail', 'tram', 'light_rail', 'narrow_gauge', + 'monorail', 'subway')) + + +def _road_kind(properties): + highway = properties.get('highway') + if highway in road_kind_highway: + return 'highway' + if highway in road_kind_major_road: + return 'major_road' + if highway in road_kind_path: + return 'path' + railway = properties.get('railway') + if railway in road_kind_rail: + return 'rail' + return 'minor_road' + def normalize_osm_id(shape, properties, fid): - osm_id = properties.get('osm_id') + osm_id = properties.pop('osm_id', None) if osm_id is None: return shape, properties, fid try: @@ -16,3 +141,121 @@ def normalize_osm_id(shape, properties, fid): else: properties['osm_id'] = int_osm_id return shape, properties, fid + + +def building_kind(shape, properties, fid): + building = _coalesce(properties, 'building:part', 'building') + if building and building != 'yes': + kind = building + else: + kind = _coalesce(properties, 'amenity', 'shop', 'tourism') + if kind: + properties['kind'] = kind + return shape, properties, fid + + +def building_height(shape, properties, fid): + height = _building_calc_height( + properties.get('height'), properties.get('building:levels'), + _building_calc_levels) + if height is not None: + properties['height'] = height + else: + properties.pop('height', None) + return shape, properties, fid + + +def building_min_height(shape, properties, fid): + min_height = _building_calc_height( + properties.get('min_height'), properties.get('building:min_levels'), + _building_calc_min_levels) + if min_height is not None: + properties['min_height'] = min_height + else: + properties.pop('min_height', None) + return shape, properties, fid + + +def building_trim_properties(shape, properties, fid): + properties = _remove_properties( + properties, + 'amenity', 'shop', 'tourism', + 'building', 'building:part', + 'building:levels', 'building:min_levels') + return shape, properties, fid + + +def road_kind(shape, properties, fid): + properties['kind'] = _road_kind(properties) + return shape, properties, fid + + +def road_classifier(shape, properties, fid): + highway = properties.get('highway') + tunnel = properties.get('tunnel') + bridge = properties.get('bridge') + is_link = 'yes' if highway and highway.endswith('_link') else 'no' + is_tunnel = 'yes' if tunnel and tunnel in ('yes', 'true') else 'no' + is_bridge = 'yes' if bridge and bridge in ('yes', 'true') else 'no' + properties['is_link'] = is_link + properties['is_tunnel'] = is_tunnel + properties['is_bridge'] = is_bridge + return shape, properties, fid + + +def road_sort_key(shape, properties, fid): + sort_val = 0 + + layer = properties.get('layer') + if layer: + layer_float = _to_float(layer) + if layer_float is not None: + sort_val += (layer_float * 1000) + + bridge = properties.get('bridge') + if bridge in ('yes', 'true'): + sort_val += 100 + + tunnel = properties.get('tunnel') + if tunnel in ('yes', 'true'): + sort_val -= 100 + + highway = properties.get('highway', '') + railway = properties.get('railway', '') + aeroway = properties.get('aeroway', '') + + if highway == 'motorway': + sort_val += 0 + elif railway in ('rail', 'tram', 'light_rail', 'narrow_guage', 'monorail'): + sort_val -= 0.5 + elif highway == 'trunk': + sort_val -= 1 + elif highway == 'primary': + sort_val -= 2 + elif highway == 'secondary': + sort_val -= 3 + elif aeroway == 'runway': + sort_val -= 3 + elif aeroway == 'taxiway': + sort_val -= 3.5 + elif highway == 'tertiary': + sort_val -= 4 + elif highway.endswith('_link'): + sort_val -= 5 + elif highway in ('residential', 'unclassified', 'road'): + sort_val -= 6 + elif highway in ('unclassified', 'service', 'minor'): + sort_val -= 7 + elif railway == 'subway': + sort_val -= 8 + else: + sort_val -= 9 + + properties['sort_key'] = sort_val + + return shape, properties, fid + + +def road_trim_properties(shape, properties, fid): + properties = _remove_properties(properties, 'bridge', 'layer', 'tunnel') + return shape, properties, fid From c4103a6cc69b3420085a616f19c3dd71f33d68fb Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 7 Apr 2015 15:59:27 -0400 Subject: [PATCH 084/344] Update osm id transform --- TileStache/Goodies/VecTiles/transform.py | 25 ++++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4362acda..031ab488 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1,5 +1,7 @@ # transformation functions to apply to features +from shapely import wkb +import md5 import re @@ -127,20 +129,17 @@ def _road_kind(properties): def normalize_osm_id(shape, properties, fid): - osm_id = properties.pop('osm_id', None) - if osm_id is None: - return shape, properties, fid - try: - int_osm_id = int(osm_id) - except ValueError: - return shape, properties, fid + if fid < 0: + properties['osm_id'] = -fid + properties['osm_relation'] = True + # preserve previous behavior and use the first 10 digits of + # the md5 sum for the id on negative ids + binary = wkb.dumps(shape) + fid = md5.new(binary).hexdigest()[:10] else: - if int_osm_id < 0: - properties['osm_id'] = -int_osm_id - properties['osm_relation'] = True - else: - properties['osm_id'] = int_osm_id - return shape, properties, fid + # always use a string id + fid = str(fid) + return shape, properties, fid def building_kind(shape, properties, fid): From 9155643764bacfcd654dfee3a04173a38ee83ea5 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 10 Apr 2015 14:34:52 -0400 Subject: [PATCH 085/344] Track transform / sort fn names in provider --- TileStache/Goodies/VecTiles/server.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 50cbb328..632b8f92 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -159,12 +159,18 @@ def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, self.simplify_until = int(simplify_until) self.suppress_simplification = set(suppress_simplification) self.geometry_types = None if geometry_types is None else set(geometry_types) + self.transform_fn_names = transform_fns self.transform_fn = make_transform_fn(resolve_transform_fns(transform_fns)) - self.sort_fn = None if sort_fn is None else loadClassPath(sort_fn) + if sort_fn: + self.sort_fn_name = sort_fn + self.sort_fn = loadClassPath(sort_fn) + else: + self.sort_fn_name = None + self.sort_fn = None self.queries = [] self.columns = {} - + for query in queries: if query is None: self.queries.append(None) From 85a4fd1da729394a12c2a200d768a21183032b67 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 14 Apr 2015 11:27:56 -0400 Subject: [PATCH 086/344] Update road sort key transform to return float The road sort key transform now always returns back a float. --- TileStache/Goodies/VecTiles/transform.py | 30 ++++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 031ab488..251ea604 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -203,52 +203,52 @@ def road_classifier(shape, properties, fid): def road_sort_key(shape, properties, fid): - sort_val = 0 + sort_val = 0.0 layer = properties.get('layer') if layer: layer_float = _to_float(layer) if layer_float is not None: - sort_val += (layer_float * 1000) + sort_val += (layer_float * 1000.0) bridge = properties.get('bridge') if bridge in ('yes', 'true'): - sort_val += 100 + sort_val += 100.0 tunnel = properties.get('tunnel') if tunnel in ('yes', 'true'): - sort_val -= 100 + sort_val -= 100.0 highway = properties.get('highway', '') railway = properties.get('railway', '') aeroway = properties.get('aeroway', '') if highway == 'motorway': - sort_val += 0 + sort_val += 0.0 elif railway in ('rail', 'tram', 'light_rail', 'narrow_guage', 'monorail'): sort_val -= 0.5 elif highway == 'trunk': - sort_val -= 1 + sort_val -= 1.0 elif highway == 'primary': - sort_val -= 2 + sort_val -= 2.0 elif highway == 'secondary': - sort_val -= 3 + sort_val -= 3.0 elif aeroway == 'runway': - sort_val -= 3 + sort_val -= 3.0 elif aeroway == 'taxiway': sort_val -= 3.5 elif highway == 'tertiary': - sort_val -= 4 + sort_val -= 4.0 elif highway.endswith('_link'): - sort_val -= 5 + sort_val -= 5.0 elif highway in ('residential', 'unclassified', 'road'): - sort_val -= 6 + sort_val -= 6.0 elif highway in ('unclassified', 'service', 'minor'): - sort_val -= 7 + sort_val -= 7.0 elif railway == 'subway': - sort_val -= 8 + sort_val -= 8.0 else: - sort_val -= 9 + sort_val -= 9.0 properties['sort_key'] = sort_val From 7617d9f91255b19aeba103c0eb9b868c16311042 Mon Sep 17 00:00:00 2001 From: Brett Camper Date: Wed, 15 Apr 2015 14:15:50 -0400 Subject: [PATCH 087/344] revise road sort order to use a range of [0, 39] drastically reduces the total range (previously +/- 3000), and ensures only positive integers are returned --- TileStache/Goodies/VecTiles/transform.py | 69 +++++++++++++----------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 031ab488..80488de4 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -203,52 +203,57 @@ def road_classifier(shape, properties, fid): def road_sort_key(shape, properties, fid): + # Calculated sort value is in the range 0 to 39 sort_val = 0 - layer = properties.get('layer') - if layer: - layer_float = _to_float(layer) - if layer_float is not None: - sort_val += (layer_float * 1000) - - bridge = properties.get('bridge') - if bridge in ('yes', 'true'): - sort_val += 100 - - tunnel = properties.get('tunnel') - if tunnel in ('yes', 'true'): - sort_val -= 100 - + # Base layer range is 15 to 24 highway = properties.get('highway', '') railway = properties.get('railway', '') aeroway = properties.get('aeroway', '') if highway == 'motorway': - sort_val += 0 + sort_val += 24 elif railway in ('rail', 'tram', 'light_rail', 'narrow_guage', 'monorail'): - sort_val -= 0.5 + sort_val += 23 elif highway == 'trunk': - sort_val -= 1 + sort_val += 22 elif highway == 'primary': - sort_val -= 2 - elif highway == 'secondary': - sort_val -= 3 - elif aeroway == 'runway': - sort_val -= 3 - elif aeroway == 'taxiway': - sort_val -= 3.5 - elif highway == 'tertiary': - sort_val -= 4 + sort_val += 21 + elif highway == 'secondary' or aeroway == 'runway': + sort_val += 20 + elif highway == 'tertiary' or aeroway == 'taxiway': + sort_val += 19 elif highway.endswith('_link'): - sort_val -= 5 + sort_val += 18 elif highway in ('residential', 'unclassified', 'road'): - sort_val -= 6 + sort_val += 17 elif highway in ('unclassified', 'service', 'minor'): - sort_val -= 7 - elif railway == 'subway': - sort_val -= 8 + sort_val += 16 else: - sort_val -= 9 + sort_val += 15 + + # Bridges and tunnels add +/- 10 + bridge = properties.get('bridge') + tunnel = properties.get('tunnel') + + if bridge in ('yes', 'true'): + sort_val += 10 + elif tunnel in ('yes', 'true') or (railway == 'subway' and tunnel not in ('no', 'false')): + sort_val -= 10 + + # Explicit layer is clipped to [-5, 5] range + layer = properties.get('layer') + + if layer: + layer_float = _to_float(layer) + if layer_float is not None: + layer_float = max(min(layer_float, 5), -5) + # Positive layer range is 20 to 24 + if layer_float > 0 + sort_val = layer_float + 34 + # Negative layer range is -11 to -15 + elif layer_float < 0 + sort_val = layer_float + 5 properties['sort_key'] = sort_val From 5e73db28ef54df498cd72b1270d41977fafa6a34 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 16 Apr 2015 15:33:10 -0400 Subject: [PATCH 088/344] Add colons to if statements --- TileStache/Goodies/VecTiles/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 80488de4..221ba489 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -249,10 +249,10 @@ def road_sort_key(shape, properties, fid): if layer_float is not None: layer_float = max(min(layer_float, 5), -5) # Positive layer range is 20 to 24 - if layer_float > 0 + if layer_float > 0: sort_val = layer_float + 34 # Negative layer range is -11 to -15 - elif layer_float < 0 + elif layer_float < 0: sort_val = layer_float + 5 properties['sort_key'] = sort_val From 5471b6bb8c8853f0e9570f48cd070eceddbc4797 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 16 Apr 2015 15:37:22 -0400 Subject: [PATCH 089/344] pep8 line width --- TileStache/Goodies/VecTiles/transform.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 221ba489..51150aca 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -238,7 +238,8 @@ def road_sort_key(shape, properties, fid): if bridge in ('yes', 'true'): sort_val += 10 - elif tunnel in ('yes', 'true') or (railway == 'subway' and tunnel not in ('no', 'false')): + elif (tunnel in ('yes', 'true') or + (railway == 'subway' and tunnel not in ('no', 'false'))): sort_val -= 10 # Explicit layer is clipped to [-5, 5] range From d5a707582e801ef59005aca6717a42f7e5e53051 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 16 Apr 2015 15:54:26 -0400 Subject: [PATCH 090/344] Update layer comments and ensure integer --- TileStache/Goodies/VecTiles/transform.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 51150aca..64e1f2fa 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -249,12 +249,14 @@ def road_sort_key(shape, properties, fid): layer_float = _to_float(layer) if layer_float is not None: layer_float = max(min(layer_float, 5), -5) - # Positive layer range is 20 to 24 + # The range of values from above is [5, 34] + # For positive layer values, we want the range to be: + # [34, 39] if layer_float > 0: - sort_val = layer_float + 34 - # Negative layer range is -11 to -15 + sort_val = int(layer_float + 34) + # For negative layer values, [0, 5] elif layer_float < 0: - sort_val = layer_float + 5 + sort_val = int(layer_float + 5) properties['sort_key'] = sort_val From d1c92ad68e4b66ce3828303ce3319178f417df4c Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 27 Apr 2015 18:00:51 -0400 Subject: [PATCH 091/344] Update road sort key calc for `living_street` --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 64e1f2fa..dbaf2307 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -225,7 +225,7 @@ def road_sort_key(shape, properties, fid): sort_val += 19 elif highway.endswith('_link'): sort_val += 18 - elif highway in ('residential', 'unclassified', 'road'): + elif highway in ('residential', 'unclassified', 'road', 'living_street'): sort_val += 17 elif highway in ('unclassified', 'service', 'minor'): sort_val += 16 From 1d5ec6d2a4f6740c114d454046edcaefe347983c Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 29 Apr 2015 17:25:12 -0400 Subject: [PATCH 092/344] Stop including clipped property in json formats --- TileStache/Goodies/VecTiles/geojson.py | 10 +++------- TileStache/Goodies/VecTiles/server.py | 8 ++++---- TileStache/Goodies/VecTiles/topojson.py | 7 ++----- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/TileStache/Goodies/VecTiles/geojson.py b/TileStache/Goodies/VecTiles/geojson.py index af014c64..ac6b4c4e 100644 --- a/TileStache/Goodies/VecTiles/geojson.py +++ b/TileStache/Goodies/VecTiles/geojson.py @@ -42,7 +42,7 @@ def decode(file): return features -def encode(file, features, zoom, is_clipped): +def encode(file, features, zoom): ''' Encode a list of (WKB, property dict) features into a GeoJSON stream. Also accept three-element tuples as features: (WKB, property dict, id). @@ -57,11 +57,7 @@ def encode(file, features, zoom, is_clipped): except ValueError: # Fall back to two-element features features = [dict(type='Feature', properties=p, geometry=loads(g).__geo_interface__) for (g, p) in features] - - if is_clipped: - for feature in features: - feature.update(dict(clipped=True)) - + geojson = dict(type='FeatureCollection', features=features) write_to_file(file, geojson, zoom) @@ -92,4 +88,4 @@ def write_to_file(file, geojson, zoom): file.write(flt_fmt % float(token)) else: - file.write(token) \ No newline at end of file + file.write(token) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 632b8f92..c7a3307e 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -344,12 +344,12 @@ def save(self, out, format): mvt.encode(out, features) elif format == 'JSON': - geojson.encode(out, features, self.zoom, self.clip) + geojson.encode(out, features, self.zoom) elif format == 'TopoJSON': ll = SphericalMercator().projLocation(Point(*self.bounds[0:2])) ur = SphericalMercator().projLocation(Point(*self.bounds[2:4])) - topojson.encode(out, features, (ll.lon, ll.lat, ur.lon, ur.lat), self.clip) + topojson.encode(out, features, (ll.lon, ll.lat, ur.lon, ur.lat)) elif format == 'OpenScienceMap': oscimap.encode(out, features, self.coord, self.layer_name) @@ -373,12 +373,12 @@ def save(self, out, format): mvt.encode(out, []) elif format == 'JSON': - geojson.encode(out, [], 0, False) + geojson.encode(out, [], 0) elif format == 'TopoJSON': ll = SphericalMercator().projLocation(Point(*self.bounds[0:2])) ur = SphericalMercator().projLocation(Point(*self.bounds[2:4])) - topojson.encode(out, [], (ll.lon, ll.lat, ur.lon, ur.lat), False) + topojson.encode(out, [], (ll.lon, ll.lat, ur.lon, ur.lat)) elif format == 'OpenScienceMap': oscimap.encode(out, [], None) diff --git a/TileStache/Goodies/VecTiles/topojson.py b/TileStache/Goodies/VecTiles/topojson.py index 53ee23b1..b1dc33ac 100644 --- a/TileStache/Goodies/VecTiles/topojson.py +++ b/TileStache/Goodies/VecTiles/topojson.py @@ -70,7 +70,7 @@ def decode(file): ''' raise NotImplementedError('topojson.decode() not yet written') -def encode(file, features, bounds, is_clipped): +def encode(file, features, bounds): ''' Encode a list of (WKB, property dict) features into a TopoJSON stream. Also accept three-element tuples as features: (WKB, property dict, id). @@ -85,10 +85,7 @@ def encode(file, features, bounds, is_clipped): shape = loads(feature[0]) geometry = dict(properties=feature[1]) geometries.append(geometry) - - if is_clipped: - geometry.update(dict(clipped=True)) - + if len(feature) >= 2: # ID is an optional third element in the feature tuple geometry.update(dict(id=feature[2])) From f4ce8966625fe58797496b24ceb40f73dd53e0c5 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 30 Apr 2015 11:43:01 -0400 Subject: [PATCH 093/344] Update id behavior --- TileStache/Goodies/VecTiles/geojson.py | 28 +++++++++++++----------- TileStache/Goodies/VecTiles/mapbox.py | 2 -- TileStache/Goodies/VecTiles/topojson.py | 24 ++++++++++---------- TileStache/Goodies/VecTiles/transform.py | 28 ++++++++++++++---------- 4 files changed, 43 insertions(+), 39 deletions(-) diff --git a/TileStache/Goodies/VecTiles/geojson.py b/TileStache/Goodies/VecTiles/geojson.py index ac6b4c4e..d086916a 100644 --- a/TileStache/Goodies/VecTiles/geojson.py +++ b/TileStache/Goodies/VecTiles/geojson.py @@ -43,28 +43,30 @@ def decode(file): return features def encode(file, features, zoom): - ''' Encode a list of (WKB, property dict) features into a GeoJSON stream. - - Also accept three-element tuples as features: (WKB, property dict, id). - + ''' Encode a list of (WKB, property dict, id) features into a GeoJSON stream. + + If no id is available, pass in None + Geometries in the features list are assumed to be unprojected lon, lats. Floating point precision in the output is truncated to six digits. ''' - try: - # Assume three-element features - features = [dict(type='Feature', properties=p, geometry=loads(g).__geo_interface__, id=i) for (g, p, i) in features] + fs = [] + for feature in features: + assert len(feature) == 3 + wkb, props, fid = feature + f = dict(type='Feature', properties=props, + geometry=loads(wkb).__geo_interface__) + if fid is not None: + f['id'] = fid + fs.append(f) - except ValueError: - # Fall back to two-element features - features = [dict(type='Feature', properties=p, geometry=loads(g).__geo_interface__) for (g, p) in features] - - geojson = dict(type='FeatureCollection', features=features) + geojson = dict(type='FeatureCollection', features=fs) write_to_file(file, geojson, zoom) def merge(file, names, tiles, config, coord): ''' Retrieve a list of GeoJSON tile responses and merge them into one. - + get_tiles() retrieves data and performs basic integrity checks. ''' output = dict(zip(names, tiles)) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index 6311e0b7..889647ad 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -40,8 +40,6 @@ def get_feature_layer(name, features): _features = [] for feature in features: - if len(feature) >= 2: - feature[1].update(uid=feature[2]) _features.append({ 'geometry': feature[0], 'properties': feature[1] diff --git a/TileStache/Goodies/VecTiles/topojson.py b/TileStache/Goodies/VecTiles/topojson.py index b1dc33ac..dc79401a 100644 --- a/TileStache/Goodies/VecTiles/topojson.py +++ b/TileStache/Goodies/VecTiles/topojson.py @@ -71,29 +71,29 @@ def decode(file): raise NotImplementedError('topojson.decode() not yet written') def encode(file, features, bounds): - ''' Encode a list of (WKB, property dict) features into a TopoJSON stream. - - Also accept three-element tuples as features: (WKB, property dict, id). - + ''' Encode a list of (WKB, property dict, id) features into a TopoJSON stream. + + If no id is available, pass in None + Geometries in the features list are assumed to be unprojected lon, lats. Bounds are given in geographic coordinates as (xmin, ymin, xmax, ymax). ''' transform, forward = get_transform(bounds) geometries, arcs = list(), list() - + for feature in features: - shape = loads(feature[0]) - geometry = dict(properties=feature[1]) + wkb, props, fid = feature + shape = loads(wkb) + geometry = dict(properties=props) geometries.append(geometry) - if len(feature) >= 2: - # ID is an optional third element in the feature tuple - geometry.update(dict(id=feature[2])) - + if fid is not None: + geometry['id'] = fid + if shape.type == 'GeometryCollection': geometries.pop() continue - + elif shape.type == 'Point': geometry.update(dict(type='Point', coordinates=forward(shape.x, shape.y))) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index dbaf2307..1c70ee20 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1,7 +1,6 @@ # transformation functions to apply to features -from shapely import wkb -import md5 +from numbers import Number import re @@ -128,20 +127,25 @@ def _road_kind(properties): return 'minor_road' -def normalize_osm_id(shape, properties, fid): - if fid < 0: - properties['osm_id'] = -fid +def add_id_to_properties(shape, properties, fid): + properties['id'] = fid + return shape, properties, fid + + +def detect_osm_relation(shape, properties, fid): + # Assume all negative ids indicate the data was a relation. At the + # moment, this is true because only osm contains negative + # identifiers. Should this change, this logic would need to become + # more robust + if isinstance(fid, Number) and fid < 0: properties['osm_relation'] = True - # preserve previous behavior and use the first 10 digits of - # the md5 sum for the id on negative ids - binary = wkb.dumps(shape) - fid = md5.new(binary).hexdigest()[:10] - else: - # always use a string id - fid = str(fid) return shape, properties, fid +def remove_feature_id(shape, properties, fid): + return shape, properties, None + + def building_kind(shape, properties, fid): building = _coalesce(properties, 'building:part', 'building') if building and building != 'yes': From 9c352053413069c4ecc54abe9584b0e633b05223 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 30 Apr 2015 12:08:38 -0400 Subject: [PATCH 094/344] Pass id through to mapbox encoder --- TileStache/Goodies/VecTiles/mapbox.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py index 889647ad..30f9f98c 100644 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ b/TileStache/Goodies/VecTiles/mapbox.py @@ -1,7 +1,3 @@ -from TileStache.Core import KnownUnknown -import re -import logging - import mapbox_vector_tile # coordindates are scaled to this range within tile @@ -40,9 +36,11 @@ def get_feature_layer(name, features): _features = [] for feature in features: + wkb, props, fid = feature _features.append({ - 'geometry': feature[0], - 'properties': feature[1] + 'geometry': wkb, + 'properties': props, + 'id': fid, }) return { From 96e5f9d886d0a8871700d495598315aee0ff037e Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 30 Apr 2015 17:00:43 -0400 Subject: [PATCH 095/344] Normalize oneway tag --- TileStache/Goodies/VecTiles/transform.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 1c70ee20..2ef78c02 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -270,3 +270,23 @@ def road_sort_key(shape, properties, fid): def road_trim_properties(shape, properties, fid): properties = _remove_properties(properties, 'bridge', 'layer', 'tunnel') return shape, properties, fid + + +def _reverse_line_direction(shape): + if shape.type != 'LineString': + return False + shape.coords = shape.coords[::-1] + return True + + +def road_oneway(shape, properties, fid): + oneway = properties.get('oneway') + if oneway in ('-1', 'reverse'): + did_reverse = _reverse_line_direction(shape) + if did_reverse: + properties['oneway'] = 'yes' + elif oneway in ('true', '1'): + properties['oneway'] = 'yes' + elif oneway in ('false', '0'): + properties['oneway'] = 'no' + return shape, properties, fid From 7ff14e5b8e32d758c0f46f6612c3db655d62a2e2 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 1 May 2015 13:07:31 -0400 Subject: [PATCH 096/344] Rename format .mapbox -> .mvt --- TileStache/Goodies/VecTiles/__init__.py | 4 +- TileStache/Goodies/VecTiles/mapbox.py | 49 ---------- TileStache/Goodies/VecTiles/mvt.py | 119 ++++++++---------------- TileStache/Goodies/VecTiles/server.py | 35 +++---- 4 files changed, 54 insertions(+), 153 deletions(-) delete mode 100644 TileStache/Goodies/VecTiles/mapbox.py diff --git a/TileStache/Goodies/VecTiles/__init__.py b/TileStache/Goodies/VecTiles/__init__.py index d8761e73..808e6af9 100644 --- a/TileStache/Goodies/VecTiles/__init__.py +++ b/TileStache/Goodies/VecTiles/__init__.py @@ -1,9 +1,7 @@ ''' VecTiles implements client and server support for efficient vector tiles. VecTiles implements a TileStache Provider that returns tiles with contents -simplified, precision reduced and often clipped. The MVT format in particular -is designed for use in Mapnik with the VecTiles Datasource, which can read -binary MVT tiles. +simplified, precision reduced and often clipped. VecTiles generates tiles in two JSON formats, GeoJSON and TopoJSON. diff --git a/TileStache/Goodies/VecTiles/mapbox.py b/TileStache/Goodies/VecTiles/mapbox.py deleted file mode 100644 index 30f9f98c..00000000 --- a/TileStache/Goodies/VecTiles/mapbox.py +++ /dev/null @@ -1,49 +0,0 @@ -import mapbox_vector_tile - -# coordindates are scaled to this range within tile -extents = 4096 - -# tiles are padded by this number of pixels for the current zoom level -padding = 0 - -def decode(file): - tile = file.read() - data = mapbox_vector_tile.decode(tile) - return data # print data or write to file? - -def encode(file, features, coord, layer_name=''): - layers = [] - - layers.append(get_feature_layer(layer_name, features)) - - data = mapbox_vector_tile.encode(layers) - file.write(data) - -def merge(file, feature_layers, coord): - ''' Retrieve a list of mapbox tile responses and merge them into one. - - get_tiles() retrieves data and performs basic integrity checks. - ''' - layers = [] - - for layer in feature_layers: - layers.append(get_feature_layer(layer['name'], layer['features'])) - - data = mapbox_vector_tile.encode(layers) - file.write(data) - -def get_feature_layer(name, features): - _features = [] - - for feature in features: - wkb, props, fid = feature - _features.append({ - 'geometry': wkb, - 'properties': props, - 'id': fid, - }) - - return { - 'name': name or '', - 'features': _features - } diff --git a/TileStache/Goodies/VecTiles/mvt.py b/TileStache/Goodies/VecTiles/mvt.py index 73dc62c9..f6025d3d 100644 --- a/TileStache/Goodies/VecTiles/mvt.py +++ b/TileStache/Goodies/VecTiles/mvt.py @@ -1,91 +1,54 @@ -''' Implementation of MVT (Mapnik Vector Tiles) data format. +import mapbox_vector_tile -Mapnik's PythonDatasource.features() method can return a list of WKB features, -pairs of WKB format geometry and dictionaries of key-value pairs that are -rendered by Mapnik directly. PythonDatasource is new in Mapnik as of version -2.1.0. +# coordindates are scaled to this range within tile +extents = 4096 -More information: - http://mapnik.org/docs/v2.1.0/api/python/mapnik.PythonDatasource-class.html +# tiles are padded by this number of pixels for the current zoom level +padding = 0 -The MVT file format is a simple container for Mapnik-compatible vector tiles -that minimizes the amount of conversion performed by the renderer, in contrast -to other file formats such as GeoJSON. -An MVT file starts with 8 bytes. - - 4 bytes "\\x89MVT" - uint32 Length of body - bytes zlib-compressed body - -The following body is a zlib-compressed bytestream. When decompressed, -it starts with four bytes indicating the total feature count. +def decode(file): + tile = file.read() + data = mapbox_vector_tile.decode(tile) + return data # print data or write to file? - uint32 Feature count - bytes Stream of feature data -Each feature has two parts, a raw WKB (well-known binary) representation of -the geometry in spherical mercator and a JSON blob for feature properties. +def encode(file, features, coord, layer_name=''): + layers = [] - uint32 Length of feature WKB - bytes Raw bytes of WKB - uint32 Length of properties JSON - bytes JSON dictionary of feature properties + layers.append(get_feature_layer(layer_name, features)) -By default, encode() approximates the floating point precision of WKB geometry -to 26 bits for a significant compression improvement and no visible impact on -rendering at zoom 18 and lower. -''' -from StringIO import StringIO -from zlib import decompress as _decompress, compress as _compress -from struct import unpack as _unpack, pack as _pack -import json + data = mapbox_vector_tile.encode(layers) + file.write(data) -from .wkb import approximate_wkb -def decode(file): - ''' Decode an MVT file into a list of (WKB, property dict) features. - - Result can be passed directly to mapnik.PythonDatasource.wkb_features(). +def merge(file, feature_layers, coord): ''' - head = file.read(4) - - if head != '\x89MVT': - raise Exception('Bad head: "%s"' % head) - - body = StringIO(_decompress(file.read(_next_int(file)))) - features = [] - - for i in range(_next_int(body)): - wkb = body.read(_next_int(body)) - raw = body.read(_next_int(body)) - - props = json.loads(raw) - features.append((wkb, props)) - - return features - -def encode(file, features): - ''' Encode a list of (WKB, property dict) features into an MVT stream. - - Geometries in the features list are assumed to be in spherical mercator. - Floating point precision in the output is approximated to 26 bits. + Retrieve a list of mapbox vector tile responses and merge them into one. + + get_tiles() retrieves data and performs basic integrity checks. ''' - parts = [] - + layers = [] + + for layer in feature_layers: + layers.append(get_feature_layer(layer['name'], layer['features'])) + + data = mapbox_vector_tile.encode(layers) + file.write(data) + + +def get_feature_layer(name, features): + _features = [] + for feature in features: - wkb = approximate_wkb(feature[0]) - prop = json.dumps(feature[1]) - - parts.extend([_pack('>I', len(wkb)), wkb, _pack('>I', len(prop)), prop]) - - body = _compress(_pack('>I', len(features)) + ''.join(parts)) - - file.write('\x89MVT') - file.write(_pack('>I', len(body))) - file.write(body) - -def _next_int(file): - ''' Read the next big-endian 4-byte unsigned int from a file. - ''' - return _unpack('!I', file.read(4))[0] + wkb, props, fid = feature + _features.append({ + 'geometry': wkb, + 'properties': props, + 'id': fid, + }) + + return { + 'name': name or '', + 'features': _features + } diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index c7a3307e..b054cbe5 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -1,8 +1,7 @@ ''' Provider that returns PostGIS vector tiles in GeoJSON or MVT format. VecTiles is intended for rendering, and returns tiles with contents simplified, -precision reduced and often clipped. The MVT format in particular is designed -for use in Mapnik with the VecTiles Datasource, which can read binary MVT tiles. +precision reduced and often clipped. For a more general implementation, try the Vector provider: http://tilestache.org/doc/#vector-provider @@ -30,7 +29,7 @@ def connect(*args, **kwargs): raise err -from . import mvt, geojson, topojson, oscimap, mapbox +from . import mvt, geojson, topojson, oscimap from ...Geography import SphericalMercator from ModestMaps.Core import Point @@ -219,7 +218,7 @@ def getTypeByExtension(self, extension): ''' Get mime-type and format by file extension, one of "mvt", "json" or "topojson". ''' if extension.lower() == 'mvt': - return 'application/octet-stream+mvt', 'MVT' + return 'application/x-protobuf', 'MVT' elif extension.lower() == 'json': return 'application/json', 'JSON' @@ -230,9 +229,6 @@ def getTypeByExtension(self, extension): elif extension.lower() == 'vtm': return 'image/png', 'OpenScienceMap' # TODO: make this proper stream type, app only seems to work with png - elif extension.lower() == 'mapbox': - return 'application/x-protobuf', 'Mapbox' - else: raise ValueError(extension + " is not a valid extension") @@ -287,8 +283,8 @@ def getTypeByExtension(self, extension): elif extension.lower() == 'vtm': return 'image/png', 'OpenScienceMap' # TODO: make this proper stream type, app only seems to work with png - elif extension.lower() == 'mapbox': - return 'application/x-protobuf', 'Mapbox' + elif extension.lower() == 'mvt': + return 'application/x-protobuf', 'MVT' else: raise ValueError(extension + " is not a valid extension for responses with multiple layers") @@ -330,10 +326,9 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli self.sort_fn = sort_fn geo_query = build_query(srid, subquery, columns, bounds, tolerance, True, clip) - merc_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip) oscimap_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, oscimap.padding * tolerances[coord.zoom], oscimap.extents) - mapbox_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, mapbox.padding * tolerances[coord.zoom], mapbox.extents) - self.query = dict(TopoJSON=geo_query, JSON=geo_query, MVT=merc_query, OpenScienceMap=oscimap_query, Mapbox=mapbox_query) + mvt_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, mvt.padding * tolerances[coord.zoom], mvt.extents) + self.query = dict(TopoJSON=geo_query, JSON=geo_query, MVT=mvt_query, OpenScienceMap=oscimap_query) def save(self, out, format): ''' @@ -341,7 +336,7 @@ def save(self, out, format): features = get_features(self.dbinfo, self.query[format], self.geometry_types, self.transform_fn, self.sort_fn) if format == 'MVT': - mvt.encode(out, features) + mvt.encode(out, features, self.coord, self.layer_name) elif format == 'JSON': geojson.encode(out, features, self.zoom) @@ -354,9 +349,6 @@ def save(self, out, format): elif format == 'OpenScienceMap': oscimap.encode(out, features, self.coord, self.layer_name) - elif format == 'Mapbox': - mapbox.encode(out, features, self.coord, self.layer_name) - else: raise ValueError(format + " is not supported") @@ -370,7 +362,7 @@ def save(self, out, format): ''' ''' if format == 'MVT': - mvt.encode(out, []) + mvt.encode(out, [], None) elif format == 'JSON': geojson.encode(out, [], 0) @@ -383,9 +375,6 @@ def save(self, out, format): elif format == 'OpenScienceMap': oscimap.encode(out, [], None) - elif format == 'Mapbox': - mapbox.encode(out, [], None) - else: raise ValueError(format + " is not supported") @@ -419,15 +408,15 @@ def save(self, out, format): feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["OpenScienceMap"], layer.provider.geometry_types, layer.provider.transform_fn, layer.provider.sort_fn)}) oscimap.merge(out, feature_layers, self.coord) - elif format == 'Mapbox': + elif format == 'MVT': feature_layers = [] layers = [self.config.layers[name] for name in self.names] for layer in layers: width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["Mapbox"], layer.provider.geometry_types, layer.provider.transform_fn, layer.provider.sort_fn)}) - mapbox.merge(out, feature_layers, self.coord) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["MVT"], layer.provider.geometry_types, layer.provider.transform_fn, layer.provider.sort_fn)}) + mvt.merge(out, feature_layers, self.coord) else: raise ValueError(format + " is not supported for responses with multiple layers") From db6b890ab1a587dadd533d427cbab2bd152a5ced Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 5 May 2015 10:48:20 -0400 Subject: [PATCH 097/344] Sort places by scale rank then population --- TileStache/Goodies/VecTiles/sort.py | 31 +++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index bb5dfd9e..f17b9154 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -8,15 +8,6 @@ def _sort_features_by_key(features, key): return features -def _place_key(feature): - wkb, properties, fid = feature - admin_level = properties.get('admin_level') - admin_level_float = _to_float(admin_level) - if admin_level_float is None: - return 1000.0 - return admin_level - - def _by_feature_id(feature): wkb, properties, fid = feature return fid @@ -33,6 +24,26 @@ def _sort_by_area_then_id(features): return features +def _by_scalerank(feature): + wkb, properties, fid = feature + value_for_none = 1000 + scalerank = properties.get('scalerank', value_for_none) + return scalerank + + +def _by_population(feature): + wkb, properties, fid = feature + value_for_none = -1000 + population = properties.get('population', value_for_none) + return population + + +def _sort_by_scalerank_then_population(features): + features.sort(key=_by_population, reverse=True) + features.sort(key=_by_scalerank) + return features + + def _road_key(feature): wkb, properties, fid = feature return properties.get('sort_key') @@ -51,7 +62,7 @@ def landuse(features): def places(features): - return _sort_features_by_key(features, _place_key) + return _sort_by_scalerank_then_population(features) def pois(features): From 5cef6ea47c9bd7a98b1216581a528ab5d07b9773 Mon Sep 17 00:00:00 2001 From: Severyn Kozak Date: Tue, 2 Jun 2015 10:37:50 -0400 Subject: [PATCH 098/344] Change the simplification logic in build_query(). TileStache/Goodies/VecTiles/server.py -Fix on-border tile seams by simplifying land/water geometries before cutting them out, rather than the other way around (see the in-depth documentation comment for elaboration). -I jerry-rigged it to do that only for earth/water layers by adding a parameter called `layer_name` to `build_query()` and checking whether it's equal to "earth" or "water", but this should be turned into something that's modifiable in the config. --- TileStache/Goodies/VecTiles/server.py | 76 ++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 8 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index b054cbe5..158fc7ee 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -325,9 +325,9 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli self.transform_fn = transform_fn self.sort_fn = sort_fn - geo_query = build_query(srid, subquery, columns, bounds, tolerance, True, clip) - oscimap_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, oscimap.padding * tolerances[coord.zoom], oscimap.extents) - mvt_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, mvt.padding * tolerances[coord.zoom], mvt.extents) + geo_query = build_query(srid, subquery, columns, bounds, tolerance, True, clip, layer_name=layer_name) + oscimap_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, oscimap.padding * tolerances[coord.zoom], oscimap.extents, layer_name=layer_name) + mvt_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, mvt.padding * tolerances[coord.zoom], mvt.extents, layer_name=layer_name) self.query = dict(TopoJSON=geo_query, JSON=geo_query, MVT=mvt_query, OpenScienceMap=oscimap_query) def save(self, out, format): @@ -496,18 +496,78 @@ def get_features(dbinfo, query, geometry_types, transform_fn, sort_fn, n_try=1): return features -def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clipped, padding=0, scale=None): +def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clipped, padding=0, scale=None, layer_name=None): ''' Build and return an PostGIS query. ''' + + # bounds argument is a 4-tuple with (xmin, ymin, xmax, ymax). bbox = 'ST_MakeBox2D(ST_MakePoint(%.12f, %.12f), ST_MakePoint(%.12f, %.12f))' % (bounds[0] - padding, bounds[1] - padding, bounds[2] + padding, bounds[3] + padding) bbox = 'ST_SetSRID(%s, %d)' % (bbox, srid) geom = 'q.__geometry__' - if is_clipped: - geom = 'ST_Intersection(%s, %s)' % (geom, bbox) + # Special care must be taken when simplifying earth/water geometries + # to prevent tile border "seams" from forming: these occur when a + # geometry is split across multiple tiles (like a continuous strip + # of land or body of water) and thus, for any such tile, the part of that + # geometry inside of it lines up along one or more of its edges. If there's + # any kind of fine geometric detail near one of these edges, simplification + # might remove it in a way that makes the edge of the geometry move off the + # edge of the tile. See this example of a tile pre-simplification: + # https://cloud.githubusercontent.com/assets/4467604/7937704/aef971b4-090f-11e5-91b9-d973ef98e5ef.png + # and post-simplification: + # https://cloud.githubusercontent.com/assets/4467604/7937705/b1129dc2-090f-11e5-9341-6893a6892a36.png + # at which point a seam formed. + # + # To get around this, for any given tile bounding box, we find the + # contained/overlapping earth/wate geometries and simplify them BEFORE + # cutting out the precise tile bounding bbox (instead of cutting out the + # tile and then simplifying everything inside of it, as we do with all of + # the other layers). + + if layer_name in ['earth', 'water']: + # Simplify, then cut tile. + + if tolerance is not None: + # The problem with simplifying all contained/overlapping geometries + # for a tile before cutting out the parts that actually lie inside + # of it is that we might end up simplifying a massive geometry just + # to extract a small portion of it (think simplifying the border of + # the US just to extract the New York City coastline). To reduce the + # performance hit, we actually identify all of the candidate + # geometries, then cut out a bounding box *slightly larger* than the + # tile bbox, THEN simplify, and only then cut out the tile itself. + # This still allows us to perform simplification of the geometry + # edges outside of the tile, which prevents any seams from forming + # when we cut it out, but means that we don't have to simplify the + # entire geometry (just the small bits lying right outside the + # desired tile). + + simplification_padding = (bounds[3] - bounds[1]) * 0.1 + simplification_bbox = ( + 'ST_MakeBox2D(ST_MakePoint(%.12f, %.12f), ' + 'ST_MakePoint(%.12f, %.12f))' % ( + bounds[0] - simplification_padding, + bounds[1] - simplification_padding, + bounds[2] + simplification_padding, + bounds[3] + simplification_padding)) + simplification_bbox = 'ST_SetSrid(%s, %d)' % ( + simplification_bbox, srid) + + geom = 'ST_Intersection(%s, %s)' % (geom, simplification_bbox) + geom = 'ST_MakeValid(ST_SimplifyPreserveTopology(%s, %.12f))' % ( + geom, tolerance) - if tolerance is not None: - geom = 'ST_SimplifyPreserveTopology(%s, %.12f)' % (geom, tolerance) + if is_clipped: + geom = 'ST_Intersection(%s, %s)' % (geom, bbox) + + else: + # Cut tile, then simplify. + + if is_clipped: + geom = 'ST_Intersection(%s, %s)' % (geom, bbox) + + if tolerance is not None: + geom = 'ST_SimplifyPreserveTopology(%s, %.12f)' % (geom, tolerance) if is_geo: geom = 'ST_Transform(%s, 4326)' % geom From b6dbd7d1918653c0cdb2b70ed96d337b39c4181b Mon Sep 17 00:00:00 2001 From: Severyn Kozak Date: Tue, 2 Jun 2015 11:43:04 -0400 Subject: [PATCH 099/344] Add a simplify_before_intersect parameter. TileStache/Goodies/VecTiles/server.py -Add a `simplify_before_intersect` instance variable to `Provider` and `Response`, and an identical parameter to `build_query()`. This controls the order of tile simplification/cutting in `build_query()`, and can be set in the Tilestache config file. --- TileStache/Goodies/VecTiles/server.py | 36 ++++++++++++++------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 158fc7ee..6a6ff1f5 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -144,7 +144,7 @@ class Provider: } } ''' - def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, simplify_until=16, suppress_simplification=(), geometry_types=None, transform_fns=None, sort_fn=None): + def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, simplify_until=16, suppress_simplification=(), geometry_types=None, transform_fns=None, sort_fn=None, simplify_before_intersect=False): ''' ''' self.layer = layer @@ -166,6 +166,7 @@ def __init__(self, layer, dbinfo, queries, clip=True, srid=900913, simplify=1.0, else: self.sort_fn_name = None self.sort_fn = None + self.simplify_before_intersect = simplify_before_intersect self.queries = [] self.columns = {} @@ -212,7 +213,7 @@ def renderTile(self, width, height, srs, coord): else: tolerance = self.simplify * tolerances[coord.zoom] if coord.zoom < self.simplify_until else None - return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name(), self.geometry_types, self.transform_fn, self.sort_fn) + return Response(self.dbinfo, self.srid, query, self.columns[query], bounds, tolerance, coord.zoom, self.clip, coord, self.layer.name(), self.geometry_types, self.transform_fn, self.sort_fn, self.simplify_before_intersect) def getTypeByExtension(self, extension): ''' Get mime-type and format by file extension, one of "mvt", "json" or "topojson". @@ -310,7 +311,7 @@ def __exit__(self, type, value, traceback): class Response: ''' ''' - def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip, coord, layer_name, geometry_types, transform_fn, sort_fn): + def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, clip, coord, layer_name, geometry_types, transform_fn, sort_fn, simplify_before_intersect): ''' Create a new response object with Postgres connection info and a query. bounds argument is a 4-tuple with (xmin, ymin, xmax, ymax). @@ -325,9 +326,9 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli self.transform_fn = transform_fn self.sort_fn = sort_fn - geo_query = build_query(srid, subquery, columns, bounds, tolerance, True, clip, layer_name=layer_name) - oscimap_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, oscimap.padding * tolerances[coord.zoom], oscimap.extents, layer_name=layer_name) - mvt_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, mvt.padding * tolerances[coord.zoom], mvt.extents, layer_name=layer_name) + geo_query = build_query(srid, subquery, columns, bounds, tolerance, True, clip, simplify_before_intersect=simplify_before_intersect) + oscimap_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, oscimap.padding * tolerances[coord.zoom], oscimap.extents, simplify_before_intersect=simplify_before_intersect) + mvt_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, mvt.padding * tolerances[coord.zoom], mvt.extents, simplify_before_intersect=simplify_before_intersect) self.query = dict(TopoJSON=geo_query, JSON=geo_query, MVT=mvt_query, OpenScienceMap=oscimap_query) def save(self, out, format): @@ -496,7 +497,7 @@ def get_features(dbinfo, query, geometry_types, transform_fn, sort_fn, n_try=1): return features -def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clipped, padding=0, scale=None, layer_name=None): +def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clipped, padding=0, scale=None, simplify_before_intersect=False): ''' Build and return an PostGIS query. ''' @@ -505,26 +506,27 @@ def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clippe bbox = 'ST_SetSRID(%s, %d)' % (bbox, srid) geom = 'q.__geometry__' - # Special care must be taken when simplifying earth/water geometries - # to prevent tile border "seams" from forming: these occur when a - # geometry is split across multiple tiles (like a continuous strip - # of land or body of water) and thus, for any such tile, the part of that - # geometry inside of it lines up along one or more of its edges. If there's - # any kind of fine geometric detail near one of these edges, simplification - # might remove it in a way that makes the edge of the geometry move off the - # edge of the tile. See this example of a tile pre-simplification: + # Special care must be taken when simplifying certain geometries (like those + # in the earth/water layer) to prevent tile border "seams" from forming: + # these occur when a geometry is split across multiple tiles (like a + # continuous strip of land or body of water) and thus, for any such tile, + # the part of that geometry inside of it lines up along one or more of its + # edges. If there's any kind of fine geometric detail near one of these + # edges, simplification might remove it in a way that makes the edge of the + # geometry move off the edge of the tile. See this example of a tile + # pre-simplification: # https://cloud.githubusercontent.com/assets/4467604/7937704/aef971b4-090f-11e5-91b9-d973ef98e5ef.png # and post-simplification: # https://cloud.githubusercontent.com/assets/4467604/7937705/b1129dc2-090f-11e5-9341-6893a6892a36.png # at which point a seam formed. # # To get around this, for any given tile bounding box, we find the - # contained/overlapping earth/wate geometries and simplify them BEFORE + # contained/overlapping geometries and simplify them BEFORE # cutting out the precise tile bounding bbox (instead of cutting out the # tile and then simplifying everything inside of it, as we do with all of # the other layers). - if layer_name in ['earth', 'water']: + if simplify_before_intersect: # Simplify, then cut tile. if tolerance is not None: From d52e54975f6ec2d11f63db13934047e7cd5fe588 Mon Sep 17 00:00:00 2001 From: Severyn Kozak Date: Wed, 3 Jun 2015 12:30:30 -0400 Subject: [PATCH 100/344] Add padding to simplification_padding. --- TileStache/Goodies/VecTiles/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 6a6ff1f5..15477ab3 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -544,7 +544,7 @@ def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clippe # entire geometry (just the small bits lying right outside the # desired tile). - simplification_padding = (bounds[3] - bounds[1]) * 0.1 + simplification_padding = padding + (bounds[3] - bounds[1]) * 0.1 simplification_bbox = ( 'ST_MakeBox2D(ST_MakePoint(%.12f, %.12f), ' 'ST_MakePoint(%.12f, %.12f))' % ( From 7cbc7fb913a0aaa2f01349d5d48f1b34e09c1880 Mon Sep 17 00:00:00 2001 From: Severyn Kozak Date: Wed, 3 Jun 2015 15:12:58 -0400 Subject: [PATCH 101/344] Assert that is_clipped=True if simplify_before_intersect=True. TileStache/Goodies/VecTiles/server.py -If `simplify_before_intersect` is true, then `is_clipped` should be true as well by definition; if it's not, it's probably a config error, so `assert` that it is. --- TileStache/Goodies/VecTiles/server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 15477ab3..856b9b26 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -559,8 +559,9 @@ def build_query(srid, subquery, subcolumns, bounds, tolerance, is_geo, is_clippe geom = 'ST_MakeValid(ST_SimplifyPreserveTopology(%s, %.12f))' % ( geom, tolerance) - if is_clipped: - geom = 'ST_Intersection(%s, %s)' % (geom, bbox) + assert is_clipped, 'If simplify_before_intersect=True, ' \ + 'is_clipped should be True as well' + geom = 'ST_Intersection(%s, %s)' % (geom, bbox) else: # Cut tile, then simplify. From 9ef9f4adf3c7ba550f19d4129648ef1f84a8f68d Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 16 Jun 2015 10:53:57 -0400 Subject: [PATCH 102/344] Remove unused import --- TileStache/Goodies/VecTiles/sort.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index f17b9154..cb607c75 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -1,7 +1,5 @@ # sort functions to apply to features -from transform import _to_float - def _sort_features_by_key(features, key): features.sort(key=key) From 011decc9531beed876bbbaa09d4af6a554faf90b Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 16 Jun 2015 11:05:45 -0400 Subject: [PATCH 103/344] Correct sorting by id function --- TileStache/Goodies/VecTiles/sort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index cb607c75..bfdafad7 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -8,7 +8,7 @@ def _sort_features_by_key(features, key): def _by_feature_id(feature): wkb, properties, fid = feature - return fid + return properties.get('id') def _by_area(feature): From 9ddf505701c517efed56f7728e07e19bf17f9843 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 16 Jun 2015 11:09:11 -0400 Subject: [PATCH 104/344] Simplify sorting by feature property --- TileStache/Goodies/VecTiles/sort.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index bfdafad7..fb51e336 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -6,19 +6,19 @@ def _sort_features_by_key(features, key): return features -def _by_feature_id(feature): - wkb, properties, fid = feature - return properties.get('id') +def _by_feature_property(property_name): + def _feature_sort_by_property(feature): + wkb, properties, fid = feature + return properties.get(property_name) + return _feature_sort_by_property -def _by_area(feature): - wkb, properties, fid = feature - return properties.get('area') +_by_feature_id = _by_feature_property('id') def _sort_by_area_then_id(features): features.sort(key=_by_feature_id) - features.sort(key=_by_area, reverse=True) + features.sort(key=_by_feature_property('area'), reverse=True) return features @@ -42,11 +42,6 @@ def _sort_by_scalerank_then_population(features): return features -def _road_key(feature): - wkb, properties, fid = feature - return properties.get('sort_key') - - def buildings(features): return _sort_by_area_then_id(features) @@ -68,7 +63,7 @@ def pois(features): def roads(features): - return _sort_features_by_key(features, _road_key) + return _sort_features_by_key(features, _by_feature_property('sort_key')) def water(features): From c511ca9e2bcb2cf7112a121e82c8dcdc5c8d07f4 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 17 Jul 2015 17:18:43 -0400 Subject: [PATCH 105/344] Add transforms to support query updates --- TileStache/Goodies/VecTiles/sort.py | 4 ++ TileStache/Goodies/VecTiles/transform.py | 87 ++++++++++++++++++++++++ setup.py | 3 +- 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index fb51e336..833b8a13 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -68,3 +68,7 @@ def roads(features): def water(features): return _sort_by_area_then_id(features) + + +def transit(features): + return _sort_features_by_key(features, _by_feature_id) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 2ef78c02..1d8939ad 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1,6 +1,8 @@ # transformation functions to apply to features from numbers import Number +from StreetNames import short_street_name +import decimal import re @@ -189,11 +191,21 @@ def building_trim_properties(shape, properties, fid): def road_kind(shape, properties, fid): + source = properties.get('source') + assert source, 'Missing source in road query' + if source == 'naturalearthdata.com': + return shape, properties, fid + properties['kind'] = _road_kind(properties) return shape, properties, fid def road_classifier(shape, properties, fid): + source = properties.get('source') + assert source, 'Missing source in road query' + if source == 'naturalearthdata.com': + return shape, properties, fid + highway = properties.get('highway') tunnel = properties.get('tunnel') bridge = properties.get('bridge') @@ -290,3 +302,78 @@ def road_oneway(shape, properties, fid): elif oneway in ('false', '0'): properties['oneway'] = 'no' return shape, properties, fid + + +def road_abbreviate_name(shape, properties, fid): + name = properties.get('name', None) + if not name: + return shape, properties, fid + short_name = short_street_name(name) + properties['name'] = short_name + return shape, properties, fid + + +def route_name(shape, properties, fid): + route_name = properties.get('route_name', '') + if route_name: + name = properties.get('name', '') + if route_name == name: + del properties['route_name'] + return shape, properties, fid + + +def tags_create_dict(shape, properties, fid): + tags_hstore = properties.get('tags') + if tags_hstore: + tags = dict(tags_hstore) + properties['tags'] = tags + return shape, properties, fid + + +def tags_remove(shape, properties, fid): + properties.pop('tags', None) + return shape, properties, fid + + +tag_name_alternates = ( + 'int_name', + 'loc_name', + 'nat_name', + 'official_name', + 'old_name', + 'reg_name', + 'short_name', +) + + +def tags_name_i18n(shape, properties, fid): + tags = properties.get('tags') + if not tags: + return shape, properties, fid + + name = properties.get('name') + if not name: + return shape, properties, fid + + for k, v in tags.items(): + if (k.startswith('name:') and v != name or + k.startswith('alt_name:') and v != name or + k.startswith('alt_name_') and v != name or + k.startswith('old_name:') and v != name): + properties[k] = v + + for alt_tag_name_candidate in tag_name_alternates: + alt_tag_name_value = tags.get(alt_tag_name_candidate) + if alt_tag_name_value and alt_tag_name_value != name: + properties[alt_tag_name_candidate] = alt_tag_name_value + + return shape, properties, fid + + +def update_scalerank_type(shape, properties, fid): + # some ne datasets return back scalerank values as decimal.Decimal values + # convert these to floats to prevent encoders from breaking + scalerank = properties.get('scalerank') + if isinstance(scalerank, decimal.Decimal): + properties['scalerank'] = float(scalerank) + return shape, properties, fid diff --git a/setup.py b/setup.py index 95b401da..df2cbae9 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,8 @@ def is_installed(name): return False -requires = ['ModestMaps >=1.3.0','simplejson', 'Werkzeug', 'mapbox-vector-tile'] +requires = ['ModestMaps >=1.3.0', 'simplejson', 'Werkzeug', + 'mapbox-vector-tile', 'StreetNames'] # Soft dependency on PIL or Pillow if is_installed('Pillow') or sys.platform == 'win32': From 016a06e763aa2da031e23fe719171743064c638a Mon Sep 17 00:00:00 2001 From: Lucas Leblow Date: Mon, 20 Jul 2015 14:42:14 -0700 Subject: [PATCH 106/344] add hard dependency on Pillow --- setup.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/setup.py b/setup.py index df2cbae9..8dcfbdc7 100644 --- a/setup.py +++ b/setup.py @@ -17,13 +17,7 @@ def is_installed(name): requires = ['ModestMaps >=1.3.0', 'simplejson', 'Werkzeug', - 'mapbox-vector-tile', 'StreetNames'] - -# Soft dependency on PIL or Pillow -if is_installed('Pillow') or sys.platform == 'win32': - requires.append('Pillow') -else: - requires.append('PIL') + 'mapbox-vector-tile', 'StreetNames', 'Pillow'] setup(name='TileStache', From c3e512f50f7cc630efacea77671ee5032a2a76f8 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 28 Jul 2015 18:05:17 -0400 Subject: [PATCH 107/344] Removed unused transform function --- TileStache/Goodies/VecTiles/transform.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 1d8939ad..a1a4b893 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -368,12 +368,3 @@ def tags_name_i18n(shape, properties, fid): properties[alt_tag_name_candidate] = alt_tag_name_value return shape, properties, fid - - -def update_scalerank_type(shape, properties, fid): - # some ne datasets return back scalerank values as decimal.Decimal values - # convert these to floats to prevent encoders from breaking - scalerank = properties.get('scalerank') - if isinstance(scalerank, decimal.Decimal): - properties['scalerank'] = float(scalerank) - return shape, properties, fid From d14479b622b7195809a29ed89ff125a86652ac61 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 30 Jul 2015 13:59:23 -0400 Subject: [PATCH 108/344] Resolve errors for high zoom requests --- TileStache/Goodies/VecTiles/geojson.py | 5 +++-- TileStache/Goodies/VecTiles/server.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/geojson.py b/TileStache/Goodies/VecTiles/geojson.py index d086916a..afcffdd7 100644 --- a/TileStache/Goodies/VecTiles/geojson.py +++ b/TileStache/Goodies/VecTiles/geojson.py @@ -78,8 +78,9 @@ def write_to_file(file, geojson, zoom): ''' encoder = json.JSONEncoder(separators=(',', ':')) encoded = encoder.iterencode(geojson) - flt_fmt = '%%.%df' % precisions[zoom] - + precision_idx = zoom if 0 <= zoom < len(precisions) else -1 + flt_fmt = '%%.%df' % precisions[precision_idx] + for token in encoded: if charfloat_pat.match(token): # in python 2.7, we see a character followed by a float literal diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 856b9b26..53e5f58f 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -327,8 +327,10 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli self.sort_fn = sort_fn geo_query = build_query(srid, subquery, columns, bounds, tolerance, True, clip, simplify_before_intersect=simplify_before_intersect) - oscimap_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, oscimap.padding * tolerances[coord.zoom], oscimap.extents, simplify_before_intersect=simplify_before_intersect) - mvt_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, mvt.padding * tolerances[coord.zoom], mvt.extents, simplify_before_intersect=simplify_before_intersect) + tol_idx = coord.zoom if 0 <= coord.zoom < len(tolerances) else -1 + tol_val = tolerances[tol_idx] + oscimap_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, oscimap.padding * tol_val, oscimap.extents, simplify_before_intersect=simplify_before_intersect) + mvt_query = build_query(srid, subquery, columns, bounds, tolerance, False, clip, mvt.padding * tol_val, mvt.extents, simplify_before_intersect=simplify_before_intersect) self.query = dict(TopoJSON=geo_query, JSON=geo_query, MVT=mvt_query, OpenScienceMap=oscimap_query) def save(self, out, format): From 25b7b33e2ee7be70a66c163e4a35b18af2140270 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 28 Jul 2015 18:04:06 -0400 Subject: [PATCH 109/344] Add zoom to sort and transform functions --- TileStache/Goodies/VecTiles/server.py | 21 ++++++++------- TileStache/Goodies/VecTiles/sort.py | 16 +++++------ TileStache/Goodies/VecTiles/transform.py | 34 ++++++++++++------------ 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index 53e5f58f..b5aed535 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -40,9 +40,9 @@ def make_transform_fn(transform_fns): if not transform_fns: return None - def transform_fn(shape, properties, fid): + def transform_fn(shape, properties, fid, zoom): for fn in transform_fns: - shape, properties, fid = fn(shape, properties, fid) + shape, properties, fid = fn(shape, properties, fid, zoom) return shape, properties, fid return transform_fn @@ -336,7 +336,7 @@ def __init__(self, dbinfo, srid, subquery, columns, bounds, tolerance, zoom, cli def save(self, out, format): ''' ''' - features = get_features(self.dbinfo, self.query[format], self.geometry_types, self.transform_fn, self.sort_fn) + features = get_features(self.dbinfo, self.query[format], self.geometry_types, self.transform_fn, self.sort_fn, self.coord.zoom) if format == 'MVT': mvt.encode(out, features, self.coord, self.layer_name) @@ -408,7 +408,7 @@ def save(self, out, format): width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["OpenScienceMap"], layer.provider.geometry_types, layer.provider.transform_fn, layer.provider.sort_fn)}) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["OpenScienceMap"], layer.provider.geometry_types, layer.provider.transform_fn, layer.provider.sort_fn, self.coord.zoom)}) oscimap.merge(out, feature_layers, self.coord) elif format == 'MVT': @@ -418,7 +418,7 @@ def save(self, out, format): width, height = layer.dim, layer.dim tile = layer.provider.renderTile(width, height, layer.projection.srs, self.coord) if isinstance(tile,EmptyResponse): continue - feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["MVT"], layer.provider.geometry_types, layer.provider.transform_fn, layer.provider.sort_fn)}) + feature_layers.append({'name': layer.name(), 'features': get_features(tile.dbinfo, tile.query["MVT"], layer.provider.geometry_types, layer.provider.transform_fn, layer.provider.sort_fn, self.coord.zoom)}) mvt.merge(out, feature_layers, self.coord) else: @@ -460,7 +460,9 @@ def query_columns(dbinfo, srid, subquery, bounds): column_names = set(x.name for x in db.description) return column_names -def get_features(dbinfo, query, geometry_types, transform_fn, sort_fn, n_try=1): + +def get_features(dbinfo, query, geometry_types, transform_fn, sort_fn, zoom, + n_try=1): features = [] with Connection(dbinfo) as db: @@ -472,7 +474,8 @@ def get_features(dbinfo, query, geometry_types, transform_fn, sort_fn, n_try=1): raise else: return get_features(dbinfo, query, geometry_types, - transform_fn, sort_fn, n_try=n_try + 1) + transform_fn, sort_fn, zoom, + n_try=n_try + 1) for row in db.fetchall(): assert '__geometry__' in row, 'Missing __geometry__ in feature result' assert '__id__' in row, 'Missing __id__ in feature result' @@ -489,13 +492,13 @@ def get_features(dbinfo, query, geometry_types, transform_fn, sort_fn, n_try=1): props = dict((k, v) for k, v in row.items() if v is not None) if transform_fn: - shape, props, id = transform_fn(shape, props, id) + shape, props, id = transform_fn(shape, props, id, zoom) wkb = dumps(shape) features.append((wkb, props, id)) if sort_fn: - features = sort_fn(features) + features = sort_fn(features, zoom) return features diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index 833b8a13..390aaaf0 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -42,33 +42,33 @@ def _sort_by_scalerank_then_population(features): return features -def buildings(features): +def buildings(features, zoom): return _sort_by_area_then_id(features) -def earth(features): +def earth(features, zoom): return _sort_features_by_key(features, _by_feature_id) -def landuse(features): +def landuse(features, zoom): return _sort_by_area_then_id(features) -def places(features): +def places(features, zoom): return _sort_by_scalerank_then_population(features) -def pois(features): +def pois(features, zoom): return _sort_features_by_key(features, _by_feature_id) -def roads(features): +def roads(features, zoom): return _sort_features_by_key(features, _by_feature_property('sort_key')) -def water(features): +def water(features, zoom): return _sort_by_area_then_id(features) -def transit(features): +def transit(features, zoom): return _sort_features_by_key(features, _by_feature_id) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a1a4b893..c3c8b076 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -129,12 +129,12 @@ def _road_kind(properties): return 'minor_road' -def add_id_to_properties(shape, properties, fid): +def add_id_to_properties(shape, properties, fid, zoom): properties['id'] = fid return shape, properties, fid -def detect_osm_relation(shape, properties, fid): +def detect_osm_relation(shape, properties, fid, zoom): # Assume all negative ids indicate the data was a relation. At the # moment, this is true because only osm contains negative # identifiers. Should this change, this logic would need to become @@ -144,11 +144,11 @@ def detect_osm_relation(shape, properties, fid): return shape, properties, fid -def remove_feature_id(shape, properties, fid): +def remove_feature_id(shape, properties, fid, zoom): return shape, properties, None -def building_kind(shape, properties, fid): +def building_kind(shape, properties, fid, zoom): building = _coalesce(properties, 'building:part', 'building') if building and building != 'yes': kind = building @@ -159,7 +159,7 @@ def building_kind(shape, properties, fid): return shape, properties, fid -def building_height(shape, properties, fid): +def building_height(shape, properties, fid, zoom): height = _building_calc_height( properties.get('height'), properties.get('building:levels'), _building_calc_levels) @@ -170,7 +170,7 @@ def building_height(shape, properties, fid): return shape, properties, fid -def building_min_height(shape, properties, fid): +def building_min_height(shape, properties, fid, zoom): min_height = _building_calc_height( properties.get('min_height'), properties.get('building:min_levels'), _building_calc_min_levels) @@ -181,7 +181,7 @@ def building_min_height(shape, properties, fid): return shape, properties, fid -def building_trim_properties(shape, properties, fid): +def building_trim_properties(shape, properties, fid, zoom): properties = _remove_properties( properties, 'amenity', 'shop', 'tourism', @@ -190,7 +190,7 @@ def building_trim_properties(shape, properties, fid): return shape, properties, fid -def road_kind(shape, properties, fid): +def road_kind(shape, properties, fid, zoom): source = properties.get('source') assert source, 'Missing source in road query' if source == 'naturalearthdata.com': @@ -200,7 +200,7 @@ def road_kind(shape, properties, fid): return shape, properties, fid -def road_classifier(shape, properties, fid): +def road_classifier(shape, properties, fid, zoom): source = properties.get('source') assert source, 'Missing source in road query' if source == 'naturalearthdata.com': @@ -218,7 +218,7 @@ def road_classifier(shape, properties, fid): return shape, properties, fid -def road_sort_key(shape, properties, fid): +def road_sort_key(shape, properties, fid, zoom): # Calculated sort value is in the range 0 to 39 sort_val = 0 @@ -279,7 +279,7 @@ def road_sort_key(shape, properties, fid): return shape, properties, fid -def road_trim_properties(shape, properties, fid): +def road_trim_properties(shape, properties, fid, zoom): properties = _remove_properties(properties, 'bridge', 'layer', 'tunnel') return shape, properties, fid @@ -291,7 +291,7 @@ def _reverse_line_direction(shape): return True -def road_oneway(shape, properties, fid): +def road_oneway(shape, properties, fid, zoom): oneway = properties.get('oneway') if oneway in ('-1', 'reverse'): did_reverse = _reverse_line_direction(shape) @@ -304,7 +304,7 @@ def road_oneway(shape, properties, fid): return shape, properties, fid -def road_abbreviate_name(shape, properties, fid): +def road_abbreviate_name(shape, properties, fid, zoom): name = properties.get('name', None) if not name: return shape, properties, fid @@ -313,7 +313,7 @@ def road_abbreviate_name(shape, properties, fid): return shape, properties, fid -def route_name(shape, properties, fid): +def route_name(shape, properties, fid, zoom): route_name = properties.get('route_name', '') if route_name: name = properties.get('name', '') @@ -322,7 +322,7 @@ def route_name(shape, properties, fid): return shape, properties, fid -def tags_create_dict(shape, properties, fid): +def tags_create_dict(shape, properties, fid, zoom): tags_hstore = properties.get('tags') if tags_hstore: tags = dict(tags_hstore) @@ -330,7 +330,7 @@ def tags_create_dict(shape, properties, fid): return shape, properties, fid -def tags_remove(shape, properties, fid): +def tags_remove(shape, properties, fid, zoom): properties.pop('tags', None) return shape, properties, fid @@ -346,7 +346,7 @@ def tags_remove(shape, properties, fid): ) -def tags_name_i18n(shape, properties, fid): +def tags_name_i18n(shape, properties, fid, zoom): tags = properties.get('tags') if not tags: return shape, properties, fid From c9d20f73b8422c43e81317fb0ebb8e6483c90b7b Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 28 Jul 2015 18:11:12 -0400 Subject: [PATCH 110/344] Apply layer logic to road sort only for zoom >= 15 --- TileStache/Goodies/VecTiles/transform.py | 31 ++++++++++++------------ 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index c3c8b076..a5c97e79 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -258,21 +258,22 @@ def road_sort_key(shape, properties, fid, zoom): (railway == 'subway' and tunnel not in ('no', 'false'))): sort_val -= 10 - # Explicit layer is clipped to [-5, 5] range - layer = properties.get('layer') - - if layer: - layer_float = _to_float(layer) - if layer_float is not None: - layer_float = max(min(layer_float, 5), -5) - # The range of values from above is [5, 34] - # For positive layer values, we want the range to be: - # [34, 39] - if layer_float > 0: - sort_val = int(layer_float + 34) - # For negative layer values, [0, 5] - elif layer_float < 0: - sort_val = int(layer_float + 5) + # Only apply layer logic for zooms >= 15 + if zoom >= 15: + # Explicit layer is clipped to [-5, 5] range + layer = properties.get('layer') + if layer: + layer_float = _to_float(layer) + if layer_float is not None: + layer_float = max(min(layer_float, 5), -5) + # The range of values from above is [5, 34] + # For positive layer values, we want the range to be: + # [34, 39] + if layer_float > 0: + sort_val = int(layer_float + 34) + # For negative layer values, [0, 5] + elif layer_float < 0: + sort_val = int(layer_float + 5) properties['sort_key'] = sort_val From 7e4248fc57e23a37ffd2abbe404e14b90c2e560f Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 31 Jul 2015 16:02:19 -0400 Subject: [PATCH 111/344] Apply bridge/tunnel logic only for zooms >= 15 --- TileStache/Goodies/VecTiles/transform.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a5c97e79..7d2ff4c5 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -248,18 +248,16 @@ def road_sort_key(shape, properties, fid, zoom): else: sort_val += 15 - # Bridges and tunnels add +/- 10 - bridge = properties.get('bridge') - tunnel = properties.get('tunnel') - - if bridge in ('yes', 'true'): - sort_val += 10 - elif (tunnel in ('yes', 'true') or - (railway == 'subway' and tunnel not in ('no', 'false'))): - sort_val -= 10 - - # Only apply layer logic for zooms >= 15 if zoom >= 15: + # Bridges and tunnels add +/- 10 + bridge = properties.get('bridge') + tunnel = properties.get('tunnel') + if bridge in ('yes', 'true'): + sort_val += 10 + elif (tunnel in ('yes', 'true') or + (railway == 'subway' and tunnel not in ('no', 'false'))): + sort_val -= 10 + # Explicit layer is clipped to [-5, 5] range layer = properties.get('layer') if layer: From bf9b3214f7f8776a33380d2f42a67534200771f8 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 3 Aug 2015 11:34:47 -0400 Subject: [PATCH 112/344] Remove unused import --- TileStache/Goodies/VecTiles/transform.py | 1 - 1 file changed, 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 7d2ff4c5..712eeaaf 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2,7 +2,6 @@ from numbers import Number from StreetNames import short_street_name -import decimal import re From 254ed25063b0bcede151f2e27fcf8ea42db644ce Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 2 Sep 2015 15:47:58 +0100 Subject: [PATCH 113/344] Add no-op intercut implementation. --- TileStache/Goodies/VecTiles/transform.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 712eeaaf..d2cef796 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -366,3 +366,12 @@ def tags_name_i18n(shape, properties, fid, zoom): properties[alt_tag_name_candidate] = alt_tag_name_value return shape, properties, fid + +def intercut(feature_layers, base_layer, cutting_layer, attribute=None): + for feature_layer in feature_layers: + layer_datum = feature_layer['layer_datum'] + layer_name = layer_datum['name'] + if layer_name == base_layer: + return feature_layer + + return None From a3255fa53820d188fc838230b0cbd1075c64d1d7 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 2 Sep 2015 18:25:08 +0100 Subject: [PATCH 114/344] Implemented intercut with a very naive algorithm. --- TileStache/Goodies/VecTiles/transform.py | 84 +++++++++++++++++++++++- 1 file changed, 81 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index d2cef796..67b4dc07 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -367,11 +367,89 @@ def tags_name_i18n(shape, properties, fid, zoom): return shape, properties, fid -def intercut(feature_layers, base_layer, cutting_layer, attribute=None): +# intercut takes features from a base layer and cuts each +# of them against a cutting layer, splitting any base +# feature which intersects into separate inside and outside +# parts. +# +# the parts of each base feature which are outside any +# cutting feature are left unchanged. the parts which are +# inside have their property with the key given by the +# 'target_attribute' parameter set to the same value as the +# property from the cutting feature with the key given by +# the 'attribute' parameter. +# +# the intended use of this is to project attributes from one +# layer to another so that they can be styled appropriately. +# +# returns a set of feature layers with the base layer +# replaced by a cut one, or None if there's an error. +def intercut(feature_layers, base_layer, cutting_layer, attribute=None, target_attribute=None): + base = None + cutting = None + + # the target attribute can default to the attribute if + # they are distinct. but often they aren't, and that's + # why target_attribute is a separate parameter. + if target_attribute is None: + target_attribute = attribute + + # search through all the layers and extract the ones + # which have the names of the base and cutting layer. + # it would seem to be better to use a dict() for + # layers, and this will give odd results if names are + # allowed to be duplicated. for feature_layer in feature_layers: layer_datum = feature_layer['layer_datum'] layer_name = layer_datum['name'] + if layer_name == base_layer: - return feature_layer + base = feature_layer + elif layer_name == cutting_layer: + cutting = feature_layer - return None + # didn't find one or other layer - what's the appropriate + # thing to do here, raise an error? + if base is None or cutting is None: + return None + + base_features = base['features'] + cutting_features = cutting['features'] + + # TODO: this is a very simple way of doing this, and would + # probably be better replaced by something that isn't O(N^2) + # and perhaps even unioned features with the same attribute + # together first. + for cutting_feature in cutting_features: + cutting_shape, cutting_props, cutting_id = cutting_feature + cutting_attr = None + if attribute is not None and attribute in cutting_props: + cutting_attr = cutting_props[attribute] + + new_features = [] + for index, base_feature in enumerate(base_features): + base_shape, base_props, base_id = base_feature + + if base_shape.intersects(cutting_shape): + inside = base_shape.intersection(cutting_shape) + outside = base_shape.difference(cutting_shape) + + if cutting_attr is not None: + inside_props = base_props.copy() + inside_props[target_attribute] = cutting_attr + else: + inside_props = base_props + + new_features.append((inside, inside_props, base_id)) + + if not outside.is_empty: + new_features.append((outside, base_props, base_id)) + + else: + new_features.append(base_feature) + + base_features = new_features + + base['features'] = base_features + + return base From 02a5becfe8c6fe906a6bb817b999ff6e61abf18d Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 3 Sep 2015 11:44:02 +0100 Subject: [PATCH 115/344] Fixes for readability. Now asserts instead of returning None on config error. --- TileStache/Goodies/VecTiles/transform.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 67b4dc07..7c571003 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -383,7 +383,7 @@ def tags_name_i18n(shape, properties, fid, zoom): # layer to another so that they can be styled appropriately. # # returns a set of feature layers with the base layer -# replaced by a cut one, or None if there's an error. +# replaced by a cut one. def intercut(feature_layers, base_layer, cutting_layer, attribute=None, target_attribute=None): base = None cutting = None @@ -408,10 +408,8 @@ def intercut(feature_layers, base_layer, cutting_layer, attribute=None, target_a elif layer_name == cutting_layer: cutting = feature_layer - # didn't find one or other layer - what's the appropriate - # thing to do here, raise an error? - if base is None or cutting is None: - return None + assert base is not None and cutting is not None, \ + 'could not find base or cutting layer in intercut. config error' base_features = base['features'] cutting_features = cutting['features'] @@ -423,11 +421,11 @@ def intercut(feature_layers, base_layer, cutting_layer, attribute=None, target_a for cutting_feature in cutting_features: cutting_shape, cutting_props, cutting_id = cutting_feature cutting_attr = None - if attribute is not None and attribute in cutting_props: - cutting_attr = cutting_props[attribute] + if attribute is not None: + cutting_attr = cutting_props.get(attribute) new_features = [] - for index, base_feature in enumerate(base_features): + for base_feature in base_features: base_shape, base_props, base_id = base_feature if base_shape.intersects(cutting_shape): From 6fe1347f15fa59bfa51bb2a9ec96916b0a5a2252 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 3 Sep 2015 17:59:22 +0100 Subject: [PATCH 116/344] Wrote a more sophisticated version of the intercut algorithm, using a list of indexes to speed up intersection calculations. --- TileStache/Goodies/VecTiles/transform.py | 122 +++++++++++++++++------ 1 file changed, 92 insertions(+), 30 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 7c571003..b89ecace 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2,6 +2,8 @@ from numbers import Number from StreetNames import short_street_name +from collections import defaultdict +from shapely.strtree import STRtree import re @@ -367,6 +369,41 @@ def tags_name_i18n(shape, properties, fid, zoom): return shape, properties, fid +# creates a list of indexes, each one for a different cut +# attribute value, in priority order. +# +# STRtree stores geometries and returns these from the query, +# but doesn't appear to allow any other attributes to be +# stored along with the geometries. this means we have to +# separate the index out into several "layers", each having +# the same attribute value. which isn't all that much of a +# pain, as we need to cut the shapes in a certain order to +# ensure priority anyway. +# +# returns a list of (attribute value, index) pairs. +def _make_cut_index(features, attrs, attribute): + group = defaultdict(list) + for feature in features: + shape, props, fid = feature + attr = props.get(attribute) + group[attr].append(shape) + + # if the user didn't supply any options for controlling + # the cutting priority, then just make some up based on + # the attributes which are present in the dataset. + if attrs is None: + all_attrs = set() + for feature in features: + all_attrs.add(feature[1].get(attribute)) + attrs = list(all_attrs) + + cut_idxs = list() + for attr in attrs: + if attr in group: + cut_idxs.append((attr, STRtree(group[attr]))) + + return cut_idxs + # intercut takes features from a base layer and cuts each # of them against a cutting layer, splitting any base # feature which intersects into separate inside and outside @@ -382,9 +419,26 @@ def tags_name_i18n(shape, properties, fid, zoom): # the intended use of this is to project attributes from one # layer to another so that they can be styled appropriately. # -# returns a set of feature layers with the base layer -# replaced by a cut one. -def intercut(feature_layers, base_layer, cutting_layer, attribute=None, target_attribute=None): +# - feature_layers: list of layers containing both the base +# and cutting layer. +# - base_layer: str name of the base layer. +# - cutting_layer: str name of the cutting layer. +# - attribute: optional str name of the property / attribute +# to take from the cutting layer. +# - target_attribute: optional str name of the property / +# attribute to assign on the base layer. defaults to the +# same as the 'attribute' parameter. +# - cutting_attrs: list of str, the priority of the values +# to be used in the cutting operation. this ensures that +# items at the beginning of the list get cut first and +# those values have priority (won't be overridden by any +# other shape cutting). +# +# returns a feature layer which is the base layer cut by the +# cutting layer. +def intercut(feature_layers, base_layer, cutting_layer, + attribute=None, target_attribute=None, + cutting_attrs=None): base = None cutting = None @@ -411,43 +465,51 @@ def intercut(feature_layers, base_layer, cutting_layer, attribute=None, target_a assert base is not None and cutting is not None, \ 'could not find base or cutting layer in intercut. config error' + # just skip the whole thing if there's no attribute to + # cut with. + if attribute is None: + return base + base_features = base['features'] cutting_features = cutting['features'] - # TODO: this is a very simple way of doing this, and would - # probably be better replaced by something that isn't O(N^2) - # and perhaps even unioned features with the same attribute - # together first. - for cutting_feature in cutting_features: - cutting_shape, cutting_props, cutting_id = cutting_feature - cutting_attr = None - if attribute is not None: - cutting_attr = cutting_props.get(attribute) + # make an index over all the cutting features + cut_idxs = _make_cut_index(cutting_features, cutting_attrs, + attribute) - new_features = [] - for base_feature in base_features: - base_shape, base_props, base_id = base_feature + new_features = [] + for base_feature in base_features: + # we use shape to track the current remainder of the + # shape after subtracting bits which are inside cuts. + shape, base_props, base_id = base_feature - if base_shape.intersects(cutting_shape): - inside = base_shape.intersection(cutting_shape) - outside = base_shape.difference(cutting_shape) + for cutting_attr, cut_idx in cut_idxs: + cutting_shapes = cut_idx.query(shape) - if cutting_attr is not None: - inside_props = base_props.copy() - inside_props[target_attribute] = cutting_attr - else: - inside_props = base_props + for cutting_shape in cutting_shapes: + if cutting_shape.intersects(shape): + inside = shape.intersection(cutting_shape) + outside = shape.difference(cutting_shape) - new_features.append((inside, inside_props, base_id)) + if cutting_attr is not None: + inside_props = base_props.copy() + inside_props[target_attribute] = cutting_attr + else: + inside_props = base_props - if not outside.is_empty: - new_features.append((outside, base_props, base_id)) + new_features.append((inside, inside_props, base_id)) + shape = outside - else: - new_features.append(base_feature) + # if there's no geometry left outside the shape, + # then we can exit the loop early, as nothing else + # will intersect. + if shape.is_empty: + break - base_features = new_features + # if there's still geometry left outside + if not shape.is_empty: + new_features.append((shape, base_props, base_id)) - base['features'] = base_features + base['features'] = new_features return base From 88cf2e15090388be46c27767f98f4f9e2d7e2177 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 4 Sep 2015 12:18:32 -0400 Subject: [PATCH 117/344] Add transform to set ne capital properties --- TileStache/Goodies/VecTiles/transform.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 712eeaaf..d627a1a4 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -320,6 +320,17 @@ def route_name(shape, properties, fid, zoom): return shape, properties, fid +def place_ne_capital(shape, properties, fid, zoom): + source = properties.get('source', '') + if source == 'naturalearthdata.com': + kind = properties.get('kind', '') + if kind == 'Admin-0 capital': + properties['capital'] = 'yes' + elif kind == 'Admin-1 capital': + properties['state_capital'] = 'yes' + return shape, properties, fid + + def tags_create_dict(shape, properties, fid, zoom): tags_hstore = properties.get('tags') if tags_hstore: From d5ebfc55006d9c231c8c63d25d1544bee3dbd28f Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 7 Sep 2015 18:14:12 +0100 Subject: [PATCH 118/344] Added a function to add the same landuse sort order that's used in the style. Refs mapzen/vector-datasource#154. --- TileStache/Goodies/VecTiles/transform.py | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index d627a1a4..d5e227b9 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -377,3 +377,52 @@ def tags_name_i18n(shape, properties, fid, zoom): properties[alt_tag_name_candidate] = alt_tag_name_value return shape, properties, fid + + +# explicit order for some kinds of landuse +_landuse_sort_order = { + 'aerodrome': 2, + 'apron': 3, + 'cemetery': 2, + 'commercial': 2, + 'conservation': 1, + 'farm': 1, + 'farmland': 1, + 'forest': 1, + 'golf_course': 2, + 'hospital': 2, + 'nature_reserve': 1, + 'park': 1, + 'parking': 2, + 'pedestrian': 2, + 'place_of_worship': 2, + 'playground': 2, + 'railway': 2, + 'recreation_ground': 1, + 'residential': 1, + 'retail': 2, + 'runway': 3, + 'rural': 1, + 'school': 2, + 'stadium': 1, + 'university': 2, + 'urban': 1, + 'zoo': 2 +} + + +# sets a key "order" on anything with a landuse kind +# specified in the landuse sort order above. this is +# to help with maintaining a consistent order across +# post-processing steps in the server and drawing +# steps on the client. +def landuse_sort_key(shape, properties, fid, zoom): + kind = properties.get('kind') + + if kind is not None: + key = _landuse_sort_order.get(kind) + if key: + properties['order'] = key + + return shape, properties, fid + From 298e710a73c44995f682ae3dcbbc3d382c2ea65c Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 7 Sep 2015 19:16:53 +0100 Subject: [PATCH 119/344] Sort population numerically, not stringly. --- TileStache/Goodies/VecTiles/sort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index 390aaaf0..e03e7a55 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -32,7 +32,7 @@ def _by_scalerank(feature): def _by_population(feature): wkb, properties, fid = feature value_for_none = -1000 - population = properties.get('population', value_for_none) + population = int(properties.get('population', value_for_none)) return population From 6e8c6dedaae55f20d09eb55b32ca810fe66d973e Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 7 Sep 2015 19:17:41 +0100 Subject: [PATCH 120/344] Change place sorting function to sort by kind rather than scalerank. --- TileStache/Goodies/VecTiles/sort.py | 86 ++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index e03e7a55..70882aa6 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -36,9 +36,89 @@ def _by_population(feature): return population -def _sort_by_scalerank_then_population(features): +# place kinds, as used by OSM and NE, mapped to their rough +# priority values so that places can be sorted into a decent +# enough draw order on the server. +_place_sort_order = { + # zoom >= 13 + 'locality': 1302, + 'isolated_dwelling': 1301, + 'farm': 1300, + + # zoom >= 12 + 'hamlet': 1201, + 'neighbourhood': 1200, + + # zoom >= 11 + 'village': 1100, + + # zoom >= 10 + 'suburb': 1002, + 'quarter': 1001, + 'borough': 1000, + + # zoom >= 8 + # note: these have 50 added, so that some can be + # taken away for capitals & state capitals. + 'town': 851, + 'Populated place': 851, + 'city': 850, + 'Admin-0 capital': 850, + 'Admin-1 capital': 850, + + # zoom >= 4 + 'province': 401, + 'state': 400, + + # zoom >= 3 + 'sea': 300, + + # always on + 'country': 102, + 'ocean': 101, + 'continent': 100, + +# these have a relatively significant level of use in +# OSM, but aren't currently selected by the query. +# perhaps we should be using these too? +# 'island': 0, +# 'islet': 0, +# 'county': 0, +# 'city_block': 0, +# 'region': 0, +# 'municipality': 0, +# 'subdistrict': 0, +# 'township': 0, +# 'archipelago': 0, +# 'country': 0, +# 'district': 0, +# 'block': 0, +# 'department': 0, +} + + +def _by_place_kind(feature): + wkb, properties, fid = feature + + kind = properties.get('kind') + state_capital = properties.get('state_capital') + capital = properties.get('capital') + + order = _place_sort_order.get(kind, 9999) + + # hmm... seems like a nasty little hack to get + # state capital status to be taken into account... + if capital == 'yes': + order -= 50 + elif state_capital == 'yes': + order -= 20 + + return order + + +def _sort_by_place_kind_then_population(features): features.sort(key=_by_population, reverse=True) - features.sort(key=_by_scalerank) + features.sort(key=_by_place_kind) return features @@ -55,7 +135,7 @@ def landuse(features, zoom): def places(features, zoom): - return _sort_by_scalerank_then_population(features) + return _sort_by_place_kind_then_population(features) def pois(features, zoom): From 3a4dbd9384f7f51641874fdd5b23e6a3f285a4df Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 7 Sep 2015 20:03:34 +0100 Subject: [PATCH 121/344] 'Fix' problem where base/cutting layer isn't available. This can happen when the client selects a subset of layers which don't include both the base and cutting layers. If the base layer isn't selected, then the appropriate thing to do, which this patch does, is skip the post-processing step. However, if the client selects the base layer, but not the cutting layer, then it's too late to do anything about it in the post-processor. The appropriate thing may be to not perform the cutting process, or it might be to change the code so that it additionally selects the cutting layer and filters it out when the client doesn't want it. --- TileStache/Goodies/VecTiles/transform.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index b89ecace..4c0d00da 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -462,8 +462,15 @@ def intercut(feature_layers, base_layer, cutting_layer, elif layer_name == cutting_layer: cutting = feature_layer - assert base is not None and cutting is not None, \ - 'could not find base or cutting layer in intercut. config error' + # base or cutting layer not available. this could happen + # because of a config problem, in which case you'd want + # it to be reported. but also can happen when the client + # selects a subset of layers which don't include either + # the base or the cutting layer. then it's not an error. + # the interesting case is when they select the base but + # not the cutting layer... + if base is None or cutting is None: + return None # just skip the whole thing if there's no attribute to # cut with. From 070e00a0e2ed1660ff8500f69e67e2be652c6465 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 9 Sep 2015 12:29:57 +0100 Subject: [PATCH 122/344] Made intercut attribute a required parameter, and added an assert for config error if it's not present. --- TileStache/Goodies/VecTiles/transform.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4c0d00da..1972909a 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -437,7 +437,7 @@ def _make_cut_index(features, attrs, attribute): # returns a feature layer which is the base layer cut by the # cutting layer. def intercut(feature_layers, base_layer, cutting_layer, - attribute=None, target_attribute=None, + attribute, target_attribute=None, cutting_attrs=None): base = None cutting = None @@ -472,10 +472,12 @@ def intercut(feature_layers, base_layer, cutting_layer, if base is None or cutting is None: return None - # just skip the whole thing if there's no attribute to - # cut with. - if attribute is None: - return base + # sanity check on the availability of the cutting + # attribute. + assert attribute is not None, \ + 'Parameter attribute to intercut was None, but ' + \ + 'should have been an attribute name. Perhaps check ' + \ + 'your configuration file and queries.' base_features = base['features'] cutting_features = cutting['features'] From 905c538c55b9547d53624e5f2d107b9377d4899f Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 9 Sep 2015 12:32:30 +0100 Subject: [PATCH 123/344] Check for not None explicitly, not just any truthy value. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index d5e227b9..f92c840e 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -421,7 +421,7 @@ def landuse_sort_key(shape, properties, fid, zoom): if kind is not None: key = _landuse_sort_order.get(kind) - if key: + if key is not None: properties['order'] = key return shape, properties, fid From 1cabcc7b0a3bb711fead3f97561270c7a36c4219 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 9 Sep 2015 18:48:37 +0100 Subject: [PATCH 124/344] Don't need extra logic any more, after #41 we can just use the explicit scalerank parameter. --- TileStache/Goodies/VecTiles/sort.py | 82 +---------------------------- 1 file changed, 1 insertion(+), 81 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index 70882aa6..3bc0ed03 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -36,89 +36,9 @@ def _by_population(feature): return population -# place kinds, as used by OSM and NE, mapped to their rough -# priority values so that places can be sorted into a decent -# enough draw order on the server. -_place_sort_order = { - # zoom >= 13 - 'locality': 1302, - 'isolated_dwelling': 1301, - 'farm': 1300, - - # zoom >= 12 - 'hamlet': 1201, - 'neighbourhood': 1200, - - # zoom >= 11 - 'village': 1100, - - # zoom >= 10 - 'suburb': 1002, - 'quarter': 1001, - 'borough': 1000, - - # zoom >= 8 - # note: these have 50 added, so that some can be - # taken away for capitals & state capitals. - 'town': 851, - 'Populated place': 851, - 'city': 850, - 'Admin-0 capital': 850, - 'Admin-1 capital': 850, - - # zoom >= 4 - 'province': 401, - 'state': 400, - - # zoom >= 3 - 'sea': 300, - - # always on - 'country': 102, - 'ocean': 101, - 'continent': 100, - -# these have a relatively significant level of use in -# OSM, but aren't currently selected by the query. -# perhaps we should be using these too? -# 'island': 0, -# 'islet': 0, -# 'county': 0, -# 'city_block': 0, -# 'region': 0, -# 'municipality': 0, -# 'subdistrict': 0, -# 'township': 0, -# 'archipelago': 0, -# 'country': 0, -# 'district': 0, -# 'block': 0, -# 'department': 0, -} - - -def _by_place_kind(feature): - wkb, properties, fid = feature - - kind = properties.get('kind') - state_capital = properties.get('state_capital') - capital = properties.get('capital') - - order = _place_sort_order.get(kind, 9999) - - # hmm... seems like a nasty little hack to get - # state capital status to be taken into account... - if capital == 'yes': - order -= 50 - elif state_capital == 'yes': - order -= 20 - - return order - - def _sort_by_place_kind_then_population(features): features.sort(key=_by_population, reverse=True) - features.sort(key=_by_place_kind) + features.sort(key=_by_scalerank) return features From 73b6efa98fe9bfa6ae5451fbcf095df4f4df7a69 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 9 Sep 2015 18:30:52 +0100 Subject: [PATCH 125/344] Add explicit, calculated default for scalerank. On features which don't already have a curated scalerank, use the place kind to calculate a default scalerank and add that as a property. --- TileStache/Goodies/VecTiles/transform.py | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index e845e16f..a1aee1d6 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -584,3 +584,52 @@ def landuse_sort_key(shape, properties, fid, zoom): return shape, properties, fid + +# place kinds, as used by OSM, mapped to their rough +# scale_ranks so that we can provide a defaulted, +# non-curated scale_rank / min_zoom value. +_default_scalerank_for_place_kind = { + 'locality': 13, + 'isolated_dwelling': 13, + 'farm': 13, + + 'hamlet': 12, + 'neighbourhood': 12, + + 'village': 11, + + 'suburb': 10, + 'quarter': 10, + 'borough': 10, + + 'town': 8, + 'city': 8, + + 'province': 4, + 'state': 4, + + 'sea': 3, + + 'country': 0, + 'ocean': 0, + 'continent': 0 +} + + +# if the feature does not have a scale_rank attribute already, +# which would have come from a curated source, then calculate +# a default one based on the kind of place it is. +def calculate_default_place_scalerank(shape, properties, fid, zoom): + # don't override an existing attribute + scalerank = properties.get('scalerank') + if scalerank is not None: + return shape, properties, fid + + # base calculation off kind + kind = properties.get('kind') + if kind is None: + return shape, properties, fid + + properties['scalerank'] = _default_scalerank_for_place_kind[kind] + + return shape, properties, fid From af942dd5d4e81a2cfc0fbcdf26e1698a61075704 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 9 Sep 2015 18:55:31 +0100 Subject: [PATCH 126/344] Add adjustment for state & national capitals. --- TileStache/Goodies/VecTiles/transform.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a1aee1d6..d82b9db3 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -630,6 +630,15 @@ def calculate_default_place_scalerank(shape, properties, fid, zoom): if kind is None: return shape, properties, fid - properties['scalerank'] = _default_scalerank_for_place_kind[kind] + scalerank = _default_scalerank_for_place_kind[kind] + + # adjust scalerank for state / country capitals + if (kind == 'city') or (kind == 'town'): + if properties.get('state_capital') == 'yes': + scalerank -= 1 + elif properties.get('capital') == 'yes': + scalerank -= 2 + + properties['scalerank'] = scalerank return shape, properties, fid From e4f9f18b8685dd5e7b17ae5cd0c00e82e6685104 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 9 Sep 2015 19:00:47 +0100 Subject: [PATCH 127/344] Don't use [] on dict when we're not sure the key will be present. Return early if there's no matching default for a kind. --- TileStache/Goodies/VecTiles/transform.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index d82b9db3..48051bcc 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -630,7 +630,9 @@ def calculate_default_place_scalerank(shape, properties, fid, zoom): if kind is None: return shape, properties, fid - scalerank = _default_scalerank_for_place_kind[kind] + scalerank = _default_scalerank_for_place_kind.get(kind) + if scalerank is None: + return shape, properties, fid # adjust scalerank for state / country capitals if (kind == 'city') or (kind == 'town'): From fb686aec31c99811403d358c60aa9da6ececa740 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 9 Sep 2015 20:19:48 +0100 Subject: [PATCH 128/344] More idiomatic and readable Python. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 48051bcc..a90118f3 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -635,7 +635,7 @@ def calculate_default_place_scalerank(shape, properties, fid, zoom): return shape, properties, fid # adjust scalerank for state / country capitals - if (kind == 'city') or (kind == 'town'): + if kind in ('city', 'town'): if properties.get('state_capital') == 'yes': scalerank -= 1 elif properties.get('capital') == 'yes': From 7e75573893e165d9bb22a0aac85325ded2591cf2 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 9 Sep 2015 16:23:49 -0400 Subject: [PATCH 129/344] Correct vtm multipoint logic --- TileStache/Goodies/VecTiles/oscimap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/oscimap.py b/TileStache/Goodies/VecTiles/oscimap.py index 69abdd5d..7dfe6d93 100644 --- a/TileStache/Goodies/VecTiles/oscimap.py +++ b/TileStache/Goodies/VecTiles/oscimap.py @@ -134,7 +134,7 @@ def addFeature(self, row, coord, this_layer): # add number of points (for multi-point) if len(geom.coordinates) > 2: logging.info('points %s' %len(geom.coordinates)) - feature.indices.add(geom.coordinates/2) + feature.indices.append(len(geom.coordinates)/2) else: # empty geometry if len(geom.index) == 0: From 6be05125302dbffbf02ab093f8486a48136f6165 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 10 Sep 2015 12:42:40 +0100 Subject: [PATCH 130/344] Add an option to only keep new geometries from intersections where they are the same type as the original. --- TileStache/Goodies/VecTiles/transform.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a90118f3..6a1e43ca 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -446,12 +446,16 @@ def _make_cut_index(features, attrs, attribute): # items at the beginning of the list get cut first and # those values have priority (won't be overridden by any # other shape cutting). +# - keep_geom_type: if truthy, then filter the output to be +# the same type as the input. defaults to True, because +# this seems like an eminently sensible behaviour. # # returns a feature layer which is the base layer cut by the # cutting layer. def intercut(feature_layers, base_layer, cutting_layer, attribute, target_attribute=None, - cutting_attrs=None): + cutting_attrs=None, + keep_geom_type=True): base = None cutting = None @@ -504,6 +508,7 @@ def intercut(feature_layers, base_layer, cutting_layer, # we use shape to track the current remainder of the # shape after subtracting bits which are inside cuts. shape, base_props, base_id = base_feature + original_geom_type = shape.geom_type for cutting_attr, cut_idx in cut_idxs: cutting_shapes = cut_idx.query(shape) @@ -519,7 +524,12 @@ def intercut(feature_layers, base_layer, cutting_layer, else: inside_props = base_props - new_features.append((inside, inside_props, base_id)) + # only keep geometries where either the + # type is the same as the original, or + # we're not trying to keep the same type. + if (not keep_geom_type or + original_geom_type == inside.geom_type): + new_features.append((inside, inside_props, base_id)) shape = outside # if there's no geometry left outside the shape, @@ -528,8 +538,11 @@ def intercut(feature_layers, base_layer, cutting_layer, if shape.is_empty: break - # if there's still geometry left outside - if not shape.is_empty: + # if there's still geometry left outside, and it's the + # same type as original (or we don't care). + if (not shape.is_empty and + (not keep_geom_type or + original_geom_type == shape.geom_type)): new_features.append((shape, base_props, base_id)) base['features'] = new_features From 916812a841cdd5b0c09b2e76457ef48f2b4940a6 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 10 Sep 2015 13:08:02 +0100 Subject: [PATCH 131/344] Refactored cutting process out as its own class, makes code a bit easier to read. --- TileStache/Goodies/VecTiles/transform.py | 151 ++++++++++++++--------- 1 file changed, 90 insertions(+), 61 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 6a1e43ca..083805ae 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -391,30 +391,92 @@ def tags_name_i18n(shape, properties, fid, zoom): # the same attribute value. which isn't all that much of a # pain, as we need to cut the shapes in a certain order to # ensure priority anyway. -# -# returns a list of (attribute value, index) pairs. -def _make_cut_index(features, attrs, attribute): - group = defaultdict(list) - for feature in features: - shape, props, fid = feature - attr = props.get(attribute) - group[attr].append(shape) - - # if the user didn't supply any options for controlling - # the cutting priority, then just make some up based on - # the attributes which are present in the dataset. - if attrs is None: - all_attrs = set() +class _Cutter: + def __init__(self, features, attrs, attribute, + target_attribute, keep_geom_type): + group = defaultdict(list) for feature in features: - all_attrs.add(feature[1].get(attribute)) - attrs = list(all_attrs) + shape, props, fid = feature + attr = props.get(attribute) + group[attr].append(shape) + + # if the user didn't supply any options for controlling + # the cutting priority, then just make some up based on + # the attributes which are present in the dataset. + if attrs is None: + all_attrs = set() + for feature in features: + all_attrs.add(feature[1].get(attribute)) + attrs = list(all_attrs) + + cut_idxs = list() + for attr in attrs: + if attr in group: + cut_idxs.append((attr, STRtree(group[attr]))) + + self.attribute = attribute + self.target_attribute = target_attribute + self.cut_idxs = cut_idxs + self.keep_geom_type = keep_geom_type + self.new_features = [] + + + # cut up the argument shape, projecting the configured + # attribute to the properties of the intersecting parts + # of the shape. adds all the cut up bits to the + # new_features list. + def cut(self, shape, props, fid): + original_geom_type = shape.geom_type + + for cutting_attr, cut_idx in self.cut_idxs: + cutting_shapes = cut_idx.query(shape) + + for cutting_shape in cutting_shapes: + if cutting_shape.intersects(shape): + shape = self._intersect( + shape, props, fid, cutting_shape, + cutting_attr, original_geom_type) - cut_idxs = list() - for attr in attrs: - if attr in group: - cut_idxs.append((attr, STRtree(group[attr]))) + # if there's no geometry left outside the + # shape, then we can exit the loop early, as + # nothing else will intersect. + if shape.is_empty: + break + + # if there's still geometry left outside, then it + # keeps the old, unaltered properties. + self._add(shape, props, fid, original_geom_type) + + + # only keep geometries where either the type is the + # same as the original, or we're not trying to keep the + # same type. + def _add(self, shape, props, fid, original_geom_type): + if (not shape.is_empty and + (not self.keep_geom_type or + original_geom_type == shape.geom_type)): + self.new_features.append((shape, props, fid)) - return cut_idxs + + # intersects the shape with the cutting shape and + # handles attribute projection. anything "inside" is + # kept as it must have intersected the highest + # priority cutting shape already. the remainder is + # returned. + def _intersect(self, shape, props, fid, cutting_shape, + cutting_attr, original_geom_type): + inside = shape.intersection(cutting_shape) + outside = shape.difference(cutting_shape) + + if cutting_attr is not None: + inside_props = props.copy() + inside_props[self.target_attribute] = cutting_attr + else: + inside_props = props + + self._add(inside, inside_props, fid, + original_geom_type) + return outside # intercut takes features from a base layer and cuts each @@ -499,53 +561,20 @@ def intercut(feature_layers, base_layer, cutting_layer, base_features = base['features'] cutting_features = cutting['features'] - # make an index over all the cutting features - cut_idxs = _make_cut_index(cutting_features, cutting_attrs, - attribute) + # make a cutter object to help out + cutter = _Cutter(cutting_features, cutting_attrs, + attribute, target_attribute, + keep_geom_type) new_features = [] for base_feature in base_features: # we use shape to track the current remainder of the # shape after subtracting bits which are inside cuts. - shape, base_props, base_id = base_feature - original_geom_type = shape.geom_type + shape, props, fid = base_feature - for cutting_attr, cut_idx in cut_idxs: - cutting_shapes = cut_idx.query(shape) - - for cutting_shape in cutting_shapes: - if cutting_shape.intersects(shape): - inside = shape.intersection(cutting_shape) - outside = shape.difference(cutting_shape) - - if cutting_attr is not None: - inside_props = base_props.copy() - inside_props[target_attribute] = cutting_attr - else: - inside_props = base_props - - # only keep geometries where either the - # type is the same as the original, or - # we're not trying to keep the same type. - if (not keep_geom_type or - original_geom_type == inside.geom_type): - new_features.append((inside, inside_props, base_id)) - shape = outside - - # if there's no geometry left outside the shape, - # then we can exit the loop early, as nothing else - # will intersect. - if shape.is_empty: - break - - # if there's still geometry left outside, and it's the - # same type as original (or we don't care). - if (not shape.is_empty and - (not keep_geom_type or - original_geom_type == shape.geom_type)): - new_features.append((shape, base_props, base_id)) + cutter.cut(shape, props, fid) - base['features'] = new_features + base['features'] = cutter.new_features return base From dc735dbb0025b24d775113de13710bae3c241689 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 10 Sep 2015 13:19:45 +0100 Subject: [PATCH 132/344] Split up multi* geometries. Because outputs can be multi*, but the original geometry may be singular, then split up multis so that we can compare against the original type. --- TileStache/Goodies/VecTiles/transform.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 083805ae..e78defaf 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -4,6 +4,7 @@ from StreetNames import short_street_name from collections import defaultdict from shapely.strtree import STRtree +from shapely.geometry.base import BaseMultipartGeometry import re @@ -457,6 +458,15 @@ def _add(self, shape, props, fid, original_geom_type): original_geom_type == shape.geom_type)): self.new_features.append((shape, props, fid)) + # if it's a multi-geometry, then split it up so + # that we can compare the types of the leaves. + # note that we compare the type first, just in + # case the original was a multi*. + elif isinstance(shape, BaseMultipartGeometry): + for geom in shape.geoms: + self._add(geom, props, fid, + original_geom_type) + # intersects the shape with the cutting shape and # handles attribute projection. anything "inside" is From ecc59cbcd19ec4c3c46e58cf2da823aa82d280f9 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 10 Sep 2015 13:23:44 +0100 Subject: [PATCH 133/344] Don't use stringly comparison of geometry object types. --- TileStache/Goodies/VecTiles/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index e78defaf..bfb1825e 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -427,7 +427,7 @@ def __init__(self, features, attrs, attribute, # of the shape. adds all the cut up bits to the # new_features list. def cut(self, shape, props, fid): - original_geom_type = shape.geom_type + original_geom_type = type(shape) for cutting_attr, cut_idx in self.cut_idxs: cutting_shapes = cut_idx.query(shape) @@ -455,7 +455,7 @@ def cut(self, shape, props, fid): def _add(self, shape, props, fid, original_geom_type): if (not shape.is_empty and (not self.keep_geom_type or - original_geom_type == shape.geom_type)): + isinstance(shape, original_geom_type))): self.new_features.append((shape, props, fid)) # if it's a multi-geometry, then split it up so From 78f05f0b5d2b82ce36efc9998e16a7203bf75ca7 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 10 Sep 2015 14:06:24 +0100 Subject: [PATCH 134/344] Put function names back the way they were, so they actually describe the function. --- TileStache/Goodies/VecTiles/sort.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index 3bc0ed03..e03e7a55 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -36,7 +36,7 @@ def _by_population(feature): return population -def _sort_by_place_kind_then_population(features): +def _sort_by_scalerank_then_population(features): features.sort(key=_by_population, reverse=True) features.sort(key=_by_scalerank) return features @@ -55,7 +55,7 @@ def landuse(features, zoom): def places(features, zoom): - return _sort_by_place_kind_then_population(features) + return _sort_by_scalerank_then_population(features) def pois(features, zoom): From ecc7a48084b9d180507e64190d90bce5a94b0173 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 10 Sep 2015 15:52:10 +0100 Subject: [PATCH 135/344] Add post-processing function to create an explicit water boundaries layer. --- TileStache/Goodies/VecTiles/transform.py | 71 ++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a90118f3..4761015a 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -644,3 +644,74 @@ def calculate_default_place_scalerank(shape, properties, fid, zoom): properties['scalerank'] = scalerank return shape, properties, fid + + +# create a new layer from the boundaries of polygons in the +# base layer, subtracting any sections of the boundary which +# intersect other polygons. +# +# the purpose of this is to provide us a shoreline / river +# bank layer from the water layer without having any of the +# shoreline / river bank draw over the top of any of the base +# polygons. +# +# properties on the lines returned are the same as the +# polygon feature they came from. +# +# any features in feature_layers[layer] which aren't +# polygons will be ignored. +def exterior_boundaries(feature_layers, base_layer, + new_layer_name): + layer = None + + # search through all the layers and extract the one + # which has the name of the base layer we were given + # as a parameter. + for feature_layer in feature_layers: + layer_datum = feature_layer['layer_datum'] + layer_name = layer_datum['name'] + + if layer_name == base_layer: + layer = feature_layer + + # if we failed to find the base layer then it's + # possible the user just didn't ask for it, so return + # an empty result. + if layer is None: + return None + + features = layer['features'] + + # create an index so that we can efficiently find the + # polygons intersecting the 'current' one. + index = STRtree([f[0] for f in features]) + + new_features = list() + # loop through all the polygons, taking the boundary + # of each and subtracting any parts which are within + # other polygons. what remains (if anything) is the + # new feature. + for feature in features: + shape, props, fid = feature + if shape.geom_type in ('Polygon', 'MultiPolygon'): + boundary = shape.boundary + cutting_shapes = index.query(boundary) + for cutting_shape in cutting_shapes: + if cutting_shape is not shape: + boundary = boundary.difference(cutting_shape) + if not boundary.is_empty: + new_features.append((boundary, props.copy(), fid)) + + # make a copy of the old layer's information - it + # shouldn't matter about most of the settings, as + # post-processing is one of the last operations. + # but we need to override the name to ensure we get + # some output. + new_layer_datum = layer['layer_datum'].copy() + new_layer_datum['name'] = new_layer_name + new_layer = layer.copy() + new_layer['layer_datum'] = new_layer_datum + new_layer['features'] = new_features + new_layer['name'] = new_layer_name + + return new_layer From e9ebd20f4220c0518420afeb553000cc8e9fd804 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 10 Sep 2015 16:12:32 +0100 Subject: [PATCH 136/344] Be robust against strings which aren't (easily) parseable as integers. --- TileStache/Goodies/VecTiles/sort.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index e03e7a55..8748c893 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -31,8 +31,12 @@ def _by_scalerank(feature): def _by_population(feature): wkb, properties, fid = feature - value_for_none = -1000 - population = int(properties.get('population', value_for_none)) + default_value = -1000 + population_str = properties.get('population', default_value) + try: + population = int(population_str) + except ValueError: + population = default_value return population From 0f7eac19f8244d1972739282e0a9bd022a3577ad Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 10 Sep 2015 16:15:16 +0100 Subject: [PATCH 137/344] Kill unused local variable. --- TileStache/Goodies/VecTiles/transform.py | 1 - 1 file changed, 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index bfb1825e..bac41325 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -576,7 +576,6 @@ def intercut(feature_layers, base_layer, cutting_layer, attribute, target_attribute, keep_geom_type) - new_features = [] for base_feature in base_features: # we use shape to track the current remainder of the # shape after subtracting bits which are inside cuts. From 7fa8c43265a2f503a60c2c26b7c0cc39c92b1f87 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 10 Sep 2015 17:46:04 +0100 Subject: [PATCH 138/344] Use to_float from transform to try and parse more robustly. --- TileStache/Goodies/VecTiles/sort.py | 13 +++++++------ TileStache/Goodies/VecTiles/transform.py | 5 ++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index 8748c893..f0a487fb 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -1,3 +1,5 @@ +from transform import to_float + # sort functions to apply to features @@ -32,12 +34,11 @@ def _by_scalerank(feature): def _by_population(feature): wkb, properties, fid = feature default_value = -1000 - population_str = properties.get('population', default_value) - try: - population = int(population_str) - except ValueError: - population = default_value - return population + population_flt = to_float(properties.get('population', default_value)) + if population_flt is not None: + return int(population_flt) + else: + return default_value def _sort_by_scalerank_then_population(features): diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a90118f3..595dce19 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -7,7 +7,10 @@ import re -def _to_float(x): +# attempts to convert x to a floating point value, +# first removing some common punctuation. returns +# None if conversion failed. +def to_float(x): if x is None: return None # normalize punctuation From b75aee725a00c043d34146c9262ca19602d42662 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 10 Sep 2015 18:07:06 +0100 Subject: [PATCH 139/344] Early exit from the loop when we find the right layer. --- TileStache/Goodies/VecTiles/transform.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4761015a..a08d7792 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -673,6 +673,7 @@ def exterior_boundaries(feature_layers, base_layer, if layer_name == base_layer: layer = feature_layer + break # if we failed to find the base layer then it's # possible the user just didn't ask for it, so return From 6aaf55c59216e8c7e1fe251164c46069e5e2b12c Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 10 Sep 2015 18:20:36 +0100 Subject: [PATCH 140/344] D'oh. Fix calls to to_float. --- TileStache/Goodies/VecTiles/transform.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 595dce19..62e73762 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -29,7 +29,7 @@ def _to_float_meters(x): if x is None: return None - as_float = _to_float(x) + as_float = to_float(x) if as_float is not None: return as_float @@ -38,7 +38,7 @@ def _to_float_meters(x): # try explicit meters suffix if x.endswith(' m'): - meters_as_float = _to_float(x[:-2]) + meters_as_float = to_float(x[:-2]) if meters_as_float is not None: return meters_as_float @@ -47,8 +47,8 @@ def _to_float_meters(x): if feet_match is not None: feet = feet_match.group(1) inches = feet_match.group(2) - feet_as_float = _to_float(feet) - inches_as_float = _to_float(inches) + feet_as_float = to_float(feet) + inches_as_float = to_float(inches) total_inches = 0.0 parsed_feet_or_inches = False @@ -65,7 +65,7 @@ def _to_float_meters(x): # try and match the first number that can be parsed for number_match in number_pattern.finditer(x): potential_number = number_match.group(1) - as_float = _to_float(potential_number) + as_float = to_float(potential_number) if as_float is not None: return as_float @@ -265,7 +265,7 @@ def road_sort_key(shape, properties, fid, zoom): # Explicit layer is clipped to [-5, 5] range layer = properties.get('layer') if layer: - layer_float = _to_float(layer) + layer_float = to_float(layer) if layer_float is not None: layer_float = max(min(layer_float, 5), -5) # The range of values from above is [5, 34] From 988b5706ea2d4879d49622ce7b04d3122c8dde9a Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 11 Sep 2015 10:42:58 +0100 Subject: [PATCH 141/344] Fix a bug where we were treating an int as a str. The `to_float` function expects a `str` or None, but we were passing it an `int`. The function then tries to replace ';' and ',', but that's not a valid operation on an `int`. Instead, where the parameter is missing, we'll default to None which is handled by `to_float` and apply the `int` default afterwards. --- TileStache/Goodies/VecTiles/sort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index f0a487fb..d51f7665 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -34,7 +34,7 @@ def _by_scalerank(feature): def _by_population(feature): wkb, properties, fid = feature default_value = -1000 - population_flt = to_float(properties.get('population', default_value)) + population_flt = to_float(properties.get('population')) if population_flt is not None: return int(population_flt) else: From 5141558d77e34f0f3aff472fba37ac920a9583f0 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 11 Sep 2015 10:41:14 +0100 Subject: [PATCH 142/344] Refactoring intercut to share code with new overlap feature. This adds a new function, overlap, which will project an attribute from the cutting layer down to the base layer for any feature which overlaps in area by more than some configurable amount (default 80%). --- TileStache/Goodies/VecTiles/transform.py | 198 +++++++++++++++++------ 1 file changed, 146 insertions(+), 52 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 3e42e76e..063e2081 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -395,9 +395,13 @@ def tags_name_i18n(shape, properties, fid, zoom): # the same attribute value. which isn't all that much of a # pain, as we need to cut the shapes in a certain order to # ensure priority anyway. +# +# intersect_func is a functor passed in to control how an +# intersection is performed. it is passed class _Cutter: def __init__(self, features, attrs, attribute, - target_attribute, keep_geom_type): + target_attribute, keep_geom_type, + intersect_func): group = defaultdict(list) for feature in features: shape, props, fid = feature @@ -422,12 +426,13 @@ def __init__(self, features, attrs, attribute, self.target_attribute = target_attribute self.cut_idxs = cut_idxs self.keep_geom_type = keep_geom_type + self.intersect_func = intersect_func self.new_features = [] # cut up the argument shape, projecting the configured # attribute to the properties of the intersecting parts - # of the shape. adds all the cut up bits to the + # of the shape. adds all the selected bits to the # new_features list. def cut(self, shape, props, fid): original_geom_type = type(shape) @@ -478,8 +483,8 @@ def _add(self, shape, props, fid, original_geom_type): # returned. def _intersect(self, shape, props, fid, cutting_shape, cutting_attr, original_geom_type): - inside = shape.intersection(cutting_shape) - outside = shape.difference(cutting_shape) + inside, outside = \ + self.intersect_func(shape, cutting_shape) if cutting_attr is not None: inside_props = props.copy() @@ -491,6 +496,111 @@ def _intersect(self, shape, props, fid, cutting_shape, original_geom_type) return outside +# intersect by cutting, so that the cutting shape defines +# a part of the shape which is inside and a part which is +# outside as two separate shapes. +def _intersect_cut(shape, cutting_shape): + inside = shape.intersection(cutting_shape) + outside = shape.difference(cutting_shape) + return inside, outside + + +# intersect by looking at the overlap size. we can define +# a cut-off fraction and if that fraction or more of the +# area of the shape is within the cutting shape, it's +# inside, else outside. +# +# this is done using a closure so that we can curry away +# the fraction parameter. +def _intersect_overlap(min_fraction): + # the inner function is what will actually get + # called, but closing over min_fraction means it + # will have access to that. + def _f(shape, cutting_shape): + overlap = shape.intersection(cutting_shape).area + area = shape.area + + # need an empty shape of the same type as the + # original shape, which should be possible, as + # it seems shapely geometries all have a default + # constructor to empty. + empty = type(shape)() + + if ((area > 0) and + (overlap / area) >= min_fraction): + return shape, empty + else: + return empty, shape + return _f + + +# find a layer by iterating through all the layers. this +# would be easier if they layers were in a dict(), but +# that's a pretty invasive change. +# +# returns None if the layer can't be found. +def _find_layer(feature_layers, name): + + for feature_layer in feature_layers: + layer_datum = feature_layer['layer_datum'] + layer_name = layer_datum['name'] + + if layer_name == name: + return feature_layer + + return None + + +# shared implementation of the intercut algorithm, used +# both when cutting shapes and using overlap to determine +# inside / outsideness. +def _intercut_impl(intersect_func, feature_layers, + base_layer, cutting_layer, attribute, + target_attribute, cutting_attrs, + keep_geom_type): + # the target attribute can default to the attribute if + # they are distinct. but often they aren't, and that's + # why target_attribute is a separate parameter. + if target_attribute is None: + target_attribute = attribute + + # search through all the layers and extract the ones + # which have the names of the base and cutting layer. + # it would seem to be better to use a dict() for + # layers, and this will give odd results if names are + # allowed to be duplicated. + base = _find_layer(feature_layers, base_layer) + cutting = _find_layer(feature_layers, cutting_layer) + + # base or cutting layer not available. this could happen + # because of a config problem, in which case you'd want + # it to be reported. but also can happen when the client + # selects a subset of layers which don't include either + # the base or the cutting layer. then it's not an error. + # the interesting case is when they select the base but + # not the cutting layer... + if base is None or cutting is None: + return None + + base_features = base['features'] + cutting_features = cutting['features'] + + # make a cutter object to help out + cutter = _Cutter(cutting_features, cutting_attrs, + attribute, target_attribute, + keep_geom_type, intersect_func) + + for base_feature in base_features: + # we use shape to track the current remainder of the + # shape after subtracting bits which are inside cuts. + shape, props, fid = base_feature + + cutter.cut(shape, props, fid) + + base['features'] = cutter.new_features + + return base + # intercut takes features from a base layer and cuts each # of them against a cutting layer, splitting any base @@ -531,39 +641,6 @@ def intercut(feature_layers, base_layer, cutting_layer, attribute, target_attribute=None, cutting_attrs=None, keep_geom_type=True): - base = None - cutting = None - - # the target attribute can default to the attribute if - # they are distinct. but often they aren't, and that's - # why target_attribute is a separate parameter. - if target_attribute is None: - target_attribute = attribute - - # search through all the layers and extract the ones - # which have the names of the base and cutting layer. - # it would seem to be better to use a dict() for - # layers, and this will give odd results if names are - # allowed to be duplicated. - for feature_layer in feature_layers: - layer_datum = feature_layer['layer_datum'] - layer_name = layer_datum['name'] - - if layer_name == base_layer: - base = feature_layer - elif layer_name == cutting_layer: - cutting = feature_layer - - # base or cutting layer not available. this could happen - # because of a config problem, in which case you'd want - # it to be reported. but also can happen when the client - # selects a subset of layers which don't include either - # the base or the cutting layer. then it's not an error. - # the interesting case is when they select the base but - # not the cutting layer... - if base is None or cutting is None: - return None - # sanity check on the availability of the cutting # attribute. assert attribute is not None, \ @@ -571,24 +648,41 @@ def intercut(feature_layers, base_layer, cutting_layer, 'should have been an attribute name. Perhaps check ' + \ 'your configuration file and queries.' - base_features = base['features'] - cutting_features = cutting['features'] - - # make a cutter object to help out - cutter = _Cutter(cutting_features, cutting_attrs, - attribute, target_attribute, - keep_geom_type) - - for base_feature in base_features: - # we use shape to track the current remainder of the - # shape after subtracting bits which are inside cuts. - shape, props, fid = base_feature + return _intercut_impl(_intersect_cut, feature_layers, + base_layer, cutting_layer, attribute, + target_attribute, cutting_attrs, keep_geom_type) - cutter.cut(shape, props, fid) - base['features'] = cutter.new_features +# overlap measures the area overlap between each feature in +# the base layer and each in the cutting layer. if the +# fraction of overlap is greater than the min_fraction +# constant, then the feature in the base layer is assigned +# a property with its value derived from the overlapping +# feature from the cutting layer. +# +# the intended use of this is to project attributes from one +# layer to another so that they can be styled appropriately. +# +# it has the same parameters as intercut, see above. +# +# returns a feature layer which is the base layer with +# overlapping features having attributes projected from the +# cutting layer. +def overlap(feature_layers, base_layer, cutting_layer, + attribute, target_attribute=None, + cutting_attrs=None, + keep_geom_type=True, + min_fraction=0.8): + # sanity check on the availability of the cutting + # attribute. + assert attribute is not None, \ + 'Parameter attribute to overlap was None, but ' + \ + 'should have been an attribute name. Perhaps check ' + \ + 'your configuration file and queries.' - return base + return _intercut_impl(_intersect_overlap(min_fraction), + feature_layers, base_layer, cutting_layer, attribute, + target_attribute, cutting_attrs, keep_geom_type) # explicit order for some kinds of landuse From aab86150533c383223ea0be792ee5bc4eb74b481 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 11 Sep 2015 15:06:00 +0100 Subject: [PATCH 143/344] Allow overlap post-processor to keep features in the same layer. This is signaled by setting `new_layer_name` to None. Also added a more flexible way to build the dictionary of properties on the new feature. Finally, added a feature to buffer geometries - although it turns out this isn't useful for water polygons it might be useful for other layers in the future. --- TileStache/Goodies/VecTiles/transform.py | 126 +++++++++++++++++------ 1 file changed, 95 insertions(+), 31 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a08d7792..aa0a6faa 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -646,22 +646,67 @@ def calculate_default_place_scalerank(shape, properties, fid, zoom): return shape, properties, fid -# create a new layer from the boundaries of polygons in the -# base layer, subtracting any sections of the boundary which -# intersect other polygons. -# -# the purpose of this is to provide us a shoreline / river -# bank layer from the water layer without having any of the -# shoreline / river bank draw over the top of any of the base -# polygons. -# -# properties on the lines returned are the same as the -# polygon feature they came from. -# -# any features in feature_layers[layer] which aren't -# polygons will be ignored. +def _make_new_properties(props, props_instructions): + """ + make new properties from existing properties and a + dict of instructions. + + the algorithm is: + - where a key appears with value True, it will be + copied from the existing properties. + - where it's a dict, the values will be looked up + in that dict. + - otherwise the value will be used directly. + """ + new_props = dict() + + for k, v in props_instructions.iteritems(): + if v is True: + # this works even when props[k] = None + if k in props: + new_props[k] = props[k] + elif isinstance(v, dict): + # this will return None, which allows us to + # use the dict to set default values. + original_v = props.get(k) + if original_v in v: + new_props[k] = v[original_v] + else: + new_props[k] = v + + print "Got %s properties, %s instructions => %s" % (repr(props), repr(props_instructions), repr(new_props)) + + return new_props + def exterior_boundaries(feature_layers, base_layer, - new_layer_name): + new_layer_name=None, + prop_transform=dict(), + buffer_size=None): + """ + create new fetures from the boundaries of polygons + in the base layer, subtracting any sections of the + boundary which intersect other polygons. this is + added as a new layer if new_layer_name is not None + otherwise appended to the base layer. + + the purpose of this is to provide us a shoreline / + river bank layer from the water layer without having + any of the shoreline / river bank draw over the top + of any of the base polygons. + + properties on the lines returned are copied / adapted + from the existing layer using the new_props dict. see + _make_new_properties above for the rules. + + buffer_size determines whether any buffering will be + done to the index polygons. a judiciously small + amount of buffering can help avoid "dashing" due to + tolerance in the intersection, but will also create + small overlaps between lines. + + any features in feature_layers[layer] which aren't + polygons will be ignored. + """ layer = None # search through all the layers and extract the one @@ -694,25 +739,44 @@ def exterior_boundaries(feature_layers, base_layer, # new feature. for feature in features: shape, props, fid = feature + if shape.geom_type in ('Polygon', 'MultiPolygon'): boundary = shape.boundary cutting_shapes = index.query(boundary) + for cutting_shape in cutting_shapes: if cutting_shape is not shape: - boundary = boundary.difference(cutting_shape) + buf = cutting_shape + + if buffer_size is not None: + buf = buf.buffer(buffer_size) + + boundary = boundary.difference(buf) + if not boundary.is_empty: - new_features.append((boundary, props.copy(), fid)) - - # make a copy of the old layer's information - it - # shouldn't matter about most of the settings, as - # post-processing is one of the last operations. - # but we need to override the name to ensure we get - # some output. - new_layer_datum = layer['layer_datum'].copy() - new_layer_datum['name'] = new_layer_name - new_layer = layer.copy() - new_layer['layer_datum'] = new_layer_datum - new_layer['features'] = new_features - new_layer['name'] = new_layer_name - - return new_layer + new_props = _make_new_properties(props, + prop_transform) + new_features.append((boundary, new_props, fid)) + + if new_layer_name is None: + # no new layer requested, instead add new + # features into the same layer. + print "Adding %d new features to %s" % (len(new_features), repr(layer['name'])) + layer['features'].extend(new_features) + + return layer + + else: + # make a copy of the old layer's information - it + # shouldn't matter about most of the settings, as + # post-processing is one of the last operations. + # but we need to override the name to ensure we get + # some output. + new_layer_datum = layer['layer_datum'].copy() + new_layer_datum['name'] = new_layer_name + new_layer = layer.copy() + new_layer['layer_datum'] = new_layer_datum + new_layer['features'] = new_features + new_layer['name'] = new_layer_name + + return new_layer From 6624ec996919dc31cce1337bcb66e0275930b702 Mon Sep 17 00:00:00 2001 From: Nathaniel Kelso Date: Fri, 11 Sep 2015 09:23:06 -0700 Subject: [PATCH 144/344] add power plant, generator, substation, and revise sort order on existing landuse --- TileStache/Goodies/VecTiles/transform.py | 51 +++++++++++++----------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 063e2081..8415d5e5 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -687,33 +687,36 @@ def overlap(feature_layers, base_layer, cutting_layer, # explicit order for some kinds of landuse _landuse_sort_order = { - 'aerodrome': 2, - 'apron': 3, - 'cemetery': 2, - 'commercial': 2, - 'conservation': 1, - 'farm': 1, - 'farmland': 1, - 'forest': 1, - 'golf_course': 2, - 'hospital': 2, - 'nature_reserve': 1, - 'park': 1, - 'parking': 2, - 'pedestrian': 2, - 'place_of_worship': 2, - 'playground': 2, - 'railway': 2, - 'recreation_ground': 1, + 'aerodrome': 4, + 'apron': 5, + 'cemetery': 4, + 'commercial': 4, + 'conservation': 2, + 'farm': 3, + 'farmland': 3, + 'forest': 3, + 'generator': 3, + 'golf_course': 4, + 'hospital': 4, + 'nature_reserve': 2, + 'park': 2, + 'parking': 4, + 'pedestrian': 4, + 'place_of_worship': 4, + 'plant': 3, + 'playground': 4, + 'railway': 4, + 'recreation_ground': 4, 'residential': 1, - 'retail': 2, - 'runway': 3, + 'retail': 4, + 'runway': 5, 'rural': 1, - 'school': 2, - 'stadium': 1, - 'university': 2, + 'school': 4, + 'stadium': 3, + 'substation': 4, + 'university': 4, 'urban': 1, - 'zoo': 2 + 'zoo': 4 } From c4e9d47922c4ba1a6cb0a1963b46bfaa7727b37b Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 11 Sep 2015 17:26:13 +0100 Subject: [PATCH 145/344] Add filter function to re-map deprecated landuse kinds. --- TileStache/Goodies/VecTiles/transform.py | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 063e2081..d054ac54 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -685,6 +685,32 @@ def overlap(feature_layers, base_layer, cutting_layer, target_attribute, cutting_attrs, keep_geom_type) +# map from old or deprecated kind value to the value that we want +# it to be. +_deprecated_landuse_kinds = { + 'station': 'substation', + 'sub_station': 'substation' +} + + +def remap_deprecated_landuse_kinds(shape, properties, fid, zoom): + """ + some landuse kinds are deprecated, or can be coalesced down to + a single value. this filter implements that by remapping kind + values. + """ + + original_kind = properties.get('kind') + + if original_kind is not None: + remapped_kind = _deprecated_landuse_kinds.get(original_kind) + + if remapped_kind is not None: + properties['kind'] = remapped_kind + + return shape, properties, fid + + # explicit order for some kinds of landuse _landuse_sort_order = { 'aerodrome': 2, From a1fb13cd6ff3eaebc823a2b8f42ee1cb72739ecf Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 11 Sep 2015 20:10:06 +0100 Subject: [PATCH 146/344] D'oh, fix a couple of debug print statements. --- TileStache/Goodies/VecTiles/transform.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index aa0a6faa..ab477c69 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -674,8 +674,6 @@ def _make_new_properties(props, props_instructions): else: new_props[k] = v - print "Got %s properties, %s instructions => %s" % (repr(props), repr(props_instructions), repr(new_props)) - return new_props def exterior_boundaries(feature_layers, base_layer, @@ -761,7 +759,6 @@ def exterior_boundaries(feature_layers, base_layer, if new_layer_name is None: # no new layer requested, instead add new # features into the same layer. - print "Adding %d new features to %s" % (len(new_features), repr(layer['name'])) layer['features'].extend(new_features) return layer From 7c57093b47a33da27fae33d7f893fa002ffc5cce Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 14 Sep 2015 13:45:34 +0100 Subject: [PATCH 147/344] Add functionality to allow re-use of order property in sort. Previously, the sort order of the keys had to be passed in directly. This would have meant duplicating the order in two places, so it's better that we can re-use the order parameter in the sort. --- TileStache/Goodies/VecTiles/transform.py | 63 ++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 24640f0c..bbf9cafe 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -385,6 +385,63 @@ def tags_name_i18n(shape, properties, fid, zoom): return shape, properties, fid +def _no_none_min(a, b): + """ + Usually, `min(None, a)` will return None. This isn't + what we want, so this one will return a non-None + argument instead. This is basically the same as + treating None as greater than any other value. + """ + + if a is None: + return b + elif b is None: + return a + else: + return min(a, b) + + +def _sorted_attributes(features, attrs, attribute): + """ + When the list of attributes is a dictionary, use the + sort key parameter to order the feature attributes. + evaluate it as a function and return it. If it's not + in the right format, attrs isn't a dict then returns + None. + """ + + sort_key = attrs.get('sort_key') + reverse = attrs.get('reverse') + + assert sort_key is not None, "Configuration " + \ + "parameter 'sort_key' is missing, please " + \ + "check yout configuration." + + # first, we find the _minimum_ ordering over the + # group of key values. this is because we only do + # the intersection in groups by the cutting + # attribute, so can only sort in accordance with + # that. + group = dict() + for feature in features: + val = feature[1].get(sort_key) + key = feature[1].get(attribute) + val = _no_none_min(val, group.get(key)) + group[key] = val + + # extract the sorted list of attributes from the + # grouped (attribute, order) pairs, ordering by + # the order. + all_attrs = sorted(group.iteritems(), + key=lambda x: x[1]) + + # if we wanted the sort reversed, then reverse it + if reverse: + all_attrs = reversed(all_attrs) + + # strip out the sort key in return + return [x[0] for x in all_attrs] + # creates a list of indexes, each one for a different cut # attribute value, in priority order. # @@ -417,6 +474,12 @@ def __init__(self, features, attrs, attribute, all_attrs.add(feature[1].get(attribute)) attrs = list(all_attrs) + # alternatively, the user can specify an ordering + # function over the attributes. + elif isinstance(attrs, dict): + attrs = _sorted_attributes(features, attrs, + attribute) + cut_idxs = list() for attr in attrs: if attr in group: From ecbada3cd42dd82fca427518f6fdfed85225870b Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 14 Sep 2015 16:56:32 +0100 Subject: [PATCH 148/344] Fixed typos and combined reversing step into sort as suggested by @rmarianski. --- TileStache/Goodies/VecTiles/transform.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index bbf9cafe..a73e9c1e 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -415,7 +415,7 @@ def _sorted_attributes(features, attrs, attribute): assert sort_key is not None, "Configuration " + \ "parameter 'sort_key' is missing, please " + \ - "check yout configuration." + "check your configuration." # first, we find the _minimum_ ordering over the # group of key values. this is because we only do @@ -433,11 +433,7 @@ def _sorted_attributes(features, attrs, attribute): # grouped (attribute, order) pairs, ordering by # the order. all_attrs = sorted(group.iteritems(), - key=lambda x: x[1]) - - # if we wanted the sort reversed, then reverse it - if reverse: - all_attrs = reversed(all_attrs) + key=lambda x: x[1], reverse=bool(reverse)) # strip out the sort key in return return [x[0] for x in all_attrs] From 8ac7ae8391882db1adb212a42dbf7ba59320f3d8 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 14 Sep 2015 20:13:43 +0100 Subject: [PATCH 149/344] Switch off boundaries post-processing when less than start zoom. --- TileStache/Goodies/VecTiles/transform.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a73e9c1e..f764fa04 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -696,7 +696,7 @@ def _intercut_impl(intersect_func, feature_layers, # # returns a feature layer which is the base layer cut by the # cutting layer. -def intercut(feature_layers, base_layer, cutting_layer, +def intercut(feature_layers, zoom, base_layer, cutting_layer, attribute, target_attribute=None, cutting_attrs=None, keep_geom_type=True): @@ -727,7 +727,7 @@ def intercut(feature_layers, base_layer, cutting_layer, # returns a feature layer which is the base layer with # overlapping features having attributes projected from the # cutting layer. -def overlap(feature_layers, base_layer, cutting_layer, +def overlap(feature_layers, zoom, base_layer, cutting_layer, attribute, target_attribute=None, cutting_attrs=None, keep_geom_type=True, @@ -912,10 +912,12 @@ def _make_new_properties(props, props_instructions): return new_props -def exterior_boundaries(feature_layers, base_layer, +def exterior_boundaries(feature_layers, zoom, + base_layer, new_layer_name=None, prop_transform=dict(), - buffer_size=None): + buffer_size=None, + start_zoom=None): """ create new fetures from the boundaries of polygons in the base layer, subtracting any sections of the @@ -943,6 +945,10 @@ def exterior_boundaries(feature_layers, base_layer, """ layer = None + # don't start processing until the start zoom + if zoom < start_zoom: + return layer + # search through all the layers and extract the one # which has the name of the base layer we were given # as a parameter. From ce48dc9c95f7e685fa0d658cf4040d4e35a14691 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 14 Sep 2015 20:36:16 +0100 Subject: [PATCH 150/344] Provide a more readable, less complicated-clever default for start_zoom. After all, zooms are non-negative integers. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index f764fa04..c22a076e 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -917,7 +917,7 @@ def exterior_boundaries(feature_layers, zoom, new_layer_name=None, prop_transform=dict(), buffer_size=None, - start_zoom=None): + start_zoom=0): """ create new fetures from the boundaries of polygons in the base layer, subtracting any sections of the From 291ac8a9a04b5029f39c87416459466d8032f5d9 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 15 Sep 2015 15:07:35 +0100 Subject: [PATCH 151/344] Only index areal geometry types for differencing with water boundaries. --- TileStache/Goodies/VecTiles/transform.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index c22a076e..0fc83b83 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -969,8 +969,15 @@ def exterior_boundaries(feature_layers, zoom, features = layer['features'] # create an index so that we can efficiently find the - # polygons intersecting the 'current' one. - index = STRtree([f[0] for f in features]) + # polygons intersecting the 'current' one. Note that + # we're only interested in intersecting with other + # areal features, and that intersecting with lines can + # give some unexpected results. + indexable_features = list() + for shape, props, fid in features: + if shape.geom_type in ('Polygon', 'MultiPolygon'): + indexable_features.append(shape) + index = STRtree(indexable_features) new_features = list() # loop through all the polygons, taking the boundary From 8a4943ee48e68a7c7e45520a98dcc86bec803719 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 15 Sep 2015 16:06:32 +0100 Subject: [PATCH 152/344] Try not to use confusing jargon. Polygonal here is fine. --- TileStache/Goodies/VecTiles/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 0fc83b83..f30a3776 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -971,8 +971,8 @@ def exterior_boundaries(feature_layers, zoom, # create an index so that we can efficiently find the # polygons intersecting the 'current' one. Note that # we're only interested in intersecting with other - # areal features, and that intersecting with lines can - # give some unexpected results. + # polygonal features, and that intersecting with lines + # can give some unexpected results. indexable_features = list() for shape, props, fid in features: if shape.geom_type in ('Polygon', 'MultiPolygon'): From be6dad797cb8e25ead6162c0b0b69ae2f2f12c93 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 16 Sep 2015 15:55:03 +0100 Subject: [PATCH 153/344] Preserve left/right name translations in filter pipeline. Refs mapzen/vector-datasource#180. --- TileStache/Goodies/VecTiles/transform.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index f30a3776..e2012fec 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -374,7 +374,9 @@ def tags_name_i18n(shape, properties, fid, zoom): if (k.startswith('name:') and v != name or k.startswith('alt_name:') and v != name or k.startswith('alt_name_') and v != name or - k.startswith('old_name:') and v != name): + k.startswith('old_name:') and v != name or + k.startswith('left:name:') and v != name or + k.startswith('right:name:') and v != name): properties[k] = v for alt_tag_name_candidate in tag_name_alternates: From 42b41e8a8ad7968b8ec07b41cdc7279d79b3acb9 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 16 Sep 2015 17:55:27 +0100 Subject: [PATCH 154/344] Move break further into the loop, as we can return from the function earlier now without needing multiple breaks for the nested loops. --- TileStache/Goodies/VecTiles/transform.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index f30a3776..79bf52d7 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -505,11 +505,11 @@ def cut(self, shape, props, fid): shape, props, fid, cutting_shape, cutting_attr, original_geom_type) - # if there's no geometry left outside the - # shape, then we can exit the loop early, as - # nothing else will intersect. - if shape.is_empty: - break + # if there's no geometry left outside the + # shape, then we can exit the function + # early, as nothing else will intersect. + if shape.is_empty: + return # if there's still geometry left outside, then it # keeps the old, unaltered properties. From 16d386c9bd36908dfc82e156bbd3fbfc45e09b82 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 16 Sep 2015 17:55:43 +0100 Subject: [PATCH 155/344] Better geometry type comparison. Previously, the comparison had been on Python type. This seemed to work most of the time, but there were some odd instances of things claiming to be of `MultiLineString` type, but where `isinstance(obj, BaseMultipartGeometry)` was `False`. I have a feeling it might be something `ctypes` related. It turns out that Shapely doesn't expose the geometry type ID, so we still have to do a stringly comparison, but I've tried to make it as simple as possible and do it only once. This is the base cause of mapzen/vector-datasource#204: Objects were returned from the intercut algorithm which were MultiLineStrings, but which were not being detected as such in the `isinstance` test. This led to some road segments being dropped from the tile. --- TileStache/Goodies/VecTiles/transform.py | 86 +++++++++++++++++++----- 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 79bf52d7..b629da9d 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -438,6 +438,53 @@ def _sorted_attributes(features, attrs, attribute): # strip out the sort key in return return [x[0] for x in all_attrs] + +# the table of geometry dimensions indexed by geometry +# type name. it would be better to use geometry type ID, +# but it seems like that isn't exposed. +# +# each of these is a bit-mask, so zero dimentions is +# represented by 1, one by 2, etc... this is to support +# things like geometry collections where the type isn't +# statically known. +_GEOMETRY_DIMENSIONS = { + 'Point': 1, + 'LineString': 2, + 'LinearRing': 2, + 'Polygon': 4, + 'MultiPoint': 1, + 'MultiLineString': 2, + 'MultiPolygon': 4, + 'GeometryCollection': 0, +} + + +# returns the dimensionality of the object. so points have +# zero dimensions, lines one, polygons two. multi* variants +# have the same as their singular variant. +# +# geometry collections can hold many different types, so +# we use a bit-mask of the dimensions and recurse down to +# find the actual dimensionality of the stored set. +# +# returns a bit-mask, with these bits ORed together: +# 1: contains a point / zero-dimensional object +# 2: contains a linestring / one-dimensional object +# 4: contains a polygon / two-dimensional object +def _geom_dimensions(g): + dim = _GEOMETRY_DIMENSIONS.get(g.geom_type) + assert dim, "Unknown geometry type %s in " + \ + "transform._geom_dimensions." % \ + repr(g.geom_type) + + # recurse for geometry collections to find the true + # dimensionality of the geometry. + if dim == 0: + for part in g.geoms: + dim = dim | _geom_dimensions(g) + + return dim + # creates a list of indexes, each one for a different cut # attribute value, in priority order. # @@ -494,7 +541,7 @@ def __init__(self, features, attrs, attribute, # of the shape. adds all the selected bits to the # new_features list. def cut(self, shape, props, fid): - original_geom_type = type(shape) + original_geom_dim = _geom_dimensions(shape) for cutting_attr, cut_idx in self.cut_idxs: cutting_shapes = cut_idx.query(shape) @@ -503,7 +550,7 @@ def cut(self, shape, props, fid): if cutting_shape.intersects(shape): shape = self._intersect( shape, props, fid, cutting_shape, - cutting_attr, original_geom_type) + cutting_attr, original_geom_dim) # if there's no geometry left outside the # shape, then we can exit the function @@ -513,27 +560,30 @@ def cut(self, shape, props, fid): # if there's still geometry left outside, then it # keeps the old, unaltered properties. - self._add(shape, props, fid, original_geom_type) + self._add(shape, props, fid, original_geom_dim) # only keep geometries where either the type is the # same as the original, or we're not trying to keep the # same type. - def _add(self, shape, props, fid, original_geom_type): - if (not shape.is_empty and - (not self.keep_geom_type or - isinstance(shape, original_geom_type))): + def _add(self, shape, props, fid, original_geom_dim): + # use a custom dimension measurement here, as it + # turns out shapely geometry objects don't always + # form a hierarchy that's usable with isinstance. + shape_dim = _geom_dimensions(shape) + + # don't add empty shapes, they're completely + # useless. + if shape.is_empty: + pass + + # add the shape as-is unless we're trying to keep + # the geometry type or the geometry dimension is + # identical. + elif not self.keep_geom_type or \ + shape_dim == original_geom_dim: self.new_features.append((shape, props, fid)) - # if it's a multi-geometry, then split it up so - # that we can compare the types of the leaves. - # note that we compare the type first, just in - # case the original was a multi*. - elif isinstance(shape, BaseMultipartGeometry): - for geom in shape.geoms: - self._add(geom, props, fid, - original_geom_type) - # intersects the shape with the cutting shape and # handles attribute projection. anything "inside" is @@ -541,7 +591,7 @@ def _add(self, shape, props, fid, original_geom_type): # priority cutting shape already. the remainder is # returned. def _intersect(self, shape, props, fid, cutting_shape, - cutting_attr, original_geom_type): + cutting_attr, original_geom_dim): inside, outside = \ self.intersect_func(shape, cutting_shape) @@ -552,7 +602,7 @@ def _intersect(self, shape, props, fid, cutting_shape, inside_props = props self._add(inside, inside_props, fid, - original_geom_type) + original_geom_dim) return outside # intersect by cutting, so that the cutting shape defines From 090a961cdbf43968f0f507d86602db9360050895 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 16 Sep 2015 20:57:00 +0100 Subject: [PATCH 156/344] Assert is not None, rather than just anything truthy. Re-order statements & return for readability. --- TileStache/Goodies/VecTiles/transform.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index b629da9d..13a7d6ff 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -473,8 +473,8 @@ def _sorted_attributes(features, attrs, attribute): # 4: contains a polygon / two-dimensional object def _geom_dimensions(g): dim = _GEOMETRY_DIMENSIONS.get(g.geom_type) - assert dim, "Unknown geometry type %s in " + \ - "transform._geom_dimensions." % \ + assert dim is not None, "Unknown geometry type " + \ + "%s in transform._geom_dimensions." % \ repr(g.geom_type) # recurse for geometry collections to find the true @@ -567,21 +567,21 @@ def cut(self, shape, props, fid): # same as the original, or we're not trying to keep the # same type. def _add(self, shape, props, fid, original_geom_dim): + # don't add empty shapes, they're completely + # useless. + if shape.is_empty: + return + # use a custom dimension measurement here, as it # turns out shapely geometry objects don't always # form a hierarchy that's usable with isinstance. shape_dim = _geom_dimensions(shape) - # don't add empty shapes, they're completely - # useless. - if shape.is_empty: - pass - # add the shape as-is unless we're trying to keep # the geometry type or the geometry dimension is # identical. - elif not self.keep_geom_type or \ - shape_dim == original_geom_dim: + if not self.keep_geom_type or \ + shape_dim == original_geom_dim: self.new_features.append((shape, props, fid)) From fbbbaaf50a5521fcc08439357e28973161187cdd Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 18 Sep 2015 18:26:39 +0100 Subject: [PATCH 157/344] Ooops. Recurse on geometry part, rather than on the original geometry. Stops infinite recursion. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 99604606..a7bca1fb 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -483,7 +483,7 @@ def _geom_dimensions(g): # dimensionality of the geometry. if dim == 0: for part in g.geoms: - dim = dim | _geom_dimensions(g) + dim = dim | _geom_dimensions(part) return dim From 0547a20204f3746350414729d466731ef14bf95e Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 23 Sep 2015 18:00:01 +0100 Subject: [PATCH 158/344] Add intracut algorithm for cutting within layers. --- TileStache/Goodies/VecTiles/transform.py | 42 ++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a7bca1fb..2620406f 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -796,6 +796,48 @@ def overlap(feature_layers, zoom, base_layer, cutting_layer, target_attribute, cutting_attrs, keep_geom_type) +# intracut cuts a layer with a set of features from that same +# layer, which are then removed. +# +# for example, with water boundaries we get one set of linestrings +# from the admin polygons and another set from the original ways +# where the `maritime=yes` tag is set. we don't actually want +# separate linestrings, we just want the `maritime=yes` attribute +# on the first set of linestrings. +def intracut(feature_layers, zoom, base_layer, attribute): + # sanity check on the availability of the cutting + # attribute. + assert attribute is not None, \ + 'Parameter attribute to intracut was None, but ' + \ + 'should have been an attribute name. Perhaps check ' + \ + 'your configuration file and queries.' + + base = _find_layer(feature_layers, base_layer) + if base is None: + return None + + # unlike intracut & overlap, which work on separate layers, + # intracut separates features in the same layer into + # different sets to work on. + base_features = list() + cutting_features = list() + for shape, props, fid in base['features']: + if attribute in props: + cutting_features.append((shape, props, fid)) + else: + base_features.append((shape, props, fid)) + + cutter = _Cutter(cutting_features, None, attribute, + attribute, True, _intersect_cut) + + for shape, props, fid in base_features: + cutter.cut(shape, props, fid) + + base['features'] = cutter.new_features + + return base + + # map from old or deprecated kind value to the value that we want # it to be. _deprecated_landuse_kinds = { From a8ca195bcc49fa019d68679b9f520e76de4b0982 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 24 Sep 2015 16:34:05 -0400 Subject: [PATCH 159/344] Add tranform to normalize water `tunnel`s --- TileStache/Goodies/VecTiles/transform.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 2620406f..08d8f9a9 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -337,6 +337,16 @@ def place_ne_capital(shape, properties, fid, zoom): return shape, properties, fid +def water_tunnel(shape, properties, fid, zoom): + tunnel = properties.get('tunnel') + if tunnel in ('yes', 'true'): + properties['is_tunnel'] = 'yes' + else: + properties.pop('is_tunnel', None) + properties.pop('tunnel', None) + return shape, properties, fid + + def tags_create_dict(shape, properties, fid, zoom): tags_hstore = properties.get('tags') if tags_hstore: From 541c17b45ba31475b661d2cce3c6f5102a72df08 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 25 Sep 2015 12:06:21 -0400 Subject: [PATCH 160/344] Reverse tunnel value logic There are more valid values for tunnels, and it's better to check if something is not a tunnel instead. --- TileStache/Goodies/VecTiles/transform.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 08d8f9a9..caaa116c 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -338,12 +338,11 @@ def place_ne_capital(shape, properties, fid, zoom): def water_tunnel(shape, properties, fid, zoom): - tunnel = properties.get('tunnel') - if tunnel in ('yes', 'true'): - properties['is_tunnel'] = 'yes' - else: + tunnel = properties.pop('tunnel', None) + if tunnel in (None, 'no', 'false', '0'): properties.pop('is_tunnel', None) - properties.pop('tunnel', None) + else: + properties['is_tunnel'] = 'yes' return shape, properties, fid From 2d74239ebfe2f3c7db4a23bc8ac3e842d011c52b Mon Sep 17 00:00:00 2001 From: Nathaniel Kelso Date: Fri, 25 Sep 2015 10:16:42 -0700 Subject: [PATCH 161/344] really, .DS_Store --- .DS_Store | Bin 6148 -> 6148 bytes .gitignore | 1 + 2 files changed, 1 insertion(+) diff --git a/.DS_Store b/.DS_Store index 06f80725eaf5e4ee6ae9466d1b3c40474a841857..ec04ea62620ebc16ad3bdb29fcf97fc89373ccb3 100644 GIT binary patch delta 35 rcmZoMXfc@J&nUPtU^g?P;AS3{mvsFx<%bqo{p=dKZ$6tN`*Axtq delta 110 zcmZoMXfc@J&nUbxU^g?P@Ma#C+-fxUDyJI7ys02MPD A<^TWy diff --git a/.gitignore b/.gitignore index 7dd3483d..49657989 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ TileStache-*.*.tar.gz *.shx .coverage .idea/ +.DS_Store From 40e353754e9123558caf3a3f97e3634daf3e5f80 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 25 Sep 2015 13:23:05 -0400 Subject: [PATCH 162/344] Add kind to boundaries based on admin_level --- TileStache/Goodies/VecTiles/transform.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index caaa116c..a5fe196b 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -346,6 +346,30 @@ def water_tunnel(shape, properties, fid, zoom): return shape, properties, fid +boundary_admin_level_mapping = { + 2: 'country', + 4: 'state', + 6: 'county', +} + + +def boundary_kind(shape, properties, fid, zoom): + kind = properties.get('kind') + if kind: + return shape, properties, fid + admin_level_str = properties.get('admin_level') + if admin_level_str is None: + return shape, properties, fid + try: + admin_level_int = int(admin_level_str) + except ValueError: + return shape, properties, fid + kind = boundary_admin_level_mapping.get(admin_level_int) + if kind: + properties['kind'] = kind + return shape, properties, fid + + def tags_create_dict(shape, properties, fid, zoom): tags_hstore = properties.get('tags') if tags_hstore: From 3d9b12388cfe8cd7f1fe16c1d11783713e7aeee6 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 25 Sep 2015 16:49:28 -0400 Subject: [PATCH 163/344] Property rename from order -> sort_key --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index caaa116c..3dd3764d 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -919,7 +919,7 @@ def landuse_sort_key(shape, properties, fid, zoom): if kind is not None: key = _landuse_sort_order.get(kind) if key is not None: - properties['order'] = key + properties['sort_key'] = key return shape, properties, fid From d67b9f9d190f68d77a63ae4fe2f0d9fe3f25610b Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 25 Sep 2015 18:40:21 -0400 Subject: [PATCH 164/344] Create new changelog --- CHANGELOG => CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) rename CHANGELOG => CHANGELOG.md (99%) diff --git a/CHANGELOG b/CHANGELOG.md similarity index 99% rename from CHANGELOG rename to CHANGELOG.md index 0b6a4326..7ad5645c 100644 --- a/CHANGELOG +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +Pre-fork changelog +------------------ + 2014-05-10: 1.49.10 - Fixed Travis build. - Fixed import errors for case-insensitive filesystems. From 79a9658314fa59eb30630ab1fcefbd3f2309ffa8 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 25 Sep 2015 18:41:05 -0400 Subject: [PATCH 165/344] Version bump -> v0.3.0 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ad5645c..91e96f4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +v0.3.0 +------ +* Add intracut (intersection) algorithm for cutting within layers. +* Add smarts for dealing with maritime boundary attributes +* Add tranform for water `tunnel`s + +0.0.2 +----- +* Stable + +-------------------------------------------------------------------------------- + Pre-fork changelog ------------------ From 58d11becac02ffbe3b22cd934c4efd52c77afd93 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 24 Sep 2015 17:53:22 -0400 Subject: [PATCH 166/344] Add transform to dynamically create label layers --- TileStache/Goodies/VecTiles/transform.py | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 3dd3764d..6c1b01c2 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1129,3 +1129,33 @@ def exterior_boundaries(feature_layers, zoom, new_layer['name'] = new_layer_name return new_layer + + +def generate_label_layer(feature_layers, zoom, source_layer=None, + new_layer_name=None): + assert source_layer, 'generate_label_layer: missing source_layer' + assert new_layer_name, 'generate_label_layer: missing new_layer_name' + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + label_features = [] + for feature in layer['features']: + shape, properties, fid = feature + # shapely does the right thing for all kinds of geometries + # it also has a function `representative_point` which we might + # want to consider using too + label_centroid = shape.centroid + label_properties = properties.copy() + label_feature = label_centroid, label_properties, fid + label_features.append(label_feature) + + label_layer_datum = layer['layer_datum'].copy() + label_layer_datum['name'] = new_layer_name + label_feature_layer = dict( + name=new_layer_name, + features=label_features, + layer_datum=label_layer_datum, + ) + return label_feature_layer From 9443f3e48502aba6494314d19707e36d38b2e66a Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 25 Sep 2015 13:23:05 -0400 Subject: [PATCH 167/344] Add kind to boundaries based on admin_level --- TileStache/Goodies/VecTiles/transform.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 3dd3764d..c9d0ca5b 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -346,6 +346,30 @@ def water_tunnel(shape, properties, fid, zoom): return shape, properties, fid +boundary_admin_level_mapping = { + 2: 'country', + 4: 'state', + 6: 'county', +} + + +def boundary_kind(shape, properties, fid, zoom): + kind = properties.get('kind') + if kind: + return shape, properties, fid + admin_level_str = properties.get('admin_level') + if admin_level_str is None: + return shape, properties, fid + try: + admin_level_int = int(admin_level_str) + except ValueError: + return shape, properties, fid + kind = boundary_admin_level_mapping.get(admin_level_int) + if kind: + properties['kind'] = kind + return shape, properties, fid + + def tags_create_dict(shape, properties, fid, zoom): tags_hstore = properties.get('tags') if tags_hstore: From b6948cd9a20e27d984fdc01cc3c1dc11345b83dc Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 28 Sep 2015 14:27:15 -0400 Subject: [PATCH 168/344] Add `municipality` to admin_level mapping --- TileStache/Goodies/VecTiles/transform.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index c9d0ca5b..00eaae37 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -350,6 +350,7 @@ def water_tunnel(shape, properties, fid, zoom): 2: 'country', 4: 'state', 6: 'county', + 8: 'municipality', } From 82a3db56d9fb1d1aa8205922a570d9da1994f91a Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 29 Sep 2015 16:42:56 +0100 Subject: [PATCH 169/344] Re-implemented admin boundary logic in Python as a post-processing transform. --- TileStache/Goodies/VecTiles/transform.py | 271 +++++++++++++++++++++++ 1 file changed, 271 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a5fe196b..027f6f70 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -5,6 +5,9 @@ from collections import defaultdict from shapely.strtree import STRtree from shapely.geometry.base import BaseMultipartGeometry +from shapely.geometry.polygon import orient +from shapely.ops import linemerge +from shapely.geometry.multilinestring import MultiLineString import re @@ -1153,3 +1156,271 @@ def exterior_boundaries(feature_layers, zoom, new_layer['name'] = new_layer_name return new_layer + + +def _inject_key(key, infix): + """ + OSM keys often have several parts, separated by ':'s. + When we merge properties from the left and right of a + boundary, we want to preserve information like the + left and right names, but prefer the form "name:left" + rather than "left:name", so we have to insert an + infix string to these ':'-delimited arrays. + + >>> _inject_key('a:b:c', 'x') + 'a:x:b:c' + >>> _inject_key('a', 'x') + 'a:x' + + """ + parts = key.split(':') + parts.insert(1, infix) + return ':'.join(parts) + + +def _merge_left_right_props(lprops, rprops): + """ + Given a set of properties to the left and right of a + boundary, we want to keep as many of these as possible, + but keeping them all might be a bit too much. + + So we want to keep the key-value pairs which are the + same in both in the output, but merge the ones which + are different by infixing them with 'left' and 'right'. + + >>> _merge_left_right_props({}, {}) + {} + >>> _merge_left_right_props({'a':1}, {}) + {'a:left': 1} + >>> _merge_left_right_props({}, {'b':2}) + {'b:right': 2} + >>> _merge_left_right_props({'a':1, 'c':3}, {'b':2, 'c':3}) + {'a:left': 1, 'c': 3, 'b:right': 2} + >>> _merge_left_right_props({'a':1},{'a':2}) + {'a:left': 1, 'a:right': 2} + """ + keys = set(lprops.keys()) | set(rprops.keys()) + new_props = dict() + + # props in both are copied directly if they're the same + # in both the left and right. they get left/right + # inserted after the first ':' if they're different. + for k in keys: + lv = lprops.get(k) + rv = rprops.get(k) + + if lv == rv: + new_props[k] = lv + else: + if lv is not None: + new_props[_inject_key(k, 'left')] = lv + if rv is not None: + new_props[_inject_key(k, 'right')] = rv + + return new_props + + +def _make_joined_name(props): + """ + Updates the argument to contain a 'name' element + generated from joining the left and right names. + + Just to make it easier for people, we generate a name + which is easy to display of the form "LEFT - RIGHT". + The individual properties are available if the user + wants to generate a more complex name. + + >>> x = {} + >>> _make_joined_name(x) + >>> x + {} + + >>> x = {'name:left':'Left'} + >>> _make_joined_name(x) + >>> x + {'name': 'Left', 'name:left': 'Left'} + + >>> x = {'name:right':'Right'} + >>> _make_joined_name(x) + >>> x + {'name': 'Right', 'name:right': 'Right'} + + >>> x = {'name:left':'Left', 'name:right':'Right'} + >>> _make_joined_name(x) + >>> x + {'name:right': 'Right', 'name': 'Left - Right', 'name:left': 'Left'} + + >>> x = {'name:left':'Left', 'name:right':'Right', 'name': 'Already Exists'} + >>> _make_joined_name(x) + >>> x + {'name:right': 'Right', 'name': 'Already Exists', 'name:left': 'Left'} + """ + + # don't overwrite an existing name + if 'name' in props: + return + + lname = props.get('name:left') + rname = props.get('name:right') + + if lname is not None: + if rname is not None: + props['name'] = "%s - %s" % (lname, rname) + else: + props['name'] = lname + elif rname is not None: + props['name'] = rname + + +def _linemerge(geom): + """ + Try to extract all the linear features from the geometry argument + and merge them all together into the smallest set of linestrings + possible. + + This is almost identical to Shapely's linemerge, and uses it, + except that Shapely's throws exceptions when passed a single + linestring, or a geometry collection with lines and points in it. + So this can be thought of as a "safer" wrapper around Shapely's + function. + """ + geom_type = geom.type + + if geom_type == 'GeometryCollection': + # collect together everything line-like from the + # geometry collection + lines = map(_linemerge, geom.geoms) + + # filter out anything that's empty + lines = filter(lambda g: not g.is_empty, lines) + + if len(lines) == 0: + return MultiLineString([]) + else: + return linemerge(lines) + + elif geom_type == 'LineString': + return geom + + elif geom_type == 'MultiLineString': + return linemerge(geom) + + else: + return MultiLineString([]) + + +def admin_boundaries(feature_layers, zoom, base_layer, + start_zoom=0): + """ + Given a layer with admin polygons and maritime boundaries, + attempts to output a set of oriented boundaries with properties + from both the left and right polygon, and also cut with the + maritime information to provide a `maritime_boundary=yes` value + where there's overlap between the maritime lines and the + polygon boundaries. + """ + + layer = None + + # don't start processing until the start zoom + if zoom < start_zoom: + return layer + + layer = _find_layer(feature_layers, base_layer) + if layer is None: + return None + + # layer will have polygonal features for the admin + # polygons and also linear features for the maritime + # boundaries. further, we want to group the admin + # polygons by their kind, as this will reduce the + # working set. + admin_features = defaultdict(list) + maritime_shapes = list() + new_features = list() + + for shape, props, fid in layer['features']: + dims = _geom_dimensions(shape) + kind = props.get('kind') + maritime_boundary = props.get('maritime_boundary') + + # note: dims=4 means polygon, as it's a bit set of + # 1 << num_dimensions. likewise, dims=2 means a + # linestring. the reason to use this rather than + # compare the string of types is to catch the + # "multi-" types as well. + if dims == 4 and kind is not None: + admin_features[kind].append((shape, props, fid)) + + elif dims == 2 and maritime_boundary == 'yes': + maritime_shapes.append(shape) + + # there are separate polygons for each admin level, and + # we only want to intersect like with like because it + # makes more sense to have Country-Country and + # State-State boundaries (and labels) rather than the + # (combinatoric) set of all different levels. + for kind, features in admin_features.iteritems(): + num_features = len(features) + envelopes = map(lambda g: g[0].envelope, features) + + for i, feature in enumerate(features): + shape, props, fid = feature + envelope = envelopes[i] + + # orient to ensure that the shape is to the + # left of the boundary. + boundary = orient(shape).boundary + + # intersect with *preceding* features to remove + # those boundary parts. this ensures that there + # are no duplicate parts. + for j in range(0, i): + cut_shape, cut_props, cut_fid = features[j] + cut_envelope = envelopes[j] + if envelope.intersects(cut_envelope): + boundary = _intersect_cut(boundary, cut_shape)[1] + + if boundary.is_empty: + break + + # intersect with every *later* feature. now each + # intersection represents a section of boundary + # that we want to keep. + for j in range(i+1, num_features): + cut_shape, cut_props, cut_fid = features[j] + cut_envelope = envelopes[j] + + if envelope.intersects(cut_envelope): + inside, boundary = _intersect_cut(boundary, cut_shape) + + inside = _linemerge(inside) + if not inside.is_empty: + new_props = _merge_left_right_props(props, cut_props) + new_props['id'] = props['id'] + _make_joined_name(new_props) + new_features.append((inside, new_props, fid)) + + if boundary.is_empty: + break + + # anything left over at the end is still a boundary, + # but a one-sided boundary to international waters. + boundary = _linemerge(boundary) + if not boundary.is_empty: + new_props = props.copy() + _make_joined_name(new_props) + new_features.append((boundary, new_props, fid)) + + + # use intracut for maritime + maritime_shape = _linemerge(MultiLineString(maritime_shapes)) + maritime_boundaries = [(maritime_shape, {'maritime_boundary': 'yes'}, 0)] + cutter = _Cutter(maritime_boundaries, None, + 'maritime_boundary', 'maritime_boundary', + 2, _intersect_cut) + for shape, props, fid in new_features: + cutter.cut(shape, props, fid) + + layer['features'] = cutter.new_features + return layer From 584cff7eebb8c1fc7b47100504060b3cd1d1c2b5 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 29 Sep 2015 18:49:35 -0400 Subject: [PATCH 170/344] Generate new label features in existing layer --- TileStache/Goodies/VecTiles/transform.py | 30 ++++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 6c1b01c2..768af702 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1131,31 +1131,31 @@ def exterior_boundaries(feature_layers, zoom, return new_layer -def generate_label_layer(feature_layers, zoom, source_layer=None, - new_layer_name=None): - assert source_layer, 'generate_label_layer: missing source_layer' - assert new_layer_name, 'generate_label_layer: missing new_layer_name' +def generate_label_features(feature_layers, zoom, source_layer=None, + label_property_name=None, + label_property_value=None): + assert source_layer, 'generate_label_features: missing source_layer' layer = _find_layer(feature_layers, source_layer) if layer is None: return None - label_features = [] + new_features = [] for feature in layer['features']: shape, properties, fid = feature + # shapely does the right thing for all kinds of geometries # it also has a function `representative_point` which we might # want to consider using too label_centroid = shape.centroid label_properties = properties.copy() + if label_property_name: + label_properties[label_property_name] = label_property_value label_feature = label_centroid, label_properties, fid - label_features.append(label_feature) - - label_layer_datum = layer['layer_datum'].copy() - label_layer_datum['name'] = new_layer_name - label_feature_layer = dict( - name=new_layer_name, - features=label_features, - layer_datum=label_layer_datum, - ) - return label_feature_layer + + # add the original feature and then the label + new_features.append(feature) + new_features.append(label_feature) + + layer['features'] = new_features + return layer From 16f6c9d6994203f8c2215697b1d8c694313c273d Mon Sep 17 00:00:00 2001 From: Nathaniel Kelso Date: Tue, 29 Sep 2015 22:19:11 -0700 Subject: [PATCH 171/344] use kind property when provided --- TileStache/Goodies/VecTiles/transform.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index caaa116c..29092174 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -155,7 +155,9 @@ def remove_feature_id(shape, properties, fid, zoom): def building_kind(shape, properties, fid, zoom): building = _coalesce(properties, 'building:part', 'building') - if building and building != 'yes': + if properties.kind: + kind = properties.kind + elif building and building != 'yes': kind = building else: kind = _coalesce(properties, 'amenity', 'shop', 'tourism') From 1d869cf7b94c18e1a8dde59b3d59c12d958a4645 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 30 Sep 2015 16:45:56 +0100 Subject: [PATCH 172/344] Now using buffered land polygons to clip the maritime segments of the boundaries, so need slightly different logic here. --- TileStache/Goodies/VecTiles/transform.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 027f6f70..883654db 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1336,7 +1336,7 @@ def admin_boundaries(feature_layers, zoom, base_layer, # polygons by their kind, as this will reduce the # working set. admin_features = defaultdict(list) - maritime_shapes = list() + maritime_features = list() new_features = list() for shape, props, fid in layer['features']: @@ -1352,8 +1352,8 @@ def admin_boundaries(feature_layers, zoom, base_layer, if dims == 4 and kind is not None: admin_features[kind].append((shape, props, fid)) - elif dims == 2 and maritime_boundary == 'yes': - maritime_shapes.append(shape) + elif dims == 4 and maritime_boundary == 'yes': + maritime_features.append((shape, {'maritime_boundary':'no'}, 0)) # there are separate polygons for each admin level, and # we only want to intersect like with like because it @@ -1413,14 +1413,23 @@ def admin_boundaries(feature_layers, zoom, base_layer, new_features.append((boundary, new_props, fid)) - # use intracut for maritime - maritime_shape = _linemerge(MultiLineString(maritime_shapes)) - maritime_boundaries = [(maritime_shape, {'maritime_boundary': 'yes'}, 0)] - cutter = _Cutter(maritime_boundaries, None, + # use intracut for maritime, but it intersects in a positive + # way - it sets the tag on anything which intersects, whereas + # we want to set maritime where it _doesn't_ intersect. so + # we have to flip the attribute afterwards. + cutter = _Cutter(maritime_features, None, 'maritime_boundary', 'maritime_boundary', 2, _intersect_cut) + for shape, props, fid in new_features: cutter.cut(shape, props, fid) + # flip the property, so define maritime_boundary=yes where + # it was previously unset and remove maritime_boundary=no. + for shape, props, fid in cutter.new_features: + maritime_boundary = props.pop('maritime_boundary', None) + if maritime_boundary is None: + props['maritime_boundary'] = 'yes' + layer['features'] = cutter.new_features return layer From dd32ff67a96fa7a26627f43d4cd7bbdb3b7c6429 Mon Sep 17 00:00:00 2001 From: Nathaniel Kelso Date: Wed, 30 Sep 2015 11:43:56 -0700 Subject: [PATCH 173/344] look at kind first, then use existing logic (python ftw) --- TileStache/Goodies/VecTiles/transform.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 29092174..526a25be 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -154,10 +154,11 @@ def remove_feature_id(shape, properties, fid, zoom): def building_kind(shape, properties, fid, zoom): + kind = properties.get('kind') + if kind: + return shape, properties, fid building = _coalesce(properties, 'building:part', 'building') - if properties.kind: - kind = properties.kind - elif building and building != 'yes': + if building and building != 'yes': kind = building else: kind = _coalesce(properties, 'amenity', 'shop', 'tourism') From cd5643e212134d44f70d1d567c4e5b872fc7c51f Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 30 Sep 2015 19:46:31 +0100 Subject: [PATCH 174/344] Made code more imperative. --- TileStache/Goodies/VecTiles/transform.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4cba237a..cb4641f4 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1288,17 +1288,15 @@ def _linemerge(geom): geom_type = geom.type if geom_type == 'GeometryCollection': - # collect together everything line-like from the - # geometry collection - lines = map(_linemerge, geom.geoms) + # collect together everything line-like from the geometry + # collection and filter out anything that's empty + lines = [] + for line in g.geoms: + line = _linemerge(line) + if not line.is_empty: + lines.append(line) - # filter out anything that's empty - lines = filter(lambda g: not g.is_empty, lines) - - if len(lines) == 0: - return MultiLineString([]) - else: - return linemerge(lines) + return linemerge(lines) if lines else MultiLineString([]) elif geom_type == 'LineString': return geom @@ -1363,7 +1361,7 @@ def admin_boundaries(feature_layers, zoom, base_layer, # (combinatoric) set of all different levels. for kind, features in admin_features.iteritems(): num_features = len(features) - envelopes = map(lambda g: g[0].envelope, features) + envelopes = [g[0].envelope for g in features] for i, feature in enumerate(features): shape, props, fid = feature From ba6fa14bcef6b8899aad7571c31cc07f3d290c9f Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 30 Sep 2015 19:53:16 +0100 Subject: [PATCH 175/344] Don't bother calculating the inside part of the intersection, it's thrown away anyway. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index cb4641f4..7bbcf8e4 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1378,7 +1378,7 @@ def admin_boundaries(feature_layers, zoom, base_layer, cut_shape, cut_props, cut_fid = features[j] cut_envelope = envelopes[j] if envelope.intersects(cut_envelope): - boundary = _intersect_cut(boundary, cut_shape)[1] + boundary = boundary.difference(cut_shape) if boundary.is_empty: break From 1bae868a917ec39e0ce625000e6461ac13702504 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 30 Sep 2015 19:57:06 +0100 Subject: [PATCH 176/344] Squishing space invaders. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 7bbcf8e4..cb992706 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1296,7 +1296,7 @@ def _linemerge(geom): if not line.is_empty: lines.append(line) - return linemerge(lines) if lines else MultiLineString([]) + return linemerge(lines) if lines else MultiLineString([]) elif geom_type == 'LineString': return geom From 3c79d22dccd0e915ec551d54ff786588362fe17b Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 30 Sep 2015 20:03:27 +0100 Subject: [PATCH 177/344] Give magic constants names. --- TileStache/Goodies/VecTiles/transform.py | 26 +++++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index cb992706..e031be21 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -486,15 +486,21 @@ def _sorted_attributes(features, attrs, attribute): # represented by 1, one by 2, etc... this is to support # things like geometry collections where the type isn't # statically known. +_NULL_DIMENSION = 0 +_POINT_DIMENSION = 1 +_LINE_DIMENSION = 2 +_POLYGON_DIMENSION = 4 + + _GEOMETRY_DIMENSIONS = { - 'Point': 1, - 'LineString': 2, - 'LinearRing': 2, - 'Polygon': 4, - 'MultiPoint': 1, - 'MultiLineString': 2, - 'MultiPolygon': 4, - 'GeometryCollection': 0, + 'Point': _POINT_DIMENSION, + 'LineString': _LINE_DIMENSION, + 'LinearRing': _LINE_DIMENSION, + 'Polygon': _POLYGON_DIMENSION, + 'MultiPoint': _POINT_DIMENSION, + 'MultiLineString': _LINE_DIMENSION, + 'MultiPolygon': _POLYGON_DIMENSION, + 'GeometryCollection': _NULL_DIMENSION, } @@ -518,7 +524,7 @@ def _geom_dimensions(g): # recurse for geometry collections to find the true # dimensionality of the geometry. - if dim == 0: + if dim == _NULL_DIMENSION: for part in g.geoms: dim = dim | _geom_dimensions(part) @@ -1418,7 +1424,7 @@ def admin_boundaries(feature_layers, zoom, base_layer, # we have to flip the attribute afterwards. cutter = _Cutter(maritime_features, None, 'maritime_boundary', 'maritime_boundary', - 2, _intersect_cut) + _LINE_DIMENSION, _intersect_cut) for shape, props, fid in new_features: cutter.cut(shape, props, fid) From e8fbe8500350a5de812d9ad33b1d37e8ccff4d9d Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 30 Sep 2015 20:11:07 +0100 Subject: [PATCH 178/344] Use new magic constant names. --- TileStache/Goodies/VecTiles/transform.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index e031be21..6c50a7ce 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1349,15 +1349,13 @@ def admin_boundaries(feature_layers, zoom, base_layer, kind = props.get('kind') maritime_boundary = props.get('maritime_boundary') - # note: dims=4 means polygon, as it's a bit set of - # 1 << num_dimensions. likewise, dims=2 means a - # linestring. the reason to use this rather than - # compare the string of types is to catch the - # "multi-" types as well. - if dims == 4 and kind is not None: + # the reason to use this rather than compare the + # string of types is to catch the "multi-" types + # as well. + if dims == _POLYGON_DIMENSION and kind is not None: admin_features[kind].append((shape, props, fid)) - elif dims == 4 and maritime_boundary == 'yes': + elif dims == _POLYGON_DIMENSION and maritime_boundary == 'yes': maritime_features.append((shape, {'maritime_boundary':'no'}, 0)) # there are separate polygons for each admin level, and From 33c63f51ff1178b1ff76f8bad2cc92f61e1fba3d Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 30 Sep 2015 15:31:14 -0400 Subject: [PATCH 179/344] Support creating a new labels layer too --- TileStache/Goodies/VecTiles/transform.py | 27 ++++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 1e10f685..f449cc53 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1441,9 +1441,10 @@ def admin_boundaries(feature_layers, zoom, base_layer, return layer -def generate_label_features(feature_layers, zoom, source_layer=None, - label_property_name=None, - label_property_value=None): +def generate_label_features( + feature_layers, zoom, source_layer=None, label_property_name=None, + label_property_value=None, new_layer_name=None): + assert source_layer, 'generate_label_features: missing source_layer' layer = _find_layer(feature_layers, source_layer) @@ -1463,9 +1464,21 @@ def generate_label_features(feature_layers, zoom, source_layer=None, label_properties[label_property_name] = label_property_value label_feature = label_centroid, label_properties, fid - # add the original feature and then the label - new_features.append(feature) + # if we're adding these features to a new layer, don't add the + # original features + if new_layer_name is None: + new_features.append(feature) new_features.append(label_feature) - layer['features'] = new_features - return layer + if new_layer_name is None: + layer['features'] = new_features + return layer + else: + label_layer_datum = layer['layer_datum'].copy() + label_layer_datum['name'] = new_layer_name + label_feature_layer = dict( + name=new_layer_name, + features=new_features, + layer_datum=label_layer_datum, + ) + return label_feature_layer From 594675d4d3ff1b7456a023752c63014c9dc0ff53 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 30 Sep 2015 15:41:49 -0400 Subject: [PATCH 180/344] Only create labels for polygonal geometries --- TileStache/Goodies/VecTiles/transform.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index f449cc53..3237ea2c 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1455,9 +1455,13 @@ def generate_label_features( for feature in layer['features']: shape, properties, fid = feature - # shapely does the right thing for all kinds of geometries - # it also has a function `representative_point` which we might - # want to consider using too + # We only want to create label features for polygonal + # geometries + if shape.geom_type not in ('Polygon', 'MultiPolygon'): + continue + + # shapely also has a function `representative_point` which we + # might want to consider using here label_centroid = shape.centroid label_properties = properties.copy() if label_property_name: From 3e6492299b4e12fe68358522d730968a04d08910 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 30 Sep 2015 17:36:58 -0400 Subject: [PATCH 181/344] Correct variable name --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 3237ea2c..d10554bb 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1300,7 +1300,7 @@ def _linemerge(geom): # collect together everything line-like from the geometry # collection and filter out anything that's empty lines = [] - for line in g.geoms: + for line in geom.geoms: line = _linemerge(line) if not line.is_empty: lines.append(line) From 8222e9bfeffa60697da55bbfd3d57a7e373fe211 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 1 Oct 2015 11:00:52 -0400 Subject: [PATCH 182/344] Remove unused import --- TileStache/Goodies/VecTiles/transform.py | 1 - 1 file changed, 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index d10554bb..9cc86507 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -4,7 +4,6 @@ from StreetNames import short_street_name from collections import defaultdict from shapely.strtree import STRtree -from shapely.geometry.base import BaseMultipartGeometry from shapely.geometry.polygon import orient from shapely.ops import linemerge from shapely.geometry.multilinestring import MultiLineString From 436a75ab01707b4631df2c5c155d5303c3085755 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 1 Oct 2015 10:55:53 -0400 Subject: [PATCH 183/344] Use None as default kwargs value to be safer --- TileStache/Goodies/VecTiles/transform.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 9cc86507..72132642 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1054,7 +1054,7 @@ def _make_new_properties(props, props_instructions): def exterior_boundaries(feature_layers, zoom, base_layer, new_layer_name=None, - prop_transform=dict(), + prop_transform=None, buffer_size=None, start_zoom=0): """ @@ -1105,6 +1105,9 @@ def exterior_boundaries(feature_layers, zoom, if layer is None: return None + if prop_transform is None: + prop_transform = {} + features = layer['features'] # create an index so that we can efficiently find the From 7d7da87c873a811cc8ed30659c15945698897b65 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 1 Oct 2015 22:40:33 +0100 Subject: [PATCH 184/344] Added ferry routes with kind ferry. --- TileStache/Goodies/VecTiles/transform.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 72132642..918eaf1a 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -133,6 +133,9 @@ def _road_kind(properties): railway = properties.get('railway') if railway in road_kind_rail: return 'rail' + route = properties.get('route') + if route == 'ferry': + return 'ferry' return 'minor_road' From 1477ef788751aaa1caff8d4b8ee0f34911a3ff04 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 2 Oct 2015 15:30:00 +0100 Subject: [PATCH 185/344] Added an explicit filter for parsing the layer tag as a floating point number, used in both the roads and buildings layers. --- TileStache/Goodies/VecTiles/transform.py | 41 +++++++++++++++++------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 918eaf1a..b4af233e 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -273,18 +273,16 @@ def road_sort_key(shape, properties, fid, zoom): # Explicit layer is clipped to [-5, 5] range layer = properties.get('layer') - if layer: - layer_float = to_float(layer) - if layer_float is not None: - layer_float = max(min(layer_float, 5), -5) - # The range of values from above is [5, 34] - # For positive layer values, we want the range to be: - # [34, 39] - if layer_float > 0: - sort_val = int(layer_float + 34) - # For negative layer values, [0, 5] - elif layer_float < 0: - sort_val = int(layer_float + 5) + if layer is not None: + layer = max(min(layer, 5), -5) + # The range of values from above is [5, 34] + # For positive layer values, we want the range to be: + # [34, 39] + if layer > 0: + sort_val = int(layer + 34) + # For negative layer values, [0, 5] + elif layer < 0: + sort_val = int(layer + 5) properties['sort_key'] = sort_val @@ -1491,3 +1489,22 @@ def generate_label_features( layer_datum=label_layer_datum, ) return label_feature_layer + + +def parse_layer_as_float(shape, properties, fid, zoom): + """ + If the 'layer' property is present on a feature, then + this attempts to parse it as a floating point number. + The old value is removed and, if it could be parsed + as a floating point number, the number replaces the + original property. + """ + + layer = properties.pop('layer', None) + + if layer: + layer_float = to_float(layer) + if layer_float is not None: + properties['layer'] = layer_float + + return shape, properties, fid From fa5dccb410e8a9f794679f4da06abf1c7b169910 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 2 Oct 2015 15:51:46 +0100 Subject: [PATCH 186/344] Better documentation of the dependency between `road_sort_key` and `parse_layer_as_float`. --- TileStache/Goodies/VecTiles/transform.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index b4af233e..1dcbe713 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -232,6 +232,8 @@ def road_classifier(shape, properties, fid, zoom): def road_sort_key(shape, properties, fid, zoom): + # Note! parse_layer_as_float must be run before this filter. + # Calculated sort value is in the range 0 to 39 sort_val = 0 @@ -271,7 +273,9 @@ def road_sort_key(shape, properties, fid, zoom): (railway == 'subway' and tunnel not in ('no', 'false'))): sort_val -= 10 - # Explicit layer is clipped to [-5, 5] range + # Explicit layer is clipped to [-5, 5] range. Note that + # the layer, if present, will be a Float due to the + # parse_layer_as_float filter. layer = properties.get('layer') if layer is not None: layer = max(min(layer, 5), -5) From 8dee3f8c939ed184dc18f363e9025fba060a5bea Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 2 Oct 2015 16:47:36 +0100 Subject: [PATCH 187/344] Set kind to `aboriginal_lands` for first nations state boundaries. --- TileStache/Goodies/VecTiles/transform.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 1dcbe713..7fd2b9d7 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -368,6 +368,15 @@ def boundary_kind(shape, properties, fid, zoom): kind = properties.get('kind') if kind: return shape, properties, fid + + # if the boundary is tagged as being that of a first nations + # state then skip the rest of the kind logic, regardless of + # the admin_level (see mapzen/vector-datasource#284). + boundary_type = properties.get('boundary_type') + if boundary_type == 'aboriginal_lands': + properties['kind'] = 'aboriginal_lands' + return shape, properties, fid + admin_level_str = properties.get('admin_level') if admin_level_str is None: return shape, properties, fid @@ -381,6 +390,13 @@ def boundary_kind(shape, properties, fid, zoom): return shape, properties, fid +def boundary_trim_properties(shape, properties, fid, zoom): + properties = _remove_properties( + properties, + 'boundary_type') + return shape, properties, fid + + def tags_create_dict(shape, properties, fid, zoom): tags_hstore = properties.get('tags') if tags_hstore: From 9716e262a555f564981b274a34477e0987525026 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 2 Oct 2015 21:53:29 -0400 Subject: [PATCH 188/344] Only include labels with name or sport tags --- TileStache/Goodies/VecTiles/transform.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 7fd2b9d7..2bd8900c 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1483,6 +1483,14 @@ def generate_label_features( if shape.geom_type not in ('Polygon', 'MultiPolygon'): continue + # Additionally, the feature needs to have a name or a sport + # tag, oherwise it's not really useful for labelling purposes + name = properties.get('name') + if not name: + sport = properties.get('sport') + if not sport: + continue + # shapely also has a function `representative_point` which we # might want to consider using here label_centroid = shape.centroid From 51025350a084d55d62e09698527c527d68ddaf7b Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 2 Oct 2015 21:59:28 -0400 Subject: [PATCH 189/344] Use representative_point instead of centroid --- TileStache/Goodies/VecTiles/transform.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 2bd8900c..384329a9 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1491,13 +1491,13 @@ def generate_label_features( if not sport: continue - # shapely also has a function `representative_point` which we - # might want to consider using here - label_centroid = shape.centroid + label_point = shape.representative_point() + label_properties = properties.copy() if label_property_name: label_properties[label_property_name] = label_property_value - label_feature = label_centroid, label_properties, fid + + label_feature = label_point, label_properties, fid # if we're adding these features to a new layer, don't add the # original features From d593e69c67b80a74be86067f078c08f1aae1de60 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 5 Oct 2015 13:27:42 -0400 Subject: [PATCH 190/344] Ensure original features always get added Previously, if a feature wasn't a candidate for adding a label, it did not get added to the layer. This corrects that. --- TileStache/Goodies/VecTiles/transform.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 384329a9..b3738d19 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1478,6 +1478,10 @@ def generate_label_features( for feature in layer['features']: shape, properties, fid = feature + # only add the original features if updating an existing layer + if new_layer_name is None: + new_features.append(feature) + # We only want to create label features for polygonal # geometries if shape.geom_type not in ('Polygon', 'MultiPolygon'): @@ -1499,10 +1503,6 @@ def generate_label_features( label_feature = label_point, label_properties, fid - # if we're adding these features to a new layer, don't add the - # original features - if new_layer_name is None: - new_features.append(feature) new_features.append(label_feature) if new_layer_name is None: From 992586a158e8a69e9e90ff4630555cde7e2f557d Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 6 Oct 2015 09:40:51 -0700 Subject: [PATCH 191/344] Added an implementation of orient which can handle multi-polygons and other geometry types without throwing an exception. --- TileStache/Goodies/VecTiles/transform.py | 45 +++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index b3738d19..1946117f 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -7,6 +7,8 @@ from shapely.geometry.polygon import orient from shapely.ops import linemerge from shapely.geometry.multilinestring import MultiLineString +from shapely.geometry.multipolygon import MultiPolygon +from shapely.geometry.collection import GeometryCollection import re @@ -1340,6 +1342,47 @@ def _linemerge(geom): return MultiLineString([]) +def _orient(geom): + """ + Given a shape, returns the counter-clockwise oriented + version. Does not affect points or lines. + + This version is required because Shapely's version is + only defined for single polygons, and we want + something that works generically. + + In the example below, note the change in order of the + coordinates in `p2`, which is initially not oriented + CCW. + + >>> p1 = Polygon([[0, 0], [1, 0], [0, 1], [0, 0]]) + >>> p2 = Polygon([[0, 1], [1, 1], [1, 0], [0, 1]]) + >>> orient(p1).wkt + 'POLYGON ((0 0, 1 0, 0 1, 0 0))' + >>> orient(p2).wkt + 'POLYGON ((0 1, 1 0, 1 1, 0 1))' + >>> _orient(MultiPolygon([p1, p2])).wkt + 'MULTIPOLYGON (((0 0, 1 0, 0 1, 0 0)), ((0 1, 1 0, 1 1, 0 1)))' + """ + + def oriented_multi(kind, geom): + oriented_geoms = [_orient(g) for g in geom.geoms] + return kind(oriented_geoms) + + geom_type = geom.type + + if geom_type == 'Polygon': + geom = orient(geom) + + elif geom_type == 'MultiPolygon': + geom = oriented_multi(MultiPolygon, geom) + + elif geom_type == 'GeometryCollection': + geom = oriented_multi(GeometryCollection, geom) + + return geom + + def admin_boundaries(feature_layers, zoom, base_layer, start_zoom=0): """ @@ -1399,7 +1442,7 @@ def admin_boundaries(feature_layers, zoom, base_layer, # orient to ensure that the shape is to the # left of the boundary. - boundary = orient(shape).boundary + boundary = _orient(shape).boundary # intersect with *preceding* features to remove # those boundary parts. this ensures that there From a4285678f10d62a96e7c5467247980d45e6e4e8d Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 7 Oct 2015 01:12:58 -0400 Subject: [PATCH 192/344] Version bump -> 0.4.0 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91e96f4e..a1c402d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +v0.4.0 +------ +* Add transform to dynamically create polygon `label_placement` features using representative point for existing layers (or exporte in new layer) as long as the input feature has a `name` or `sport` tag. +* Ported PostGIS SQL logic from Vector Datasource to Python, with spatial intersection logic for `maritime` boundaries. +* Use the input data's `kind` property when provided when determining building feature's kind. +* Add `ferry` lines to high roads logic. + v0.3.0 ------ * Add intracut (intersection) algorithm for cutting within layers. From 9f7d97236071dbf5d3690b63ccf9868652c61be6 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 13 Oct 2015 13:51:28 +0100 Subject: [PATCH 193/344] Make admin boundaries post-processing filter work with boundary linestring fragments rather than needing an oriented polygon. --- TileStache/Goodies/VecTiles/transform.py | 25 ++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 1946117f..5311da53 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1386,12 +1386,17 @@ def oriented_multi(kind, geom): def admin_boundaries(feature_layers, zoom, base_layer, start_zoom=0): """ - Given a layer with admin polygons and maritime boundaries, - attempts to output a set of oriented boundaries with properties - from both the left and right polygon, and also cut with the - maritime information to provide a `maritime_boundary=yes` value - where there's overlap between the maritime lines and the - polygon boundaries. + Given a layer with admin boundaries and inclusion polygons for + land-based boundaries, attempts to output a set of oriented + boundaries with properties from both the left and right admin + boundary, and also cut with the maritime information to provide + a `maritime_boundary=yes` value where there's overlap between + the maritime lines and the admin boundaries. + + Note that admin boundaries must alread be correctly oriented. + In other words, it must have a positive area and run counter- + clockwise around the polygon for which it is an outer (or + clockwise if it was an inner). """ layer = None @@ -1421,7 +1426,7 @@ def admin_boundaries(feature_layers, zoom, base_layer, # the reason to use this rather than compare the # string of types is to catch the "multi-" types # as well. - if dims == _POLYGON_DIMENSION and kind is not None: + if dims == _LINE_DIMENSION and kind is not None: admin_features[kind].append((shape, props, fid)) elif dims == _POLYGON_DIMENSION and maritime_boundary == 'yes': @@ -1437,13 +1442,9 @@ def admin_boundaries(feature_layers, zoom, base_layer, envelopes = [g[0].envelope for g in features] for i, feature in enumerate(features): - shape, props, fid = feature + boundary, props, fid = feature envelope = envelopes[i] - # orient to ensure that the shape is to the - # left of the boundary. - boundary = _orient(shape).boundary - # intersect with *preceding* features to remove # those boundary parts. this ensures that there # are no duplicate parts. From 702cca32f142ca6ab9348fbb4cebe8ac10dbc22c Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Sat, 10 Oct 2015 18:32:48 -0700 Subject: [PATCH 194/344] Added a post-process function to generate address points from building outlines. Updated building logic to remove 'POI-like' kinds - these will go in the POIs layer instead. --- TileStache/Goodies/VecTiles/transform.py | 63 ++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 5311da53..6d352d0b 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -165,10 +165,11 @@ def building_kind(shape, properties, fid, zoom): if kind: return shape, properties, fid building = _coalesce(properties, 'building:part', 'building') - if building and building != 'yes': - kind = building - else: - kind = _coalesce(properties, 'amenity', 'shop', 'tourism') + if building: + if building != 'yes': + kind = building + else: + kind = 'building' if kind: properties['kind'] = kind return shape, properties, fid @@ -199,7 +200,6 @@ def building_min_height(shape, properties, fid, zoom): def building_trim_properties(shape, properties, fid, zoom): properties = _remove_properties( properties, - 'amenity', 'shop', 'tourism', 'building', 'building:part', 'building:levels', 'building:min_levels') return shape, properties, fid @@ -1563,6 +1563,59 @@ def generate_label_features( return label_feature_layer +def generate_address_points( + feature_layers, zoom, source_layer=None): + """ + Generates address points from building polygons where there is an + addr:housenumber tag on the building. Removes those tags from the + building. + """ + + assert source_layer, 'generate_label_features: missing source_layer' + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + new_features = [] + for feature in layer['features']: + shape, properties, fid = feature + + # remove address tags on the original polygon, + # we only want them on the address point. + addr_housenumber = properties.pop('addr_housenumber', None) + + # We only want to create address points for polygonal + # buildings with address tags. + if shape.geom_type not in ('Polygon', 'MultiPolygon') or \ + addr_housenumber is None: + # keep the feature as-is, no modifications. + continue + + label_point = shape.representative_point() + + # we're only interested in a very few properties for + # address points. + label_properties = dict( + addr_housenumber=addr_housenumber, + kind='address') + + source = properties.get('source') + if source is not None: + label_properties['source'] = source + + addr_street = properties.pop('addr_street', None) + if addr_street is not None: + label_properties['addr_street'] = addr_street + + label_feature = label_point, label_properties, fid + + new_features.append(label_feature) + + layer['features'].extend(new_features) + return layer + + def parse_layer_as_float(shape, properties, fid, zoom): """ If the 'layer' property is present on a feature, then From 77b92cc500e5dc04f4477bf50ad5b71babefe5ed Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 13 Oct 2015 17:30:26 +0100 Subject: [PATCH 195/344] Add zoom level filter to address point generator. --- TileStache/Goodies/VecTiles/transform.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 6d352d0b..b3c59661 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1564,7 +1564,7 @@ def generate_label_features( def generate_address_points( - feature_layers, zoom, source_layer=None): + feature_layers, zoom, source_layer=None, start_zoom=0): """ Generates address points from building polygons where there is an addr:housenumber tag on the building. Removes those tags from the @@ -1573,6 +1573,9 @@ def generate_address_points( assert source_layer, 'generate_label_features: missing source_layer' + if zoom < start_zoom: + return None + layer = _find_layer(feature_layers, source_layer) if layer is None: return None From c182c9cc94bc4ef29e8fa5b42567268740868c0f Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 13 Oct 2015 18:16:22 +0100 Subject: [PATCH 196/344] Suppress building names when they are copies of the address, or convert them to addresses if they are numeric. --- TileStache/Goodies/VecTiles/transform.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index b3c59661..2d5f6ed8 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1571,7 +1571,7 @@ def generate_address_points( building. """ - assert source_layer, 'generate_label_features: missing source_layer' + assert source_layer, 'generate_address_points: missing source_layer' if zoom < start_zoom: return None @@ -1588,6 +1588,18 @@ def generate_address_points( # we only want them on the address point. addr_housenumber = properties.pop('addr_housenumber', None) + # also consider it an address if the name of + # the building is just a number. + name = properties.get('name') + if name is not None and re.match('^[0-9-]+$', name): + if addr_housenumber is None: + addr_housenumber = properties.pop('name') + + # and also suppress the name if it's the same as + # the address. + elif name == addr_housenumber: + properties.pop('name') + # We only want to create address points for polygonal # buildings with address tags. if shape.geom_type not in ('Polygon', 'MultiPolygon') or \ From 9fec4936dd77914e0f5288ca6b413ba5f53ed115 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 13 Oct 2015 19:38:21 +0100 Subject: [PATCH 197/344] Fix logic for address points already in the data. --- TileStache/Goodies/VecTiles/transform.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 2d5f6ed8..14f9b499 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1584,12 +1584,10 @@ def generate_address_points( for feature in layer['features']: shape, properties, fid = feature - # remove address tags on the original polygon, - # we only want them on the address point. - addr_housenumber = properties.pop('addr_housenumber', None) + addr_housenumber = properties.get('addr_housenumber') - # also consider it an address if the name of - # the building is just a number. + # consider it an address if the name of the building + # is just a number. name = properties.get('name') if name is not None and re.match('^[0-9-]+$', name): if addr_housenumber is None: @@ -1607,6 +1605,10 @@ def generate_address_points( # keep the feature as-is, no modifications. continue + # remove address tags on the original polygon, + # we only want them on the address point. + properties.pop('addr_housenumber') + label_point = shape.representative_point() # we're only interested in a very few properties for @@ -1623,6 +1625,10 @@ def generate_address_points( if addr_street is not None: label_properties['addr_street'] = addr_street + oid = properties.get('id') + if oid is not None: + label_properties['id'] = oid + label_feature = label_point, label_properties, fid new_features.append(label_feature) From dd9f5766592672d17ed2a814977c5e347a084808 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 13 Oct 2015 15:09:35 -0400 Subject: [PATCH 198/344] Version bump -> 0.4.1 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1c402d9..20f5b9f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v0.4.1 +------ +* Make admin boundaries post-processing filter work with boundary linestring fragments rather than needing an oriented polygon. + v0.4.0 ------ * Add transform to dynamically create polygon `label_placement` features using representative point for existing layers (or exporte in new layer) as long as the input feature has a `name` or `sport` tag. From 5403334ce2fcd0b3afa429e48dc0f36926979e63 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 14 Oct 2015 12:50:14 +0100 Subject: [PATCH 199/344] As per comment, we do want house numbers on the label points, so leave that property set. --- TileStache/Goodies/VecTiles/transform.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 14f9b499..3163f562 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1605,10 +1605,6 @@ def generate_address_points( # keep the feature as-is, no modifications. continue - # remove address tags on the original polygon, - # we only want them on the address point. - properties.pop('addr_housenumber') - label_point = shape.representative_point() # we're only interested in a very few properties for From 848e4d413eb368696d5c80fb8254284aa3222e21 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 14 Oct 2015 12:58:09 +0100 Subject: [PATCH 200/344] Don't remove properties from the polygon any more. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 3163f562..0271a554 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1617,7 +1617,7 @@ def generate_address_points( if source is not None: label_properties['source'] = source - addr_street = properties.pop('addr_street', None) + addr_street = properties.get('addr_street') if addr_street is not None: label_properties['addr_street'] = addr_street From 088db0145757f649fb2a3e119a60f928cad0e0b5 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 14 Oct 2015 13:58:44 +0100 Subject: [PATCH 201/344] Added post-process filter to drop certain features. This is needed so that we can generate landuse labels without having landuse polygons. --- TileStache/Goodies/VecTiles/transform.py | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 0271a554..d9c0745e 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1650,3 +1650,52 @@ def parse_layer_as_float(shape, properties, fid, zoom): properties['layer'] = layer_float return shape, properties, fid + + +def drop_features_where( + feature_layers, zoom, source_layer=None, start_zoom=0, + property_name=None, drop_property=True, + geom_types=None): + """ + Drops some features entirely when they have a property + named `property_name` and its value is true. Also can + drop the property if `drop_property` is truthy. If + `geom_types` is present and not None, then only types in + that list are considered for dropping. + + This is useful for dropping features which we want to use + earlier in the pipeline (e.g: to generate points), but + that we don't want to appear in the final output. + """ + + assert source_layer, 'drop_features_where: missing source layer' + assert property_name, 'drop_features_where: missing property name' + + if zoom < start_zoom: + return None + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + new_features = [] + for feature in layer['features']: + shape, properties, fid = feature + + matches_geom_type = \ + geom_types is None or \ + shape.geom_type in geom_types + + val = None + if drop_property: + val = properties.pop(property_name, None) + else: + val = properties.get(property_name) + + if val == True and matches_geom_type: + pass + else: + new_features.append((shape, properties, fid)) + + layer['features'] = new_features + return layer From d61f3477acc27e68ee0ac3b197c16f1460181469 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 14 Oct 2015 18:57:18 +0100 Subject: [PATCH 202/344] Fix issue with footpaths not showing around Civic Center Plaza. --- TileStache/Goodies/VecTiles/transform.py | 125 +++++++++++++++++++++-- 1 file changed, 116 insertions(+), 9 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index d9c0745e..8d79ecaf 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -6,6 +6,7 @@ from shapely.strtree import STRtree from shapely.geometry.polygon import orient from shapely.ops import linemerge +from shapely.geometry.multipoint import MultiPoint from shapely.geometry.multilinestring import MultiLineString from shapely.geometry.multipolygon import MultiPolygon from shapely.geometry.collection import GeometryCollection @@ -555,6 +556,111 @@ def _geom_dimensions(g): return dim + +def _flatten_geoms(shape): + """ + Flatten a shape so that it is returned as a list + of single geometries. + + >>> [g.wkt for g in _flatten_geoms(shapely.wkt.loads('GEOMETRYCOLLECTION (MULTIPOINT(-1 -1, 0 0), GEOMETRYCOLLECTION (POINT(1 1), POINT(2 2), GEOMETRYCOLLECTION (POINT(3 3))), LINESTRING(0 0, 1 1))'))] + ['POINT (-1 -1)', 'POINT (0 0)', 'POINT (1 1)', 'POINT (2 2)', 'POINT (3 3)', 'LINESTRING (0 0, 1 1)'] + >>> _flatten_geoms(Polygon()) + [] + >>> _flatten_geoms(MultiPolygon()) + [] + """ + if shape.geom_type.startswith('Multi'): + return shape.geoms + + elif shape.is_empty: + return [] + + elif shape.type == 'GeometryCollection': + geoms = [] + + for g in shape.geoms: + geoms.extend(_flatten_geoms(g)) + + return geoms + + else: + return [shape] + + +def _filter_geom_types(shape, keep_dim): + """ + Return a geometry which consists of the geometries in + the input shape filtered so that only those of the + given dimension remain. Collapses any structure (e.g: + of geometry collections) down to a single or multi- + geometry. + + >>> _filter_geom_types(GeometryCollection(), _POINT_DIMENSION).wkt + 'GEOMETRYCOLLECTION EMPTY' + >>> _filter_geom_types(Point(0,0), _POINT_DIMENSION).wkt + 'POINT (0 0)' + >>> _filter_geom_types(Point(0,0), _LINE_DIMENSION).wkt + 'GEOMETRYCOLLECTION EMPTY' + >>> _filter_geom_types(Point(0,0), _POLYGON_DIMENSION).wkt + 'GEOMETRYCOLLECTION EMPTY' + >>> _filter_geom_types(LineString([(0,0),(1,1)]), _LINE_DIMENSION).wkt + 'LINESTRING (0 0, 1 1)' + >>> _filter_geom_types(Polygon([(0,0),(1,1),(1,0),(0,0)],[]), _POLYGON_DIMENSION).wkt + 'POLYGON ((0 0, 1 1, 1 0, 0 0))' + >>> _filter_geom_types(shapely.wkt.loads('GEOMETRYCOLLECTION (POINT(0 0), LINESTRING(0 0, 1 1))'), _POINT_DIMENSION).wkt + 'POINT (0 0)' + >>> _filter_geom_types(shapely.wkt.loads('GEOMETRYCOLLECTION (POINT(0 0), LINESTRING(0 0, 1 1))'), _LINE_DIMENSION).wkt + 'LINESTRING (0 0, 1 1)' + >>> _filter_geom_types(shapely.wkt.loads('GEOMETRYCOLLECTION (POINT(0 0), LINESTRING(0 0, 1 1))'), _POLYGON_DIMENSION).wkt + 'GEOMETRYCOLLECTION EMPTY' + >>> _filter_geom_types(shapely.wkt.loads('GEOMETRYCOLLECTION (POINT(0 0), GEOMETRYCOLLECTION (POINT(1 1), LINESTRING(0 0, 1 1)))'), _POINT_DIMENSION).wkt + 'MULTIPOINT (0 0, 1 1)' + >>> _filter_geom_types(shapely.wkt.loads('GEOMETRYCOLLECTION (MULTIPOINT(-1 -1, 0 0), GEOMETRYCOLLECTION (POINT(1 1), POINT(2 2), GEOMETRYCOLLECTION (POINT(3 3))), LINESTRING(0 0, 1 1))'), _POINT_DIMENSION).wkt + 'MULTIPOINT (-1 -1, 0 0, 1 1, 2 2, 3 3)' + >>> _filter_geom_types(shapely.wkt.loads('GEOMETRYCOLLECTION (LINESTRING(-1 -1, 0 0), GEOMETRYCOLLECTION (LINESTRING(1 1, 2 2), GEOMETRYCOLLECTION (POINT(3 3))), LINESTRING(0 0, 1 1))'), _LINE_DIMENSION).wkt + 'MULTILINESTRING ((-1 -1, 0 0), (1 1, 2 2), (0 0, 1 1))' + >>> _filter_geom_types(shapely.wkt.loads('GEOMETRYCOLLECTION (POLYGON((-2 -2, -2 2, 2 2, 2 -2, -2 -2)), GEOMETRYCOLLECTION (LINESTRING(1 1, 2 2), GEOMETRYCOLLECTION (POLYGON((3 3, 0 0, 1 0, 3 3)))), LINESTRING(0 0, 1 1))'), _POLYGON_DIMENSION).wkt + 'MULTIPOLYGON (((-2 -2, -2 2, 2 2, 2 -2, -2 -2)), ((3 3, 0 0, 1 0, 3 3)))' + """ + + # flatten the geometries, and keep the parts with the + # dimension that we want. each item in the parts list + # should be a single (non-multi) geometry. + parts = [] + for g in _flatten_geoms(shape): + if _geom_dimensions(g) == keep_dim: + parts.append(g) + + if len(parts) == 0: + # the only way we can signal an empty geometry, as + # MultiPoint([]) throws an exception. + return GeometryCollection() + + elif len(parts) == 1: + # return the singular geometry + return parts[0] + + else: + # try to make a multi-geometry of the desirect type + if keep_dim == _POINT_DIMENSION: + # not sure why the MultiPoint constructor wants + # its coordinates differently from MultiPolygon + # and MultiLineString... + coords = [] + for p in parts: + coords.extend(p.coords) + return MultiPoint(coords) + + elif keep_dim == _LINE_DIMENSION: + return MultiLineString(parts) + + elif keep_dim == _POLYGON_DIMENSION: + return MultiPolygon(parts) + + else: + raise Exception("Unknown dimension %d in _filter_geom_types" % keep_dim) + + # creates a list of indexes, each one for a different cut # attribute value, in priority order. # @@ -637,22 +743,23 @@ def cut(self, shape, props, fid): # same as the original, or we're not trying to keep the # same type. def _add(self, shape, props, fid, original_geom_dim): + # if keeping the same geometry type, then filter + # out anything that's different. + if self.keep_geom_type: + shape = _filter_geom_types( + shape, original_geom_dim) + # don't add empty shapes, they're completely - # useless. + # useless. the previous step may also have created + # an empty geometry if there weren't any items of + # the type we're looking for. if shape.is_empty: return - # use a custom dimension measurement here, as it - # turns out shapely geometry objects don't always - # form a hierarchy that's usable with isinstance. - shape_dim = _geom_dimensions(shape) - # add the shape as-is unless we're trying to keep # the geometry type or the geometry dimension is # identical. - if not self.keep_geom_type or \ - shape_dim == original_geom_dim: - self.new_features.append((shape, props, fid)) + self.new_features.append((shape, props, fid)) # intersects the shape with the cutting shape and From 2d3e405f07066b9d8539111ed48e728f568be2d0 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 15 Oct 2015 15:23:52 +0100 Subject: [PATCH 203/344] Add function to drop the area property if it's zero, as this isn't useful information (the area probably isn't zero, just unspecified). --- TileStache/Goodies/VecTiles/transform.py | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 8d79ecaf..a4d0804b 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1806,3 +1806,33 @@ def drop_features_where( layer['features'] = new_features return layer + + +def remove_zero_area(shape, properties, fid, zoom): + """ + All features get a numeric area tag, but for points this + is zero. The area probably isn't exactly zero, so it's + probably less confusing to just remove the tag to show + that the value is probably closer to "unspecified". + """ + + # remove the property if it's present. we _only_ want + # to replace it if it matches the positive, float + # criteria. + area = properties.pop("area", None) + + # try to parse a string if the area has been sent as a + # string. it should come through as a float, though, + # since postgres treats it as a real. + if isinstance(area, str): + area = to_float(area) + + if area is not None: + # cast to integer to match what we do for polygons. + # also the fractional parts of a sq.m are just + # noise really. + area = int(area) + if area > 0: + properties['area'] = area + + return shape, properties, fid From 15ed9096343acec4a3e3240498596b7533da653e Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 15 Oct 2015 16:55:12 +0100 Subject: [PATCH 204/344] Extract multi-geometry type constructor logic to higher level. --- TileStache/Goodies/VecTiles/transform.py | 27 ++++++++++++++---------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a4d0804b..173fbc14 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -631,17 +631,28 @@ def _filter_geom_types(shape, keep_dim): if _geom_dimensions(g) == keep_dim: parts.append(g) + # figure out how to construct a multi-geometry of the + # dimension wanted. + if keep_dim == _POINT_DIMENSION: + constructor = MultiPoint + + elif keep_dim == _LINE_DIMENSION: + constructor = MultiLineString + + elif keep_dim == _POLYGON_DIMENSION: + constructor = MultiPolygon + + else: + raise Exception("Unknown dimension %d in _filter_geom_types" % keep_dim) + if len(parts) == 0: - # the only way we can signal an empty geometry, as - # MultiPoint([]) throws an exception. - return GeometryCollection() + return constructor() elif len(parts) == 1: # return the singular geometry return parts[0] else: - # try to make a multi-geometry of the desirect type if keep_dim == _POINT_DIMENSION: # not sure why the MultiPoint constructor wants # its coordinates differently from MultiPolygon @@ -651,14 +662,8 @@ def _filter_geom_types(shape, keep_dim): coords.extend(p.coords) return MultiPoint(coords) - elif keep_dim == _LINE_DIMENSION: - return MultiLineString(parts) - - elif keep_dim == _POLYGON_DIMENSION: - return MultiPolygon(parts) - else: - raise Exception("Unknown dimension %d in _filter_geom_types" % keep_dim) + return constructor(parts) # creates a list of indexes, each one for a different cut From 33030aa7904bce6f16b449d7d5b19d1c4a35b927 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 15 Oct 2015 16:58:51 +0100 Subject: [PATCH 205/344] Use more descriptive exception type. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 173fbc14..c2409880 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -643,7 +643,7 @@ def _filter_geom_types(shape, keep_dim): constructor = MultiPolygon else: - raise Exception("Unknown dimension %d in _filter_geom_types" % keep_dim) + raise ValueError("Unknown dimension %d in _filter_geom_types" % keep_dim) if len(parts) == 0: return constructor() From e14e9763e3569a850317cfdf53418553257acfbb Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 15 Oct 2015 17:02:09 +0100 Subject: [PATCH 206/344] Move shape type filter up to avoid doing more work than we have to. --- TileStache/Goodies/VecTiles/transform.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index c2409880..09676f4d 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1696,6 +1696,11 @@ def generate_address_points( for feature in layer['features']: shape, properties, fid = feature + # We only want to create address points for polygonal + # buildings with address tags. + if shape.geom_type not in ('Polygon', 'MultiPolygon'): + continue + addr_housenumber = properties.get('addr_housenumber') # consider it an address if the name of the building @@ -1710,11 +1715,9 @@ def generate_address_points( elif name == addr_housenumber: properties.pop('name') - # We only want to create address points for polygonal - # buildings with address tags. - if shape.geom_type not in ('Polygon', 'MultiPolygon') or \ - addr_housenumber is None: - # keep the feature as-is, no modifications. + # if there's no address, then keep the feature as-is, + # no modifications. + if addr_housenumber is None: continue label_point = shape.representative_point() From 256b963d3385cd80ac1616144d272551b0546599 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 15 Oct 2015 17:10:32 +0100 Subject: [PATCH 207/344] Treat str, unicode the same way. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 09676f4d..cd65cb27 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1832,7 +1832,7 @@ def remove_zero_area(shape, properties, fid, zoom): # try to parse a string if the area has been sent as a # string. it should come through as a float, though, # since postgres treats it as a real. - if isinstance(area, str): + if isinstance(area, (str, unicode)): area = to_float(area) if area is not None: From 391da8ac5736aa2fb4e6b8ee9660f579f4ff735c Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 15 Oct 2015 17:47:18 +0100 Subject: [PATCH 208/344] Made docstring more clear about values for the property. Updated property getting/dropping logic to be clearer. Dropped non-idiomatic use of pass in favour of continue. --- TileStache/Goodies/VecTiles/transform.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index cd65cb27..b4b87503 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1773,7 +1773,8 @@ def drop_features_where( geom_types=None): """ Drops some features entirely when they have a property - named `property_name` and its value is true. Also can + named `property_name` and its value is true. Note that it + must be identically True, not just truthy. Also can drop the property if `drop_property` is truthy. If `geom_types` is present and not None, then only types in that list are considered for dropping. @@ -1801,16 +1802,21 @@ def drop_features_where( geom_types is None or \ shape.geom_type in geom_types - val = None + # figure out what to do with the property - do we + # want to drop it, or just fetch it? + func = properties.get if drop_property: - val = properties.pop(property_name, None) - else: - val = properties.get(property_name) + func = properties.drop + + val = func(property_name, None) + # skip (i.e: drop) the geometry if the value is + # true and it's the geometry type we want. if val == True and matches_geom_type: - pass - else: - new_features.append((shape, properties, fid)) + continue + + # default case is to keep the feature + new_features.append((shape, properties, fid)) layer['features'] = new_features return layer From ce54fdac226b8ae44c04958152dcd8bff30136c5 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 15 Oct 2015 19:01:58 +0100 Subject: [PATCH 209/344] Fixed typo - method is called pop, not drop. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index b4b87503..989e4977 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1806,7 +1806,7 @@ def drop_features_where( # want to drop it, or just fetch it? func = properties.get if drop_property: - func = properties.drop + func = properties.pop val = func(property_name, None) From f40c21d670335877f4e79354b1cb72c3498a7ff6 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Sat, 17 Oct 2015 15:40:48 +0100 Subject: [PATCH 210/344] Added post-process filter to remove duplicate features. Where being a duplicate is defined by having some subset of properties in common and being within a certain distance of the "original" feature. Feature order is not changed, so more features should be sorted into importance order by the sorting step which runs at the end of the filter functions. The type of feature to filter out may be selected using the geometry filter. This is primarily of use when filtering out duplicate POIs / labels only. --- TileStache/Goodies/VecTiles/transform.py | 82 ++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 989e4977..9786a11c 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1850,3 +1850,85 @@ def remove_zero_area(shape, properties, fid, zoom): properties['area'] = area return shape, properties, fid + + +# circumference of the extent of the world in mercator "meters" +_MERCATOR_CIRCUMFERENCE = 40075016.68 + + +def remove_duplicate_features( + feature_layers, zoom, source_layer=None, start_zoom=0, + property_keys=None, geometry_types=None, min_distance=0.0): + """ + Removes duplicate features from the layer. The definition of + duplicate is anything which has the same values for the tuple + of values associated with the property_keys. + + For example, if property_keys was ['name', 'kind'], then only + the first feature of those with the same value for the name + and kind properties would be kept in the output. + """ + + assert source_layer, 'remove_duplicate_features: missing source layer' + + # note that the property keys or geometry types could be empty, + # but then this post-process filter would do nothing. so we + # assume that the user didn't intend this, or they wouldn't have + # included the filter in the first place. + assert property_keys, 'remove_duplicate_features: missing or empty property keys' + assert geometry_types, 'remove_duplicate_features: missing or empty geometry types' + + if zoom < start_zoom: + return None + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + # keep a set of the tuple of the property keys. this will tell + # us if the feature is unique while allowing us to maintain the + # sort order by only dropping later, presumably less important, + # features. we keep the geometry of the seen items too, so that + # we can tell if any new feature is significantly far enough + # away that it should be shown again. + seen_items = dict() + + def meters_to_pixels(distance): + return distance * float(1 << (zoom + 8)) / 40075016.68 + + new_features = [] + for feature in layer['features']: + shape, props, fid = feature + + keep_feature = True + if shape.geom_type in geometry_types: + key = tuple([props.get(k) for k in property_keys]) + seen_geoms = seen_items.get(key) + + if seen_geoms is None: + # first time we've seen this item, so keep it in + # the output. + seen_items[key] = [shape] + + else: + # if the distance is greater than the minimum set + # for this zoom, then we also keep it. + distance = min([shape.distance(s) for s in seen_geoms]) + + # correct for zoom - we want visual distance, which + # means (pseudo) pixels. + distance = meters_to_pixels(distance) + + if distance > min_distance: + # keep this geom to suppress any other labels + # nearby. + seen_geoms.append(shape) + + else: + keep_feature = False + + if keep_feature: + new_features.append(feature) + + layer['features'] = new_features + return layer From a8d1c3ac41ffaab5fe47bdcf6d3e92c3a22bc66b Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 19 Oct 2015 16:53:26 +0100 Subject: [PATCH 211/344] Sort by number of subway lines descending. --- TileStache/Goodies/VecTiles/sort.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index d51f7665..881bd796 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -47,6 +47,23 @@ def _sort_by_scalerank_then_population(features): return features +def _by_subway_lines(feature): + wkb, props, fid = feature + + num_lines = 0 + subway_lines = props.get('subway_lines') + if subway_lines is not None: + num_lines = len(subway_lines) + + return num_lines + + +def _sort_by_subway_lines_then_feature_id(features): + features.sort(key=_by_feature_id) + features.sort(key=_by_subway_lines, reverse=True) + return features + + def buildings(features, zoom): return _sort_by_area_then_id(features) @@ -64,7 +81,7 @@ def places(features, zoom): def pois(features, zoom): - return _sort_features_by_key(features, _by_feature_id) + return _sort_by_subway_lines_then_feature_id(features) def roads(features, zoom): From 799b24dc4d6f26aabc31ea805a23f18bffa46de2 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 20 Oct 2015 16:33:43 +0100 Subject: [PATCH 212/344] Move to_float to its own file, as we need to re-sort after merging subway stations, which means importing the POIs sort function and we can't do circular imports. --- TileStache/Goodies/VecTiles/sort.py | 2 +- TileStache/Goodies/VecTiles/transform.py | 15 +-------------- TileStache/Goodies/VecTiles/util.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 15 deletions(-) create mode 100644 TileStache/Goodies/VecTiles/util.py diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index 881bd796..a85d66ea 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -1,4 +1,4 @@ -from transform import to_float +from util import to_float # sort functions to apply to features diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 9786a11c..0ab1df54 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -10,23 +10,10 @@ from shapely.geometry.multilinestring import MultiLineString from shapely.geometry.multipolygon import MultiPolygon from shapely.geometry.collection import GeometryCollection +from util import to_float import re -# attempts to convert x to a floating point value, -# first removing some common punctuation. returns -# None if conversion failed. -def to_float(x): - if x is None: - return None - # normalize punctuation - x = x.replace(';', '.').replace(',', '.') - try: - return float(x) - except ValueError: - return None - - feet_pattern = re.compile('([+-]?[0-9.]+)\'(?: *([+-]?[0-9.]+)")?') number_pattern = re.compile('([+-]?[0-9.]+)') diff --git a/TileStache/Goodies/VecTiles/util.py b/TileStache/Goodies/VecTiles/util.py new file mode 100644 index 00000000..1a3dc560 --- /dev/null +++ b/TileStache/Goodies/VecTiles/util.py @@ -0,0 +1,12 @@ +# attempts to convert x to a floating point value, +# first removing some common punctuation. returns +# None if conversion failed. +def to_float(x): + if x is None: + return None + # normalize punctuation + x = x.replace(';', '.').replace(',', '.') + try: + return float(x) + except ValueError: + return None From e6ff5d78c80f88b23618058eee8fa30c22a3a3d9 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 20 Oct 2015 16:41:35 +0100 Subject: [PATCH 213/344] Add post-process filter to normalise and merge station POIs and another to keep only N features of a certain type in a layer. --- TileStache/Goodies/VecTiles/transform.py | 174 ++++++++++++++++++++++- 1 file changed, 173 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 0ab1df54..6ba44e9a 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -17,6 +17,13 @@ feet_pattern = re.compile('([+-]?[0-9.]+)\'(?: *([+-]?[0-9.]+)")?') number_pattern = re.compile('([+-]?[0-9.]+)') +# used to detect if the "name" of a building is +# actually a house number. +digits_pattern = re.compile('^[0-9-]+$') + +# used to detect station names which are followed by a +# parenthetical list of line names. +station_pattern = re.compile('([^(]*)\(([^)]*)\).*') def _to_float_meters(x): if x is None: @@ -1693,7 +1700,7 @@ def generate_address_points( # consider it an address if the name of the building # is just a number. name = properties.get('name') - if name is not None and re.match('^[0-9-]+$', name): + if name is not None and digits_pattern.match(name): if addr_housenumber is None: addr_housenumber = properties.pop('name') @@ -1919,3 +1926,168 @@ def meters_to_pixels(distance): layer['features'] = new_features return layer + + +def normalize_and_merge_duplicate_stations( + feature_layers, zoom, source_layer=None, start_zoom=0, + end_zoom=None): + """ + Normalise station names by removing any parenthetical lines + lists at the end (e.g: "Foo St (A, C, E)"). Parse this and + use it to replace the `subway_lines` list if that is empty + or isn't present. + + Use the name, now appropriately trimmed, to merge station + POIs together, unioning their subway lines. + + Finally, re-sort the features in case the merging has caused + the subway stations to be out-of-order. + """ + + assert source_layer, 'normalize_and_merge_duplicate_stations: missing source layer' + + if zoom < start_zoom: + return None + + # we probably don't want to do this at higher zooms (e.g: 17 & + # 18), even if there are a bunch of stations very close + # together. + if end_zoom is not None and zoom > end_zoom: + return None + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + seen_stations = {} + new_features = [] + for feature in layer['features']: + shape, props, fid = feature + + kind = props.get('kind') + name = props.get('name') + if name is not None and kind == 'station': + # this should match station names where the name is + # followed by a ()-bracketed list of line names. this + # is common in NYC, and we want to normalise by + # stripping these off and using it to provide the + # list of lines if we haven't already got that info. + m = station_pattern.match(name) + + subway_lines = props.get('subway_lines', []) + + if m: + # if the lines aren't present or are empty + if not subway_lines: + lines = m.group(2).split(',') + subway_lines = [x.strip() for x in lines] + props['subway_lines'] = subway_lines + + # update name so that it doesn't contain all the + # lines. + name = m.group(1).strip() + props['name'] = name + + seen_idx = seen_stations.get(name) + if seen_idx is None: + seen_stations[name] = len(new_features) + + # ensure that subway lines is present and is of + # list type for when we append to it later if we + # find a duplicate. + props['subway_lines'] = subway_lines + new_features.append(feature) + + else: + # get the properties and append this duplicate's + # subway lines to the list on the original + # feature. + seen_props = new_features[seen_idx][1] + + # make sure lines are unique + seen_subway_lines = set(seen_props['subway_lines']) + subway_lines = set(subway_lines) + subway_lines.update(seen_subway_lines) + + seen_props['subway_lines'] = list(subway_lines) + + else: + # not a station, or name is missing - we can't + # de-dup these. + new_features.append(feature) + + # need to re-sort: removing duplicates would have changed + # the number of lines for each station. + sort_pois(new_features, zoom) + + layer['features'] = new_features + return layer + + +def _match_props(props, items_matching): + """ + Checks if all the items in `items_matching` are also + present in `props`. If so, returns true. Otherwise + returns false. + """ + + for k, v in items_matching.iteritems(): + if props.get(k) != v: + return False + + return True + + +def keep_n_features( + feature_layers, zoom, source_layer=None, start_zoom=0, + end_zoom=None, items_matching=None, max_items=None): + """ + Keep only the first N features matching `items_matching` + in the layer. This is primarily useful for removing + features which are abundant in some places but scarce in + others. Rather than try to set some global threshold which + works well nowhere, instead sort appropriately and take a + number of features which is appropriate per-tile. + + This is done by counting each feature which matches _all_ + the key-value pairs in `items_matching` and, when the + count is larger than `max_items`, dropping those features. + """ + + assert source_layer, 'keep_n_features: missing source layer' + + # leaving items_matching or max_items as None (or zero) + # would mean that this filter would do nothing, so assume + # that this is really a configuration error. + assert items_matching, 'keep_n_features: missing or empty item match dict' + assert max_items, 'keep_n_features: missing or zero max number of items' + + if zoom < start_zoom: + return None + + # we probably don't want to do this at higher zooms (e.g: 17 & + # 18), even if there are a bunch of stations very close + # together. + if end_zoom is not None and zoom > end_zoom: + return None + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + count = 0 + new_features = [] + for shape, props, fid in layer['features']: + keep_feature = True + + if _match_props(props, items_matching): + count += 1 + if count > max_items: + keep_feature = False + + if keep_feature: + new_features.append((shape, props, fid)) + + + layer['features'] = new_features + return layer From 2aedbd7ccaf24759ac7488f798335629d4dde85c Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 21 Oct 2015 14:29:59 +0100 Subject: [PATCH 214/344] Add function to rank POIs of a certain type within the layer, which can be useful for the client. --- TileStache/Goodies/VecTiles/transform.py | 36 +++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 6ba44e9a..f14fdd8b 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -11,6 +11,7 @@ from shapely.geometry.multipolygon import MultiPolygon from shapely.geometry.collection import GeometryCollection from util import to_float +from sort import pois as sort_pois import re @@ -2088,6 +2089,39 @@ def keep_n_features( if keep_feature: new_features.append((shape, props, fid)) - layer['features'] = new_features return layer + + +def rank_features( + feature_layers, zoom, source_layer=None, start_zoom=0, + items_matching=None, rank_key=None): + """ + Enumerate the features matching `items_matching` and insert + the rank as a property with the key `rank_key`. This is + useful for the client, so that it can selectively display + only the top features, or de-emphasise the later features. + """ + + assert source_layer, 'rank_features: missing source layer' + + # leaving items_matching or rank_key as None would mean + # that this filter would do nothing, so assume that this + # is really a configuration error. + assert items_matching, 'rank_features: missing or empty item match dict' + assert rank_key, 'rank_features: missing or empty rank key' + + if zoom < start_zoom: + return None + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + count = 0 + for shape, props, fid in layer['features']: + if _match_props(props, items_matching): + count += 1 + props[rank_key] = count + + return layer From 742e3c18f4885f32706f49bb30360073ce98801d Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 21 Oct 2015 18:02:32 +0100 Subject: [PATCH 215/344] Made logic more readable, more efficient and better documented. --- TileStache/Goodies/VecTiles/transform.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index f14fdd8b..8e8dc721 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2006,20 +2006,20 @@ def normalize_and_merge_duplicate_stations( seen_props = new_features[seen_idx][1] # make sure lines are unique - seen_subway_lines = set(seen_props['subway_lines']) - subway_lines = set(subway_lines) - subway_lines.update(seen_subway_lines) - - seen_props['subway_lines'] = list(subway_lines) + unique_subway_lines = set(subway_lines) & \ + set(seen_props['subway_lines']) + seen_props['subway_lines'] = list(unique_subway_lines) else: # not a station, or name is missing - we can't # de-dup these. new_features.append(feature) - # need to re-sort: removing duplicates would have changed - # the number of lines for each station. - sort_pois(new_features, zoom) + # might need to re-sort, if we merged any stations: + # removing duplicates would have changed the number + # of lines for each station. + if seen_stations: + sort_pois(new_features, zoom) layer['features'] = new_features return layer @@ -2067,8 +2067,9 @@ def keep_n_features( return None # we probably don't want to do this at higher zooms (e.g: 17 & - # 18), even if there are a bunch of stations very close - # together. + # 18), even if there are a bunch of features in the tile, as + # we use the high-zoom tiles for overzooming to 20+, and we'd + # eventually expect to see _everything_. if end_zoom is not None and zoom > end_zoom: return None From 3e7c3e7d7cdf488019e8cf0a7c4ab0970a72ef61 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 23 Oct 2015 16:54:27 +0100 Subject: [PATCH 216/344] Snap geometry coordinates to a static grid. This helps paper over numerical rounding differences between different import pipelines. --- TileStache/Goodies/VecTiles/transform.py | 124 ++++++++++++++++++----- 1 file changed, 100 insertions(+), 24 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 8e8dc721..9ae91a7f 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -6,6 +6,10 @@ from shapely.strtree import STRtree from shapely.geometry.polygon import orient from shapely.ops import linemerge +from shapely.geometry import Point +from shapely.geometry import LineString +from shapely.geometry import LinearRing +from shapely.geometry import Polygon from shapely.geometry.multipoint import MultiPoint from shapely.geometry.multilinestring import MultiLineString from shapely.geometry.multipolygon import MultiPolygon @@ -1181,12 +1185,86 @@ def _make_new_properties(props, props_instructions): return new_props + +def _snap_to_grid(shape, grid_size): + """ + Snap coordinates of a shape to a multiple of `grid_size`. + + This can be useful when there's some error in point + positions, but we're using an algorithm which is very + sensitive to coordinate exactness. For example, when + calculating the boundary of several items, it makes a + big difference whether the shapes touch or there's a + very small gap between them. + + This is implemented here because it doesn't exist in + GEOS or Shapely. It exists in PostGIS, but only because + it's implemented there as well. Seems like it would be a + useful thing to have in GEOS, though. + + >>> _snap_to_grid(Point(0.5, 0.5), 1).wkt + 'POINT (1 1)' + >>> _snap_to_grid(Point(0.1, 0.1), 1).wkt + 'POINT (0 0)' + >>> _snap_to_grid(Point(-0.1, -0.1), 1).wkt + 'POINT (-0 -0)' + >>> _snap_to_grid(LineString([(1.1,1.1),(1.9,0.9)]), 1).wkt + 'LINESTRING (1 1, 2 1)' + _snap_to_grid(Polygon([(0.1,0.1),(3.1,0.1),(3.1,3.1),(0.1,3.1),(0.1,0.1)],[[(1.1,0.9),(1.1,1.9),(2.1,1.9),(2.1,0.9),(1.1,0.9)]]), 1).wkt + 'POLYGON ((0 0, 3 0, 3 3, 0 3, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))' + >>> _snap_to_grid(MultiPoint([Point(0.1, 0.1), Point(0.9, 0.9)]), 1).wkt + 'MULTIPOINT (0 0, 1 1)' + >>> _snap_to_grid(MultiLineString([LineString([(0.1, 0.1), (0.9, 0.9)]), LineString([(0.9, 0.1),(0.1,0.9)])]), 1).wkt + 'MULTILINESTRING ((0 0, 1 1), (1 0, 0 1))' + """ + + # snap a single coordinate value + def _snap(c): + return grid_size * round(c / grid_size, 0) + + # snap all coordinate pairs in something iterable + def _snap_coords(c): + return [(_snap(x), _snap(y)) for x, y in c] + + # recursively snap all coordinates in an iterable over + # geometries. + def _snap_multi(geoms): + return [_snap_to_grid(g, grid_size) for g in geoms] + + shape_type = shape.geom_type + if shape_type == 'Point': + return Point(_snap(shape.x), _snap(shape.y)) + + elif shape_type == 'LineString': + return LineString(_snap_coords(shape.coords)) + + elif shape_type == 'Polygon': + exterior = LinearRing(_snap_coords(shape.exterior.coords)) + interiors = [] + for interior in shape.interiors: + interiors.append(LinearRing(_snap_coords(interior.coords))) + return Polygon(exterior, interiors) + + elif shape_type == 'MultiPoint': + return MultiPoint(_snap_multi(shape.geoms)) + + elif shape_type == 'MultiLineString': + return MultiLineString(_snap_multi(shape.geoms)) + + elif shape_type == 'MultiPolygon': + return MultiPolygon(_snap_multi(shape.geoms)) + + else: + raise Exception("_snap_to_grid unimplemented for shape type %s" % repr(shape_type)) + + def exterior_boundaries(feature_layers, zoom, base_layer, new_layer_name=None, prop_transform=None, buffer_size=None, - start_zoom=0): + start_zoom=0, + snap_tolerance=None): """ create new fetures from the boundaries of polygons in the base layer, subtracting any sections of the @@ -1221,13 +1299,7 @@ def exterior_boundaries(feature_layers, zoom, # search through all the layers and extract the one # which has the name of the base layer we were given # as a parameter. - for feature_layer in feature_layers: - layer_datum = feature_layer['layer_datum'] - layer_name = layer_datum['name'] - - if layer_name == base_layer: - layer = feature_layer - break + layer = _find_layer(feature_layers, base_layer) # if we failed to find the base layer then it's # possible the user just didn't ask for it, so return @@ -1246,36 +1318,40 @@ def exterior_boundaries(feature_layers, zoom, # polygonal features, and that intersecting with lines # can give some unexpected results. indexable_features = list() + indexable_shapes = list() for shape, props, fid in features: if shape.geom_type in ('Polygon', 'MultiPolygon'): - indexable_features.append(shape) - index = STRtree(indexable_features) + snapped = shape + if snap_tolerance is not None: + snapped = _snap_to_grid(shape, snap_tolerance) + indexable_features.append((snapped, props, fid)) + indexable_shapes.append(snapped) + index = STRtree(indexable_shapes) new_features = list() # loop through all the polygons, taking the boundary # of each and subtracting any parts which are within # other polygons. what remains (if anything) is the # new feature. - for feature in features: + for feature in indexable_features: shape, props, fid = feature - if shape.geom_type in ('Polygon', 'MultiPolygon'): - boundary = shape.boundary - cutting_shapes = index.query(boundary) + boundary = shape.boundary + cutting_shapes = index.query(boundary) - for cutting_shape in cutting_shapes: - if cutting_shape is not shape: - buf = cutting_shape + for cutting_shape in cutting_shapes: + if cutting_shape is not shape: + buf = cutting_shape - if buffer_size is not None: - buf = buf.buffer(buffer_size) + if buffer_size is not None: + buf = buf.buffer(buffer_size) - boundary = boundary.difference(buf) + boundary = boundary.difference(buf) - if not boundary.is_empty: - new_props = _make_new_properties(props, - prop_transform) - new_features.append((boundary, new_props, fid)) + if not boundary.is_empty: + new_props = _make_new_properties(props, + prop_transform) + new_features.append((boundary, new_props, fid)) if new_layer_name is None: # no new layer requested, instead add new From fad1f0eee34e5d56d51a9e8320350dcd44fbcfed Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 23 Oct 2015 17:07:04 +0100 Subject: [PATCH 217/344] Dirty hack to treat shapes with _exactly_ the same area as identical. --- TileStache/Goodies/VecTiles/transform.py | 37 ++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 9ae91a7f..13eca61a 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1312,6 +1312,28 @@ def exterior_boundaries(feature_layers, zoom, features = layer['features'] + # this exists to enable a dirty hack to try and work + # around duplicate geometries in the database. this + # happens when a multipolygon relation can't + # supersede a member way because the way contains tags + # which aren't present on the relation. working around + # this by calling "union" on geometries proved to be + # too expensive (~3x current), so this hack looks at + # the way_area of each object, and uses that as a + # proxy for identity. it's not perfect, but the chance + # that there are two overlapping polygons of exactly + # the same size must be pretty small. however, the + # STRTree we're using as a spatial index doesn't + # directly support setting attributes on the indexed + # geometries, so this class exists to carry the area + # attribute through the index to the point where we + # want to use it. + class geom_with_area: + def __init__(self, geom, area): + self.geom = geom + self.area = area + self._geom = geom._geom + # create an index so that we can efficiently find the # polygons intersecting the 'current' one. Note that # we're only interested in intersecting with other @@ -1325,7 +1347,7 @@ def exterior_boundaries(feature_layers, zoom, if snap_tolerance is not None: snapped = _snap_to_grid(shape, snap_tolerance) indexable_features.append((snapped, props, fid)) - indexable_shapes.append(snapped) + indexable_shapes.append(geom_with_area(snapped, props.get('area'))) index = STRtree(indexable_shapes) new_features = list() @@ -1339,8 +1361,12 @@ def exterior_boundaries(feature_layers, zoom, boundary = shape.boundary cutting_shapes = index.query(boundary) - for cutting_shape in cutting_shapes: - if cutting_shape is not shape: + for cutting_item in cutting_shapes: + cutting_shape = cutting_item.geom + cutting_area = cutting_item.area + + if cutting_shape is not shape and \ + cutting_area != props.get('area'): buf = cutting_shape if buffer_size is not None: @@ -1348,6 +1374,11 @@ def exterior_boundaries(feature_layers, zoom, boundary = boundary.difference(buf) + # filter only linestring-like objects. we don't + # want any points which might have been created + # by the intersection. + boundary = _filter_geom_types(boundary, _LINE_DIMENSION) + if not boundary.is_empty: new_props = _make_new_properties(props, prop_transform) From f35a65195e198eac350b0206ba32bec8cabad82c Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 23 Oct 2015 17:21:34 +0100 Subject: [PATCH 218/344] Add extra check to avoid duplicate boundary geometries. --- TileStache/Goodies/VecTiles/transform.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 13eca61a..456bcafd 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1365,8 +1365,17 @@ def __init__(self, geom, area): cutting_shape = cutting_item.geom cutting_area = cutting_item.area + # dirty hack: this object is probably a + # superseded way if the ID is positive and + # the area is the same as the cutting area. + # using the ID check here prevents the + # boundary from being duplicated. + is_superseded_way = \ + cutting_area == props.get('area') and \ + props.get('id') > 0 + if cutting_shape is not shape and \ - cutting_area != props.get('area'): + not is_superseded_way: buf = cutting_shape if buffer_size is not None: From d5f8e73a4a0193a07a6d9d0b9732ac416d5571b0 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 23 Oct 2015 19:48:35 +0100 Subject: [PATCH 219/344] Use ValueError instead of plain Exception. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 456bcafd..af82d28d 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1255,7 +1255,7 @@ def _snap_multi(geoms): return MultiPolygon(_snap_multi(shape.geoms)) else: - raise Exception("_snap_to_grid unimplemented for shape type %s" % repr(shape_type)) + raise ValueError("_snap_to_grid: unimplemented for shape type %s" % repr(shape_type)) def exterior_boundaries(feature_layers, zoom, From 17bb9ada4a6d3377cb8c8b138f3f244a00b21d34 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 26 Oct 2015 15:02:33 +0000 Subject: [PATCH 220/344] Update sort order to account for service values. --- TileStache/Goodies/VecTiles/transform.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index af82d28d..c0efbbde 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -243,10 +243,13 @@ def road_sort_key(shape, properties, fid, zoom): highway = properties.get('highway', '') railway = properties.get('railway', '') aeroway = properties.get('aeroway', '') + service = properties.get('service') + + is_railway = railway in ('rail', 'tram', 'light_rail', 'narrow_guage', 'monorail') if highway == 'motorway': sort_val += 24 - elif railway in ('rail', 'tram', 'light_rail', 'narrow_guage', 'monorail'): + elif is_railway: sort_val += 23 elif highway == 'trunk': sort_val += 22 @@ -265,6 +268,20 @@ def road_sort_key(shape, properties, fid, zoom): else: sort_val += 15 + if is_railway and service is not None: + if service in ('spur', 'siding'): + # make sort val more like residential, unclassified which + # also come in at zoom 12 + sort_val -= 6 + elif service == 'yard': + sort_val -= 7 + else: + sort_val -= 8 + + if highway == 'service' and service is not None: + # sort alley, driveway, etc... under service + sort_val -= 1 + if zoom >= 15: # Bridges and tunnels add +/- 10 bridge = properties.get('bridge') From f3b947eb44d5e91df1d88b34a33bbe9feede93dc Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 26 Oct 2015 16:43:30 +0000 Subject: [PATCH 221/344] Try using the 'buffer' trick to make polygons valid after snapping. If that doesn't work, drop them and continue. Better to have a tile missing some water boundaries than no tile at all. --- TileStache/Goodies/VecTiles/transform.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index c0efbbde..fba4ed24 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1363,8 +1363,24 @@ def __init__(self, geom, area): snapped = shape if snap_tolerance is not None: snapped = _snap_to_grid(shape, snap_tolerance) + + # snapping coordinates might make the shape + # invalid, so we need a way to clean them. + # one simple, but not foolproof, way is to + # buffer them by 0. + if not snapped.is_valid: + snapped = snapped.buffer(0) + + # that still might not have done the trick, + # so drop any polygons which are still + # invalid so as not to cause errors later. + if not snapped.is_valid: + # TODO: log this as a warning! + continue + indexable_features.append((snapped, props, fid)) indexable_shapes.append(geom_with_area(snapped, props.get('area'))) + index = STRtree(indexable_shapes) new_features = list() From 9a9928e9cefd7736d7b4606c2e049bddf24c30e9 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 26 Oct 2015 18:11:31 +0000 Subject: [PATCH 222/344] Synthesize a volume attribute for the client to filter on. --- TileStache/Goodies/VecTiles/transform.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index c0efbbde..8cf8036d 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -179,8 +179,11 @@ def building_height(shape, properties, fid, zoom): height = _building_calc_height( properties.get('height'), properties.get('building:levels'), _building_calc_levels) + area = properties.get('area') if height is not None: properties['height'] = height + if area is not None: + properties['volume'] = height * area else: properties.pop('height', None) return shape, properties, fid From 2528f2d1ad88964411e457735a59225f8f5da172 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 26 Oct 2015 19:04:26 +0000 Subject: [PATCH 223/344] Synthesize volume as a separate function. --- TileStache/Goodies/VecTiles/transform.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 8cf8036d..fcb1c607 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -179,11 +179,8 @@ def building_height(shape, properties, fid, zoom): height = _building_calc_height( properties.get('height'), properties.get('building:levels'), _building_calc_levels) - area = properties.get('area') if height is not None: properties['height'] = height - if area is not None: - properties['volume'] = height * area else: properties.pop('height', None) return shape, properties, fid @@ -200,6 +197,14 @@ def building_min_height(shape, properties, fid, zoom): return shape, properties, fid +def synthesize_volume(shape, props, fid, zoom): + area = props.get('area') + height = props.get('height') + if area is not None and height is not None: + props['volume'] = area * height + return shape, props, fid + + def building_trim_properties(shape, properties, fid, zoom): properties = _remove_properties( properties, From d3dbd75baa61c471393ea6dfc6a5c00272b4d6ad Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 26 Oct 2015 19:45:01 +0000 Subject: [PATCH 224/344] Add post-process filter function to ensure certain numeric minima. Used to double-check the filter on building area / volume before sending data to the client. --- TileStache/Goodies/VecTiles/transform.py | 56 ++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index fcb1c607..8e058bb8 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2267,3 +2267,59 @@ def rank_features( props[rank_key] = count return layer + + +def numeric_min_filter( + feature_layers, zoom, source_layer=None, filters=None, + mode=None): + """ + Keep only features which have properties equal or greater + than the configured minima. These are in a dict per zoom + like this: + + { 15: { 'area': 1000 }, 16: { 'area': 2000 } } + + This would mean that at zooms 15 and 16, the filter was + active. At other zooms it would do nothing. + + Multiple filters can be given for a single zoom. The + `mode` parameter can be set to 'any' to require that only + one of the filters needs to match, or any other value to + use the default 'all', which requires all filters to + match. + """ + + assert source_layer, 'rank_features: missing source layer' + + # assume missing filter is a config error. + assert filters, 'numeric_min_filter: missing or empty filters dict' + + # get the minimum filters for this zoom, and return if + # there are none to apply. + minima = filters.get(zoom) + if not minima: + return None + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + # choose whether all minima have to be met, or just + # one of them. + aggregate_func = all + if mode == 'any': + aggregate_func = any + + new_features = [] + for shape, props, fid in layer['features']: + keep = [] + + for prop, min_val in minima.iteritems(): + val = props.get(prop) + keep.append(val >= min_val) + + if aggregate_func(keep): + new_features.append((shape, props, fid)) + + layer['features'] = new_features + return layer From 1d9f765afa8a01594963f63cb8973dc355296dbc Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 26 Oct 2015 20:48:41 +0000 Subject: [PATCH 225/344] Process aerialways and try to keep values distinct from roads. --- TileStache/Goodies/VecTiles/transform.py | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index fba4ed24..76b66c2f 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -122,6 +122,10 @@ def _building_calc_height(height_val, levels_val, levels_calc_fn): 'path', 'cycleway')) road_kind_rail = set(('rail', 'tram', 'light_rail', 'narrow_gauge', 'monorail', 'subway')) +road_kind_aerialway = set(('gondola', 'cable_car', 'chair_lift', 'drag_lift', + 'platter', 't-bar', 'goods', 'magic_carpet', + 'rope_tow', 'yes', 'zip_line', 'j-bar', 'unknown', + 'mixed_lift', 'canopy', 'cableway')) def _road_kind(properties): @@ -138,6 +142,9 @@ def _road_kind(properties): route = properties.get('route') if route == 'ferry': return 'ferry' + aerialway = properties.get('aerialway') + if aerialway in road_kind_aerialway: + return 'aerialway' return 'minor_road' @@ -243,6 +250,7 @@ def road_sort_key(shape, properties, fid, zoom): highway = properties.get('highway', '') railway = properties.get('railway', '') aeroway = properties.get('aeroway', '') + aerialway = properties.get('aerialway', '') service = properties.get('service') is_railway = railway in ('rail', 'tram', 'light_rail', 'narrow_guage', 'monorail') @@ -265,6 +273,12 @@ def road_sort_key(shape, properties, fid, zoom): sort_val += 17 elif highway in ('unclassified', 'service', 'minor'): sort_val += 16 + elif aerialway in ('gondola', 'cable_car'): + sort_val += 19 + elif aerialway in ('chair_lift'): + sort_val += 18 + elif aerialway is not None: + sort_val += 16 else: sort_val += 15 @@ -2275,3 +2289,20 @@ def rank_features( props[rank_key] = count return layer + + +def normalize_aerialways(shape, props, fid, zoom): + aerialway = props.get('aerialway') + + # normalise cableway, apparently a deprecated + # value. + if aerialway == 'cableway': + props['aerialway'] = 'zip_line' + + # 'yes' is a pretty unhelpful value, so normalise + # to a slightly more meaningful 'unknown', which + # is also a commonly-used value. + if aerialway == 'yes': + props['aerialway'] = 'unknown' + + return shape, props, fid From 711a785d7590d43fdb3fc0edd321df36ea41b3c0 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 27 Oct 2015 11:42:43 +0000 Subject: [PATCH 226/344] Filter out features with empty subway_lines, as this is misleading. --- TileStache/Goodies/VecTiles/transform.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index c3bcfb9d..ac7f88dc 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2112,6 +2112,8 @@ def normalize_and_merge_duplicate_stations( Use the name, now appropriately trimmed, to merge station POIs together, unioning their subway lines. + Stations with empty subway_lines have that property removed. + Finally, re-sort the features in case the merging has caused the subway stations to be out-of-order. """ @@ -2186,6 +2188,16 @@ def normalize_and_merge_duplicate_stations( # de-dup these. new_features.append(feature) + # remove anything that has an empty subway_lines + # list, as this most likely indicates that we were + # not able to _detect_ what lines it's part of, as + # it seems unlikely that a station would be part of + # _zero_ lines. + for shape, props, fid in new_features: + subway_lines = props.pop('subway_lines', []) + if subway_lines: + props['subway_lines'] = subway_lines + # might need to re-sort, if we merged any stations: # removing duplicates would have changed the number # of lines for each station. From 3371be273e57894ffabb8601178c9cb6776ce0be Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 27 Oct 2015 12:22:50 +0000 Subject: [PATCH 227/344] Rename: subway lines -> transit routes. --- TileStache/Goodies/VecTiles/transform.py | 42 ++++++++++++------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index ac7f88dc..188d6a18 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2106,16 +2106,16 @@ def normalize_and_merge_duplicate_stations( """ Normalise station names by removing any parenthetical lines lists at the end (e.g: "Foo St (A, C, E)"). Parse this and - use it to replace the `subway_lines` list if that is empty + use it to replace the `transit_routes` list if that is empty or isn't present. Use the name, now appropriately trimmed, to merge station - POIs together, unioning their subway lines. + POIs together, unioning their transit routes. - Stations with empty subway_lines have that property removed. + Stations with empty transit_routes have that property removed. Finally, re-sort the features in case the merging has caused - the subway stations to be out-of-order. + the station POIs to be out-of-order. """ assert source_layer, 'normalize_and_merge_duplicate_stations: missing source layer' @@ -2148,14 +2148,14 @@ def normalize_and_merge_duplicate_stations( # list of lines if we haven't already got that info. m = station_pattern.match(name) - subway_lines = props.get('subway_lines', []) + transit_routes = props.get('transit_routes', []) if m: # if the lines aren't present or are empty - if not subway_lines: + if not transit_routes: lines = m.group(2).split(',') - subway_lines = [x.strip() for x in lines] - props['subway_lines'] = subway_lines + transit_routes = [x.strip() for x in lines] + props['transit_routes'] = transit_routes # update name so that it doesn't contain all the # lines. @@ -2166,41 +2166,41 @@ def normalize_and_merge_duplicate_stations( if seen_idx is None: seen_stations[name] = len(new_features) - # ensure that subway lines is present and is of + # ensure that transit routes is present and is of # list type for when we append to it later if we # find a duplicate. - props['subway_lines'] = subway_lines + props['transit_routes'] = transit_routes new_features.append(feature) else: # get the properties and append this duplicate's - # subway lines to the list on the original + # transit routes to the list on the original # feature. seen_props = new_features[seen_idx][1] - # make sure lines are unique - unique_subway_lines = set(subway_lines) & \ - set(seen_props['subway_lines']) - seen_props['subway_lines'] = list(unique_subway_lines) + # make sure routes are unique + unique_transit_routes = set(transit_routes) & \ + set(seen_props['transit_routes']) + seen_props['transit_routes'] = list(unique_transit_routes) else: # not a station, or name is missing - we can't # de-dup these. new_features.append(feature) - # remove anything that has an empty subway_lines + # remove anything that has an empty transit_routes # list, as this most likely indicates that we were # not able to _detect_ what lines it's part of, as # it seems unlikely that a station would be part of - # _zero_ lines. + # _zero_ routes. for shape, props, fid in new_features: - subway_lines = props.pop('subway_lines', []) - if subway_lines: - props['subway_lines'] = subway_lines + transit_routes = props.pop('transit_routes', []) + if transit_routes: + props['transit_routes'] = transit_routes # might need to re-sort, if we merged any stations: # removing duplicates would have changed the number - # of lines for each station. + # of routes for each station. if seen_stations: sort_pois(new_features, zoom) From c257cd1398989655067537605b9a3c4978b6d643 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 27 Oct 2015 17:41:25 +0000 Subject: [PATCH 228/344] Add support for dropping certain keys when copying them for label features. --- TileStache/Goodies/VecTiles/transform.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 188d6a18..a5944e47 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1788,7 +1788,7 @@ def admin_boundaries(feature_layers, zoom, base_layer, def generate_label_features( feature_layers, zoom, source_layer=None, label_property_name=None, - label_property_value=None, new_layer_name=None): + label_property_value=None, new_layer_name=None, drop_keys=None): assert source_layer, 'generate_label_features: missing source_layer' @@ -1796,6 +1796,9 @@ def generate_label_features( if layer is None: return None + if drop_keys is None: + drop_keys = [] + new_features = [] for feature in layer['features']: shape, properties, fid = feature @@ -1820,6 +1823,13 @@ def generate_label_features( label_point = shape.representative_point() label_properties = properties.copy() + + # drop particular keys which might not be relevant any more. + # for example, mz_is_building, which is used by a later + # polygon processing stage, but irrelevant to label processing. + for k in drop_keys: + label_properties.pop(k, None) + if label_property_name: label_properties[label_property_name] = label_property_value From adf0f456d41d09af561c760f8454f484f7812042 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 28 Oct 2015 17:24:13 +0000 Subject: [PATCH 229/344] Extracted deduplicator to a class, makes logic easier to read. Allowed the duplicate remover to work across several layers. Requires a bit of abnormal fiddling with the layers to get it to work. --- TileStache/Goodies/VecTiles/transform.py | 162 +++++++++++++++++------ 1 file changed, 118 insertions(+), 44 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a5944e47..de215f34 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2032,20 +2032,85 @@ def remove_zero_area(shape, properties, fid, zoom): _MERCATOR_CIRCUMFERENCE = 40075016.68 +# _Deduplicator handles the logic for deduplication. a feature +# is considered a duplicate if it has the same property tuple +# as another and is within a certain distance of the other. +# +# the property tuple is calculated by taking a tuple or list +# of keys and extracting the value of the matching property +# or None. if none_means_unique is true, then if any tuple +# entry is None the feature is considered unique and kept. +# +# note: distance here is measured in coordinate units; i.e: +# mercator meters! +class _Deduplicator: + def __init__(self, property_keys, min_distance, + none_means_unique): + self.property_keys = property_keys + self.min_distance = min_distance + self.none_means_unique = none_means_unique + self.seen_items = dict() + + def keep_feature(self, feature): + """ + Returns true if the feature isn't a duplicate, and should + be kept in the output. Otherwise, returns false, as + another feature had the same tuple of values. + """ + shape, props, fid = feature + + key = tuple([props.get(k) for k in self.property_keys]) + if self.none_means_unique and any([v is None for v in key]): + return True + + seen_geoms = self.seen_items.get(key) + if seen_geoms is None: + # first time we've seen this item, so keep it in + # the output. + self.seen_items[key] = [shape] + return True + + else: + # if the distance is greater than the minimum set + # for this zoom, then we also keep it. + distance = min([shape.distance(s) for s in seen_geoms]) + + if distance > self.min_distance: + # this feature is far enough away to count as + # distinct, but keep this geom to suppress any + # other labels nearby. + seen_geoms.append(shape) + return True + + else: + # feature is a duplicate + return False + + def remove_duplicate_features( - feature_layers, zoom, source_layer=None, start_zoom=0, - property_keys=None, geometry_types=None, min_distance=0.0): + feature_layers, zoom, source_layer=None, source_layers=None, + start_zoom=0, property_keys=None, geometry_types=None, + min_distance=0.0, none_means_unique=True): """ - Removes duplicate features from the layer. The definition of - duplicate is anything which has the same values for the tuple - of values associated with the property_keys. + Removes duplicate features from a layer, or set of layers. The + definition of duplicate is anything which has the same values + for the tuple of values associated with the property_keys. + + If `none_means_unique` is set, which it is by default, then a + value of None for *any* of the values in the tuple causes the + feature to be considered unique and completely by-passed. This + is mainly to handle things like features missing their name, + where we don't want to remove all but one unnamed feature. For example, if property_keys was ['name', 'kind'], then only the first feature of those with the same value for the name and kind properties would be kept in the output. """ - assert source_layer, 'remove_duplicate_features: missing source layer' + # can use either a single source layer, or multiple source + # layers, but not both. + assert bool(source_layer) ^ bool(source_layers), \ + 'remove_duplicate_features: define either source layer or source layers, but not both' # note that the property keys or geometry types could be empty, # but then this post-process filter would do nothing. so we @@ -2057,9 +2122,14 @@ def remove_duplicate_features( if zoom < start_zoom: return None - layer = _find_layer(feature_layers, source_layer) - if layer is None: - return None + # allow either a single or multiple layers to be used. + if source_layer: + source_layers = [source_layer] + + # correct for zoom: min_distance is given in pixels, but we + # want to do the comparison in coordinate units to avoid + # repeated conversions. + min_distance = min_distance * _MERCATOR_CIRCUMFERENCE / float(1 << (zoom + 8)) # keep a set of the tuple of the property keys. this will tell # us if the feature is unique while allowing us to maintain the @@ -2067,47 +2137,51 @@ def remove_duplicate_features( # features. we keep the geometry of the seen items too, so that # we can tell if any new feature is significantly far enough # away that it should be shown again. - seen_items = dict() - - def meters_to_pixels(distance): - return distance * float(1 << (zoom + 8)) / 40075016.68 - - new_features = [] - for feature in layer['features']: - shape, props, fid = feature - - keep_feature = True - if shape.geom_type in geometry_types: - key = tuple([props.get(k) for k in property_keys]) - seen_geoms = seen_items.get(key) - - if seen_geoms is None: - # first time we've seen this item, so keep it in - # the output. - seen_items[key] = [shape] + deduplicator = _Deduplicator(property_keys, min_distance, + none_means_unique) + + for source_layer in source_layers: + layer_index = -1 + # because this post-processor can potentially modify + # multiple layers, and that wasn't how the return value + # system was designed, instead it modifies layers + # *in-place*. this is abnormal, and as such requires a + # nice big comment like this! + for index, feature_layer in enumerate(feature_layers): + layer_datum = feature_layer['layer_datum'] + layer_name = layer_datum['name'] + if layer_name == source_layer: + layer_index = index + break + + if layer_index < 0: + # TODO: warn about missing layer when we get the + # ability to log. + continue - else: - # if the distance is greater than the minimum set - # for this zoom, then we also keep it. - distance = min([shape.distance(s) for s in seen_geoms]) + layer = feature_layers[layer_index] - # correct for zoom - we want visual distance, which - # means (pseudo) pixels. - distance = meters_to_pixels(distance) + new_features = [] + for feature in layer['features']: + shape, props, fid = feature + keep_feature = True - if distance > min_distance: - # keep this geom to suppress any other labels - # nearby. - seen_geoms.append(shape) + if geometry_types is not None and \ + shape.geom_type in geometry_types: + keep_feature = deduplicator.keep_feature(feature) - else: - keep_feature = False + if keep_feature: + new_features.append(feature) - if keep_feature: - new_features.append(feature) + # NOTE! modifying the layer *in-place*. + layer['features'] = new_features + feature_layers[index] = layer - layer['features'] = new_features - return layer + # returning None here would normally indicate that the + # post-processor has done nothing. but because this + # modifies the layers *in-place* then all the return + # value is superfluous. + return None def normalize_and_merge_duplicate_stations( From 26f0c15c0425c1684bc2a162e404737a40b40f98 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 28 Oct 2015 19:48:53 +0000 Subject: [PATCH 230/344] Implement copy post-processor, which copies matching features to another layer. --- TileStache/Goodies/VecTiles/transform.py | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index de215f34..0a38ef9d 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2466,3 +2466,46 @@ def numeric_min_filter( layer['features'] = new_features return layer + + +def copy_features( + feature_layers, zoom, source_layer=None, target_layer=None, + where=None, geometry_types=None): + """ + Copy features matching _both_ the `where` selection and the + `geometry_types` list to another layer. If the target layer + doesn't exist, it is created. + """ + + assert source_layer, 'copy_features: source layer not configured' + assert target_layer, 'copy_features: target layer not configured' + assert where, 'copy_features: you must specify how to match features in the where parameter' + assert geometry_types, 'copy_features: you must specify at least one type of geometry in geometry_types' + + src_layer = _find_layer(feature_layers, source_layer) + if src_layer is None: + return None + + tgt_layer = _find_layer(feature_layers, target_layer) + if tgt_layer is None: + # create target layer if it doesn't already exist. + tgt_layer_datum = src_layer['layer_datum'].copy() + tgt_layer_datum['name'] = target_layer + tgt_layer = dict( + name=target_layer, + features=[], + layer_datum=tgt_layer_datum, + ) + + new_features = [] + for feature in src_layer['features']: + shape, props, fid = feature + + if _match_props(props, where): + # need to deep copy, otherwise we could have some + # unintended side effects if either layer is + # mutated later on. + new_features.append((shape.copy(), props.copy(), fid)) + + tgt_layer['features'].extend(new_features) + return tgt_layer From 4017f33192ba186846c884a0c0909207e4259c6b Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 30 Oct 2015 12:34:54 +0000 Subject: [PATCH 231/344] Volume should be an int. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 0a38ef9d..0df3a91f 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -208,7 +208,7 @@ def synthesize_volume(shape, props, fid, zoom): area = props.get('area') height = props.get('height') if area is not None and height is not None: - props['volume'] = area * height + props['volume'] = int(area * height) return shape, props, fid From b2f4b646dc5bc00d39788729008ed185f1cac82c Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 30 Oct 2015 15:34:46 +0000 Subject: [PATCH 232/344] Added post-processor function to replace geometry with representative point. This is for use in the POIs layer, where previously we were using a PostGIS-calculated centroid. --- TileStache/Goodies/VecTiles/transform.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 0a38ef9d..b1a68b5c 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2509,3 +2509,17 @@ def copy_features( tgt_layer['features'].extend(new_features) return tgt_layer + + +def make_representative_point(shape, properties, fid, zoom): + """ + Replaces the geometry of each feature with its + representative point. This is a point which should be + within the interior of the geometry, which can be + important for labelling concave or doughnut-shaped + polygons. + """ + + shape = shape.representative_point() + + return shape, properties, fid From 4ca3ebbd241d1d5c3b787d16efbe6baeaa2ee6a8 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 30 Oct 2015 16:43:37 +0000 Subject: [PATCH 233/344] Replaced subway lines with transit routes, but needed to update the sort function too. --- TileStache/Goodies/VecTiles/sort.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index a85d66ea..d29cd825 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -47,20 +47,20 @@ def _sort_by_scalerank_then_population(features): return features -def _by_subway_lines(feature): +def _by_transit_routes(feature): wkb, props, fid = feature num_lines = 0 - subway_lines = props.get('subway_lines') - if subway_lines is not None: - num_lines = len(subway_lines) + transit_routes = props.get('transit_routes') + if transit_routes is not None: + num_lines = len(transit_routes) return num_lines -def _sort_by_subway_lines_then_feature_id(features): +def _sort_by_transit_routes_then_feature_id(features): features.sort(key=_by_feature_id) - features.sort(key=_by_subway_lines, reverse=True) + features.sort(key=_by_transit_routes, reverse=True) return features @@ -81,7 +81,7 @@ def places(features, zoom): def pois(features, zoom): - return _sort_by_subway_lines_then_feature_id(features) + return _sort_by_transit_routes_then_feature_id(features) def roads(features, zoom): From a4667002d360de13b64adc2ec9dfdc9fa15138a6 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 3 Nov 2015 16:46:19 -0500 Subject: [PATCH 234/344] Remove neighbourhood scalerank --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 48e6e2fd..dbf4758b 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -30,6 +30,7 @@ # parenthetical list of line names. station_pattern = re.compile('([^(]*)\(([^)]*)\).*') + def _to_float_meters(x): if x is None: return None @@ -1142,7 +1143,6 @@ def landuse_sort_key(shape, properties, fid, zoom): 'farm': 13, 'hamlet': 12, - 'neighbourhood': 12, 'village': 11, From de368a4591f3bb1263348f8739aaeafe70899ad9 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 3 Nov 2015 16:46:50 -0500 Subject: [PATCH 235/344] Update places sort to include n_photos and area --- TileStache/Goodies/VecTiles/sort.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index d29cd825..5676a142 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -18,9 +18,16 @@ def _feature_sort_by_property(feature): _by_feature_id = _by_feature_property('id') +def _by_area(feature): + wkb, properties, fid = feature + default_value = -1000 + sort_key = properties.get('area', default_value) + return sort_key + + def _sort_by_area_then_id(features): features.sort(key=_by_feature_id) - features.sort(key=_by_feature_property('area'), reverse=True) + features.sort(key=_by_area, reverse=True) return features @@ -41,12 +48,6 @@ def _by_population(feature): return default_value -def _sort_by_scalerank_then_population(features): - features.sort(key=_by_population, reverse=True) - features.sort(key=_by_scalerank) - return features - - def _by_transit_routes(feature): wkb, props, fid = feature @@ -76,8 +77,16 @@ def landuse(features, zoom): return _sort_by_area_then_id(features) +def _place_key_desc(feature): + sort_key = _by_population(feature), _by_area(feature) + return sort_key + + def places(features, zoom): - return _sort_by_scalerank_then_population(features) + features.sort(key=_place_key_desc, reverse=True) + features.sort(key=_by_scalerank) + features.sort(key=_by_feature_property('n_photos'), reverse=True) + return features def pois(features, zoom): From db95c014212b452b02d9b5558a1c7b05f9c636e4 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 3 Nov 2015 16:47:19 -0500 Subject: [PATCH 236/344] Update _match_props to support value lists --- TileStache/Goodies/VecTiles/transform.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index dbf4758b..b3996003 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2297,10 +2297,16 @@ def _match_props(props, items_matching): Checks if all the items in `items_matching` are also present in `props`. If so, returns true. Otherwise returns false. + Each value in `items_matching` can be a list, in which case the + value from `props` must be any one of those values. """ for k, v in items_matching.iteritems(): - if props.get(k) != v: + prop_val = props.get(k) + if isinstance(v, list): + if prop_val not in v: + return False + elif prop_val != v: return False return True From 3b7e8c8191d8ad6994260df037c1a21a5fdf7362 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 4 Nov 2015 18:25:48 +0000 Subject: [PATCH 237/344] Try not to split up roads, or merge them back together if possible. --- TileStache/Goodies/VecTiles/transform.py | 33 ++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index b3996003..ab90dd99 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -816,6 +816,39 @@ def _intersect(self, shape, props, fid, cutting_shape, inside, outside = \ self.intersect_func(shape, cutting_shape) + # intersections are tricky, and it seems that the geos + # library (perhaps only certain versions of it) don't + # handle intersection of a polygon with its boundary + # very well. for example: + # + # >>> import shapely.geometry as g + # >>> p = g.Point(0,0).buffer(1.0, resolution=2) + # >>> b = p.boundary + # >>> b.intersection(p).wkt + # 'MULTILINESTRING ((1 0, 0.7071067811865481 -0.7071067811865469), (0.7071067811865481 -0.7071067811865469, 1.615544574432587e-15 -1), (1.615544574432587e-15 -1, -0.7071067811865459 -0.7071067811865491), (-0.7071067811865459 -0.7071067811865491, -1 -3.231089148865173e-15), (-1 -3.231089148865173e-15, -0.7071067811865505 0.7071067811865446), (-0.7071067811865505 0.7071067811865446, -4.624589118372729e-15 1), (-4.624589118372729e-15 1, 0.7071067811865436 0.7071067811865515), (0.7071067811865436 0.7071067811865515, 1 0))' + # + # the result multilinestring could be joined back into + # the original object. but because it has separate parts, + # each requires duplicating the start and end point, and + # each separate segment gets a different polygon buffer + # in Tangram - basically, it's a problem all round. + # + # two solutions to this: given that we're cutting, then + # the inside and outside should union back to the + # original shape - if either is empty then the whole + # object ought to be in the other. + # + # the second solution, for when there is actually some + # part cut, is that we can attempt to merge lines back + # together. + if outside.is_empty and not inside.is_empty: + inside = shape + elif inside.is_empty and not outside.is_empty: + outside = shape + elif original_geom_dim == _LINE_DIMENSION: + inside = _linemerge(inside) + outside = _linemerge(outside) + if cutting_attr is not None: inside_props = props.copy() inside_props[self.target_attribute] = cutting_attr From fd5f5f138ae31d2858df350216a593f9cf576e80 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 4 Nov 2015 13:46:26 -0500 Subject: [PATCH 238/344] Add transform to drop feature properties --- TileStache/Goodies/VecTiles/transform.py | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index ab90dd99..51a9446c 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2031,6 +2031,32 @@ def drop_features_where( return layer +def drop_properties( + feature_layers, zoom, source_layer=None, start_zoom=0, + properties=None): + """ + Drop all configured properties for features in source_layer + """ + + assert source_layer, 'drop_properties: missing source layer' + assert properties, 'drop_properties: missing properties' + + if zoom < start_zoom: + return None + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + for feature in layer['features']: + shape, f_props, fid = feature + + for prop_to_drop in properties: + f_props.pop(prop_to_drop, None) + + return layer + + def remove_zero_area(shape, properties, fid, zoom): """ All features get a numeric area tag, but for points this From 17a93905049d87f2dc08402b36e1e6f3d1786cf0 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 4 Nov 2015 13:47:01 -0500 Subject: [PATCH 239/344] Update n_photos property name to mz_n_photos --- TileStache/Goodies/VecTiles/sort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index 5676a142..c1d26698 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -85,7 +85,7 @@ def _place_key_desc(feature): def places(features, zoom): features.sort(key=_place_key_desc, reverse=True) features.sort(key=_by_scalerank) - features.sort(key=_by_feature_property('n_photos'), reverse=True) + features.sort(key=_by_feature_property('mz_n_photos'), reverse=True) return features From f39686f4cd92833050c0c1e71a8b87246575bcdf Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 6 Nov 2015 10:04:59 -0500 Subject: [PATCH 240/344] Add sort by `min_zoom` to `places` layer --- TileStache/Goodies/VecTiles/sort.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index c1d26698..ed53bcbf 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -86,6 +86,7 @@ def places(features, zoom): features.sort(key=_place_key_desc, reverse=True) features.sort(key=_by_scalerank) features.sort(key=_by_feature_property('mz_n_photos'), reverse=True) + features.sort(key=_by_feature_property('min_zoom')) return features From f64bf0bd8630c1676001331734f38bb20acc0005 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 13 Nov 2015 12:16:16 -0500 Subject: [PATCH 241/344] Version bump -> 0.5.0 --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20f5b9f3..345cbe7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,26 @@ +v0.5.0 +------ +* Update sorts for `pois` and `places` +* Update `kind` calculation for roads to include `aerialway`s +* Add volume calculation transform +* Update road sort key calculation +* Remove `scalerank` for neighbourhoods +* Allow exterior boundaries to be snapped to a grid +* Improve exterior boundaries processing +* Add post processing functions: + - generate address points from buildings + - drop certain features + - drop certain feature properties + - remove features with zero area + - remove duplicate features + - normalize duplicate stations + - only keep the first N features matching a criteria + - rank features based on a key + - normalize `aerialway`s + - numeric min filter + - copy features across `layers` + - replace geometry with a representative point + v0.4.1 ------ * Make admin boundaries post-processing filter work with boundary linestring fragments rather than needing an oriented polygon. From 13e77b278fada19bbebe816f7a60c99f2f687486 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 20 Nov 2015 12:11:37 -0500 Subject: [PATCH 242/344] Protect against empty snap geometries --- TileStache/Goodies/VecTiles/transform.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 51a9446c..62c571bd 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1433,6 +1433,10 @@ def __init__(self, geom, area): # TODO: log this as a warning! continue + # skip any geometries that may have become empty + if snapped.is_empty: + continue + indexable_features.append((snapped, props, fid)) indexable_shapes.append(geom_with_area(snapped, props.get('area'))) From e1cda82ceb009ea4d0a9bd2065cf9b3acca693a7 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 24 Nov 2015 11:19:02 -0500 Subject: [PATCH 243/344] Version bump -> v0.5.1 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 345cbe7f..4e55ae32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v0.5.1 +------ +* Protect against empty snapped geometries - resolves segfault + v0.5.0 ------ * Update sorts for `pois` and `places` From 7dc9a5d391628e50b84d6013df50c2248b53eede Mon Sep 17 00:00:00 2001 From: "Nathaniel V. KELSO" Date: Tue, 24 Nov 2015 15:49:58 -0800 Subject: [PATCH 244/344] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e55ae32..6ffa7477 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,7 +42,7 @@ v0.3.0 * Add smarts for dealing with maritime boundary attributes * Add tranform for water `tunnel`s -0.0.2 +v0.2.0 ----- * Stable From d4808750b11a3ab0067b2d319b94f41bab3ed9d9 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 30 Nov 2015 17:41:44 -0500 Subject: [PATCH 245/344] Add transform to make population an integer --- TileStache/Goodies/VecTiles/sort.py | 8 +++----- TileStache/Goodies/VecTiles/transform.py | 8 ++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index ed53bcbf..5e04faca 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -41,11 +41,9 @@ def _by_scalerank(feature): def _by_population(feature): wkb, properties, fid = feature default_value = -1000 - population_flt = to_float(properties.get('population')) - if population_flt is not None: - return int(population_flt) - else: - return default_value + # depends on a transform run to convert population to an integer + population = properties.get('population') + return default_value if population is None else population def _by_transit_routes(feature): diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 62c571bd..9ff8c147 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -389,6 +389,14 @@ def place_ne_capital(shape, properties, fid, zoom): return shape, properties, fid +def place_population_int(shape, properties, fid, zoom): + population_str = properties.get('population') + population = to_float(population_str) + if population is not None: + properties['population'] = int(population) + return shape, properties, fid + + def water_tunnel(shape, properties, fid, zoom): tunnel = properties.pop('tunnel', None) if tunnel in (None, 'no', 'false', '0'): From ef91c10d8a300b5fdeb0cf3fb32a50f7cc2286f5 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 1 Dec 2015 09:53:54 -0500 Subject: [PATCH 246/344] Remove population from props if parsing fails --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 9ff8c147..342792da 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -390,7 +390,7 @@ def place_ne_capital(shape, properties, fid, zoom): def place_population_int(shape, properties, fid, zoom): - population_str = properties.get('population') + population_str = properties.pop('population', None) population = to_float(population_str) if population is not None: properties['population'] = int(population) From de107c945754451345f3bdc6986baffdee795c02 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 4 Dec 2015 14:07:30 +0000 Subject: [PATCH 247/344] Added piste as a kind of road and function to remove abandoned pistes. --- TileStache/Goodies/VecTiles/transform.py | 42 ++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 342792da..5dec8dd9 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -127,6 +127,9 @@ def _building_calc_height(height_val, levels_val, levels_calc_fn): 'platter', 't-bar', 'goods', 'magic_carpet', 'rope_tow', 'yes', 'zip_line', 'j-bar', 'unknown', 'mixed_lift', 'canopy', 'cableway')) +# top 10 values for piste:type from taginfo +road_kind_piste = set(('nordic', 'downhill', 'sleigh', 'skitour', 'hike', + 'sled', 'yes', 'snow_park', 'playground', 'ski_jump')) def _road_kind(properties): @@ -135,6 +138,9 @@ def _road_kind(properties): return 'highway' if highway in road_kind_major_road: return 'major_road' + piste_type = properties.get('piste_type') + if piste_type in road_kind_piste: + return 'piste' if highway in road_kind_path: return 'path' railway = properties.get('railway') @@ -2600,3 +2606,39 @@ def make_representative_point(shape, properties, fid, zoom): shape = shape.representative_point() return shape, properties, fid + + +def remove_abandoned_pistes( + feature_layers, zoom, source_layer=None, start_zoom=0): + """ + Removes features tagged as abandoned pistes. + + It checks the kind, because it doesn't matter if the piste is abandoned if + the kind was detected as a road or track. It also checks the value, as it + appears that 'piste:abandoned = no' accounts for 30% of the instances. + + Finally, the piste_abandoned property is removed from the feature, as we + have filtered out all the 'yes' values, meaning that it conveys no useful + information any more. + """ + + assert source_layer, 'remove_abandoned_pistes: missing source layer' + + if zoom < start_zoom: + return None + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + new_features = [] + for feature in layer['features']: + shape, props, fid = feature + + piste_abandoned = props.pop('piste_abandoned') + kind = props.get('kind') + if piste_abandoned != 'yes' or kind != 'piste': + new_features.append(feature) + + layer['features'] = new_features + return layer From b03a9d9ee34108a743cd259d33f79332cd40b982 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 4 Dec 2015 18:43:42 +0000 Subject: [PATCH 248/344] Add default for pop. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 5dec8dd9..efec5781 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2635,7 +2635,7 @@ def remove_abandoned_pistes( for feature in layer['features']: shape, props, fid = feature - piste_abandoned = props.pop('piste_abandoned') + piste_abandoned = props.pop('piste_abandoned', None) kind = props.get('kind') if piste_abandoned != 'yes' or kind != 'piste': new_features.append(feature) From f43ce4a38ba4d6bd18895f66c885f5cf0d572702 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 7 Dec 2015 19:33:21 +0000 Subject: [PATCH 249/344] Give exit nodes the kind 'exit', as that's a bit easier to understand than a point with kind 'minor_road'. --- TileStache/Goodies/VecTiles/transform.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index efec5781..bb81e100 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -152,6 +152,8 @@ def _road_kind(properties): aerialway = properties.get('aerialway') if aerialway in road_kind_aerialway: return 'aerialway' + if highway == 'motorway_junction': + return 'exit' return 'minor_road' From c2e2cb0eec9ca8bd92aa1dce5bdae0e3395ebfb1 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 9 Dec 2015 17:12:05 +0000 Subject: [PATCH 250/344] Add racetracks as a kind of roads. --- TileStache/Goodies/VecTiles/transform.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index bb81e100..e0af9781 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -154,6 +154,11 @@ def _road_kind(properties): return 'aerialway' if highway == 'motorway_junction': return 'exit' + leisure = properties.get('leisure') + if leisure == 'track': + # note: racetrack rather than track, as track might be confusing + # between a track for racing and a track as in a faint trail. + return 'racetrack' return 'minor_road' From 746fcf012b795dea15d4e9250e510b01026868b6 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 10 Dec 2015 16:34:26 +0000 Subject: [PATCH 251/344] Added function to grab the IATA code, if any, from the tags and put it on the feature. --- TileStache/Goodies/VecTiles/transform.py | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index e0af9781..72743785 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -30,6 +30,12 @@ # parenthetical list of line names. station_pattern = re.compile('([^(]*)\(([^)]*)\).*') +# used to detect if an airport's IATA code is the "short" +# 3-character type. there are also longer codes, and ones +# which include numbers, but those seem to be used for +# less important airports. +iata_short_code_pattern = re.compile('^[A-Z]{3}$') + def _to_float_meters(x): if x is None: @@ -2649,3 +2655,32 @@ def remove_abandoned_pistes( layer['features'] = new_features return layer + + +def add_iata_code_to_airports(shape, properties, fid, zoom): + """ + If the feature is an airport, and it has a 3-character + IATA code in its tags, then move that code to its + properties. + """ + + kind = properties.get('kind') + if kind not in ('aerodrome', 'airport'): + return shape, properties, fid + + tags = properties.get('tags') + if not tags: + return shape, properties, fid + + iata_code = tags.get('iata') + if not iata_code: + return shape, properties, fid + + # IATA codes should be uppercase, and most are, but there + # might be some in lowercase, so just normalise to upper + # here. + iata_code = iata_code.upper() + if iata_short_code_pattern.match(iata_code): + properties['iata'] = iata_code + + return shape, properties, fid From c2cbe3cbcd1e0d57a8a1f49b335c2cc92cf31570 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 11 Dec 2015 17:05:01 +0000 Subject: [PATCH 252/344] Return `kind:path` for piers. --- TileStache/Goodies/VecTiles/transform.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 72743785..4cde152c 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -165,6 +165,9 @@ def _road_kind(properties): # note: racetrack rather than track, as track might be confusing # between a track for racing and a track as in a faint trail. return 'racetrack' + man_made = properties.get('man_made') + if man_made == 'pier': + return 'path' return 'minor_road' From af41fdcd39f4acc0820fc2f6d49212096d97856e Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 14 Dec 2015 13:23:13 +0000 Subject: [PATCH 253/344] Added explicit beach and winter sports landuse area sort orders. --- TileStache/Goodies/VecTiles/transform.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4cde152c..fdb699a7 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1150,6 +1150,7 @@ def remap_deprecated_landuse_kinds(shape, properties, fid, zoom): _landuse_sort_order = { 'aerodrome': 4, 'apron': 5, + 'beach': 4, 'cemetery': 4, 'commercial': 4, 'conservation': 2, @@ -1177,6 +1178,7 @@ def remap_deprecated_landuse_kinds(shape, properties, fid, zoom): 'substation': 4, 'university': 4, 'urban': 1, + 'winter_sports': 4, 'zoo': 4 } From a6b7a100827d7a69bce89b7b360b16b3998d7429 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 14 Dec 2015 14:29:10 +0000 Subject: [PATCH 254/344] Fixed bug: we default aerialway to the blank string, so it will always be `not None`. Further, testing `in` a single tuple doesn't make much sense, and that isn't how it's spelled anyway (should be `in ('chair_lift',)`. --- TileStache/Goodies/VecTiles/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4cde152c..a4d013fb 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -306,9 +306,9 @@ def road_sort_key(shape, properties, fid, zoom): sort_val += 16 elif aerialway in ('gondola', 'cable_car'): sort_val += 19 - elif aerialway in ('chair_lift'): + elif aerialway == 'chair_lift': sort_val += 18 - elif aerialway is not None: + elif aerialway != '': sort_val += 16 else: sort_val += 15 From 5f4101535fa33d035d6130ee66f8a15416043acc Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 14 Dec 2015 14:31:37 +0000 Subject: [PATCH 255/344] Keep aerialways above other linear features. --- TileStache/Goodies/VecTiles/transform.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a4d013fb..9a9b7749 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -305,11 +305,11 @@ def road_sort_key(shape, properties, fid, zoom): elif highway in ('unclassified', 'service', 'minor'): sort_val += 16 elif aerialway in ('gondola', 'cable_car'): - sort_val += 19 + sort_val += 27 elif aerialway == 'chair_lift': - sort_val += 18 + sort_val += 26 elif aerialway != '': - sort_val += 16 + sort_val += 25 else: sort_val += 15 @@ -337,6 +337,11 @@ def road_sort_key(shape, properties, fid, zoom): (railway == 'subway' and tunnel not in ('no', 'false'))): sort_val -= 10 + # Keep aerialways above (almost) everything else, including bridges, + # but make sure it doesn't go beyond the 0-39 range. + if aerialway != '': + sort_val = min(39, sort_val + 10) + # Explicit layer is clipped to [-5, 5] range. Note that # the layer, if present, will be a Float due to the # parse_layer_as_float filter. From 500ec2073ef697bb60ecb40ca3e052b5fe0c48d3 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 14 Dec 2015 14:33:54 +0000 Subject: [PATCH 256/344] Don't encroach on the reserved 'layer' range of orders. --- TileStache/Goodies/VecTiles/transform.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 9a9b7749..0d473076 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -338,9 +338,10 @@ def road_sort_key(shape, properties, fid, zoom): sort_val -= 10 # Keep aerialways above (almost) everything else, including bridges, - # but make sure it doesn't go beyond the 0-39 range. + # but make sure it doesn't go beyond the 0-34 range. (still need to + # leave space for explicit layer). if aerialway != '': - sort_val = min(39, sort_val + 10) + sort_val = min(34, sort_val + 10) # Explicit layer is clipped to [-5, 5] range. Note that # the layer, if present, will be a Float due to the From c0424ce4e81f0bcbd3fc570c509f0a89fcc69b21 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 14 Dec 2015 18:21:59 +0000 Subject: [PATCH 257/344] Make winter sports areas show underneath forests/parks. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 1d10cc1f..7347006f 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1184,7 +1184,7 @@ def remap_deprecated_landuse_kinds(shape, properties, fid, zoom): 'substation': 4, 'university': 4, 'urban': 1, - 'winter_sports': 4, + 'winter_sports': 2, 'zoo': 4 } From 3c12101cf975cea349769abfa31a1761cd7644cc Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 15 Dec 2015 18:07:30 -0500 Subject: [PATCH 258/344] Only include affirmative calculated road props --- TileStache/Goodies/VecTiles/transform.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 7347006f..a27fee68 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -259,15 +259,21 @@ def road_classifier(shape, properties, fid, zoom): if source == 'naturalearthdata.com': return shape, properties, fid - highway = properties.get('highway') - tunnel = properties.get('tunnel') - bridge = properties.get('bridge') - is_link = 'yes' if highway and highway.endswith('_link') else 'no' - is_tunnel = 'yes' if tunnel and tunnel in ('yes', 'true') else 'no' - is_bridge = 'yes' if bridge and bridge in ('yes', 'true') else 'no' - properties['is_link'] = is_link - properties['is_tunnel'] = is_tunnel - properties['is_bridge'] = is_bridge + properties.pop('is_link', None) + properties.pop('is_tunnel', None) + properties.pop('is_bridge', None) + + highway = properties.get('highway', '') + tunnel = properties.get('tunnel', '') + bridge = properties.get('bridge', '') + + if highway.endswith('_link'): + properties['is_link'] = 'yes' + if tunnel in ('yes', 'true'): + properties['is_tunnel'] = 'yes' + if bridge in ('yes', 'true'): + properties['is_bridge'] = 'yes' + return shape, properties, fid From 21c2888caaeeff30326ec0199372da72c6eb53e1 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 16 Dec 2015 17:33:54 +0000 Subject: [PATCH 259/344] Updated changelog for v0.6.0. --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ffa7477..ffdf8930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +v0.6.0 +------ +* Ensure that the `population` property, if present, is an integer. +* Add IATA short (3-character) codes to airports. +* Interpret road kinds for pistes, motorway junctions, racetracks and piers. +* Normalise `is_link`, `is_tunnel` and `is_bridge` so that it is not present in the negative; it should only be present when its value is positive. +* Re-order aerialways to be above all road types, including bridges, unless overriden by a `layer` property. +* Added sort order properties for beaches and winter sports areas. +* Added a function to remove abandoned pistes from the output. + v0.5.1 ------ * Protect against empty snapped geometries - resolves segfault From 097b429a09603c8b5fc4baa62b83c4a551caae41 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 21 Dec 2015 17:43:15 -0500 Subject: [PATCH 260/344] Add transform to include aeroway tag for kind=gate --- TileStache/Goodies/VecTiles/transform.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a27fee68..aef56a09 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -243,6 +243,16 @@ def building_trim_properties(shape, properties, fid, zoom): return shape, properties, fid +def pois_kind_aeroway_gate(shape, properties, fid, zoom): + aeroway = properties.pop('aeroway', None) + if aeroway is None: + return shape, properties, fid + kind = properties.get('kind') + if kind == 'gate' and aeroway: + properties['aeroway'] = aeroway + return shape, properties, fid + + def road_kind(shape, properties, fid, zoom): source = properties.get('source') assert source, 'Missing source in road query' From e0c58666d043e6d7512e3a1b6e6d6fe5bd997119 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 5 Jan 2016 15:01:17 +0000 Subject: [PATCH 261/344] Add code to normalise leisure kinds for fitness-related POIs. --- TileStache/Goodies/VecTiles/transform.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a27fee68..bc9d36d1 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2701,3 +2701,22 @@ def add_iata_code_to_airports(shape, properties, fid, zoom): properties['iata'] = iata_code return shape, properties, fid + + +def normalize_leisure_kind(shape, properties, fid, zoom): + """ + Normalise the various ways of representing fitness POIs to a + single kind=fitness. + """ + + kind = properties.get('kind') + if kind in ('fitness_centre', 'gym'): + properties['kind'] = 'fitness' + + elif kind == 'sports_centre': + sport = properties.get('sport') + if sport in ('fitness', 'gym'): + properties.pop('sport') + properties['kind'] = 'fitness' + + return shape, properties, fid From 0a96ab8fad2384919f653b960593bdbe7b70fe79 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 5 Jan 2016 12:07:33 +0000 Subject: [PATCH 262/344] Implement merging for linear features. This also includes an ad-hoc implementation of feature dropping for roads, which should be moved to a separate function and made configurable for use in `queries.yaml`. --- TileStache/Goodies/VecTiles/transform.py | 152 +++++++++++++++++++++-- 1 file changed, 144 insertions(+), 8 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index a27fee68..a924b752 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -17,6 +17,7 @@ from util import to_float from sort import pois as sort_pois import re +import sys feet_pattern = re.compile('([+-]?[0-9.]+)\'(?: *([+-]?[0-9.]+)")?') @@ -371,6 +372,35 @@ def road_sort_key(shape, properties, fid, zoom): def road_trim_properties(shape, properties, fid, zoom): properties = _remove_properties(properties, 'bridge', 'layer', 'tunnel') + + kind = properties.get('kind') + props_to_drop = [] + + if kind == 'path': + if zoom < 15: + props_to_drop.extend(['is_bridge', 'is_tunnel', 'oneway']) + if zoom < 17: + props_to_drop.extend(['name', 'ref', 'network']) + + elif kind == 'minor_road': + if zoom < 13: + props_to_drop.extend(['is_bridge', 'is_tunnel', 'oneway']) + if zoom < 15: + props_to_drop.extend(['name', 'ref', 'network']) + + elif kind == 'major_road': + if zoom < 13: + props_to_drop.extend(['is_bridge', 'is_tunnel', 'oneway']) + if zoom < 12: + props_to_drop.extend(['name', 'ref', 'network']) + + elif kind == 'highway': + if zoom < 13: + props_to_drop.extend(['is_bridge', 'is_tunnel', 'oneway']) + if zoom < 7: + props_to_drop.extend(['name', 'ref', 'network']) + + properties = _remove_properties(properties, *props_to_drop) return shape, properties, fid @@ -2079,19 +2109,23 @@ def drop_features_where( return layer -def drop_properties( - feature_layers, zoom, source_layer=None, start_zoom=0, - properties=None): +def _project_properties( + feature_layers, zoom, property_func, source_layer=None, start_zoom=0, + end_zoom=None): """ - Drop all configured properties for features in source_layer + Project properties down to a subset of the existing properties based on a + predicate `property_func` which returns true when the property should be + kept. """ - assert source_layer, 'drop_properties: missing source layer' - assert properties, 'drop_properties: missing properties' + assert source_layer, '_project_properties: missing source layer' if zoom < start_zoom: return None + if end_zoom is not None and zoom > end_zoom: + return None + layer = _find_layer(feature_layers, source_layer) if layer is None: return None @@ -2099,12 +2133,45 @@ def drop_properties( for feature in layer['features']: shape, f_props, fid = feature - for prop_to_drop in properties: - f_props.pop(prop_to_drop, None) + for p in f_props: + if not property_func(p): + f_props.pop(p) return layer +def drop_properties( + feature_layers, zoom, source_layer=None, start_zoom=0, + properties=None, end_zoom=None): + """ + Drop all configured properties for features in source_layer + """ + + assert properties, 'drop_properties: missing properties' + + def keep_property(p): + return p not in properties + + return _project_properties(feature_layers, zoom, keep_property, + source_layer, start_zoom, end_zoom) + + +def keep_properties( + feature_layers, zoom, source_layer=None, start_zoom=0, + properties=None, end_zoom=None): + """ + Keep only configured properties for features in source_layer + """ + + assert properties, 'keep_properties: missing properties' + + def keep_property(p): + return p in properties + + return _project_properties(feature_layers, zoom, keep_property, + source_layer, start_zoom, end_zoom) + + def remove_zero_area(shape, properties, fid, zoom): """ All features get a numeric area tag, but for points this @@ -2701,3 +2768,72 @@ def add_iata_code_to_airports(shape, properties, fid, zoom): properties['iata'] = iata_code return shape, properties, fid + + +def merge_features( + feature_layers, zoom, source_layer=None, start_zoom=0, + end_zoom=None): + """ + Merge (linear) features with the same properties together, attempting to + make the resulting geometry as large as possible. Note that this will + remove the IDs from any merged features. + + At the moment, only merging for linear features is implemented, although + it would be possible to extend to other geometry types. + """ + + assert source_layer, 'merge_features: missing source layer' + + if zoom < start_zoom: + return None + + if end_zoom is not None and zoom > end_zoom: + return None + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + # A dictionary mapping the properties of a feature to a tuple of the feature + # IDs and a list of shapes. When we merge the features, they will lose their + # individual IDs, so only keep the first. + # + # Note that, because dicts are mutable and therefore not hashable, we have + # to transform their items into a frozenset instead. + features_by_property = {} + + # A list of all the features that we can't currently merge (at this time; + # points and polygons) which will be skipped by this procedure. + skipped_features = [] + + for shape, props, fid in layer['features']: + dims = _geom_dimensions(shape) + p_id = props.pop('id', None) + frozen_props = frozenset(props.items()) + + if dims != _LINE_DIMENSION: + skipped_features.append((shape, props, fid)) + + elif frozen_props in features_by_property: + features_by_property[frozen_props][2].append(shape) + + else: + features_by_property[frozen_props] = (fid, p_id, [shape]) + + new_features = [] + for frozen_props, (fid, p_id, shapes) in features_by_property.iteritems(): + # we only have lines, so _linemerge is the best we can attempt. + #print>>sys.stderr, "SHAPES: %r" % shapes + s = [] + for s2 in shapes: + s.extend(_flatten_geoms(s2)) + multi = MultiLineString(s) + props = dict(frozen_props) + if p_id is not None: + props['id'] = p_id + new_features.append((_linemerge(multi), props, fid)) + + new_features.extend(skipped_features) + layer['features'] = new_features + + return layer From 695b056e858406f778c1f47ae732fd4687b6b919 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 6 Jan 2016 19:01:18 +0000 Subject: [PATCH 263/344] Rewrite for clarity and move the drop-features functionality to config file. --- TileStache/Goodies/VecTiles/transform.py | 110 +++++++++++------------ 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 3dcecc14..c2568c26 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -17,7 +17,6 @@ from util import to_float from sort import pois as sort_pois import re -import sys feet_pattern = re.compile('([+-]?[0-9.]+)\'(?: *([+-]?[0-9.]+)")?') @@ -382,35 +381,6 @@ def road_sort_key(shape, properties, fid, zoom): def road_trim_properties(shape, properties, fid, zoom): properties = _remove_properties(properties, 'bridge', 'layer', 'tunnel') - - kind = properties.get('kind') - props_to_drop = [] - - if kind == 'path': - if zoom < 15: - props_to_drop.extend(['is_bridge', 'is_tunnel', 'oneway']) - if zoom < 17: - props_to_drop.extend(['name', 'ref', 'network']) - - elif kind == 'minor_road': - if zoom < 13: - props_to_drop.extend(['is_bridge', 'is_tunnel', 'oneway']) - if zoom < 15: - props_to_drop.extend(['name', 'ref', 'network']) - - elif kind == 'major_road': - if zoom < 13: - props_to_drop.extend(['is_bridge', 'is_tunnel', 'oneway']) - if zoom < 12: - props_to_drop.extend(['name', 'ref', 'network']) - - elif kind == 'highway': - if zoom < 13: - props_to_drop.extend(['is_bridge', 'is_tunnel', 'oneway']) - if zoom < 7: - props_to_drop.extend(['name', 'ref', 'network']) - - properties = _remove_properties(properties, *props_to_drop) return shape, properties, fid @@ -2120,12 +2090,13 @@ def drop_features_where( def _project_properties( - feature_layers, zoom, property_func, source_layer=None, start_zoom=0, + feature_layers, zoom, where, action, source_layer=None, start_zoom=0, end_zoom=None): """ Project properties down to a subset of the existing properties based on a - predicate `property_func` which returns true when the property should be - kept. + predicate `where` which returns true when the function `action` should be + performed. The value returned from `action` replaces the properties of the + feature. """ assert source_layer, '_project_properties: missing source layer' @@ -2140,43 +2111,62 @@ def _project_properties( if layer is None: return None + if where is not None: + where = compile(where, 'queries.yaml', 'eval') + + new_features = [] for feature in layer['features']: - shape, f_props, fid = feature + shape, props, fid = feature + + # copy params to add a 'zoom' one. would prefer '$zoom', but apparently + # that's not allowed in python syntax. + local = props.copy() + local['zoom'] = zoom - for p in f_props: - if not property_func(p): - f_props.pop(p) + if where is None or eval(where, {}, local): + props = action(props) + new_features.append((shape, props, fid)) + + layer['features'] = new_features return layer def drop_properties( feature_layers, zoom, source_layer=None, start_zoom=0, - properties=None, end_zoom=None): + properties=None, end_zoom=None, where=None): """ Drop all configured properties for features in source_layer """ assert properties, 'drop_properties: missing properties' - def keep_property(p): - return p not in properties + def action(p): + return _remove_properties(p, *properties) - return _project_properties(feature_layers, zoom, keep_property, + return _project_properties(feature_layers, zoom, where, action, source_layer, start_zoom, end_zoom) def keep_properties( feature_layers, zoom, source_layer=None, start_zoom=0, - properties=None, end_zoom=None): + properties=None, end_zoom=None, where=None): """ Keep only configured properties for features in source_layer """ assert properties, 'keep_properties: missing properties' - def keep_property(p): - return p in properties + if where is not None: + where = compile(where, 'queries.yaml', 'eval') + + def keep_property(p, props): + # copy params to add a 'zoom' one. would prefer '$zoom', but apparently + # that's not allowed in python syntax. + local = props.copy() + local['zoom'] = zoom + + return p in properties and (where is None or eval(where, {}, local)) return _project_properties(feature_layers, zoom, keep_property, source_layer, start_zoom, end_zoom) @@ -2823,21 +2813,24 @@ def merge_features( if layer is None: return None - # A dictionary mapping the properties of a feature to a tuple of the feature + # a dictionary mapping the properties of a feature to a tuple of the feature # IDs and a list of shapes. When we merge the features, they will lose their # individual IDs, so only keep the first. - # - # Note that, because dicts are mutable and therefore not hashable, we have - # to transform their items into a frozenset instead. features_by_property = {} - # A list of all the features that we can't currently merge (at this time; + # a list of all the features that we can't currently merge (at this time; # points and polygons) which will be skipped by this procedure. skipped_features = [] for shape, props, fid in layer['features']: dims = _geom_dimensions(shape) + + # keep the 'id' property as well as the feature ID, as these are often + # distinct. p_id = props.pop('id', None) + + # because dicts are mutable and therefore not hashable, we have to + # transform their items into a frozenset instead. frozen_props = frozenset(props.items()) if dims != _LINE_DIMENSION: @@ -2851,15 +2844,22 @@ def merge_features( new_features = [] for frozen_props, (fid, p_id, shapes) in features_by_property.iteritems(): - # we only have lines, so _linemerge is the best we can attempt. - #print>>sys.stderr, "SHAPES: %r" % shapes - s = [] - for s2 in shapes: - s.extend(_flatten_geoms(s2)) - multi = MultiLineString(s) + # we only have lines, so _linemerge is the best we can attempt. however, + # the `shapes` we're operating on may be linestrings, multi-linestrings + # or even empty, so the first thing to do is to flatten them into a + # single geometry. + list_of_linestrings = [] + for shape in shapes: + list_of_linestrings.extend(_flatten_geoms(shape)) + multi = MultiLineString(list_of_linestrings) + + # thaw the frozen properties to use in the new feature. props = dict(frozen_props) + + # restore any 'id' property. if p_id is not None: props['id'] = p_id + new_features.append((_linemerge(multi), props, fid)) new_features.extend(skipped_features) From ed3d8c6109f5892ce6514e443f82f65a2d73090b Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 19 Jan 2016 11:38:55 +0000 Subject: [PATCH 264/344] Add function to normalise tourism kind and related properties. --- TileStache/Goodies/VecTiles/transform.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index c2568c26..92b152d7 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2866,3 +2866,27 @@ def merge_features( layer['features'] = new_features return layer + + +def normalize_tourism_kind(shape, properties, fid, zoom): + """ + There are many tourism-related tags, including 'zoo=*' and 'attraction=*' in + addition to 'tourism=*'. This function promotes things with zoo and + attraction tags have those values as their main kind. + + See https://github.com/mapzen/vector-datasource/issues/440 for more details. + """ + + zoo = properties.pop('zoo', None) + if zoo is not None: + properties['kind'] = zoo + properties['tourism'] = 'attraction' + return (shape, properties, fid) + + attraction = properties.pop('attraction', None) + if attraction is not None: + properties['kind'] = attraction + properties['tourism'] = 'attraction' + return (shape, properties, fid) + + return (shape, properties, fid) From 6e5c14998411c90d9d69e88a1a37e5db25d99781 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 21 Jan 2016 19:49:12 +0000 Subject: [PATCH 265/344] Update changelog for v0.7 release. --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffdf8930..173f5ded 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +v0.7.0 +------ + +* Add function to normalise tourism kind and related properties. +* Add function to drop properties from features under some configurable set of conditions. +* Implement merging for linear features. +* Add code to normalise leisure kinds for fitness-related POIs. +* Add transform to include aeroway tag for `kind=gate`. + v0.6.0 ------ * Ensure that the `population` property, if present, is an integer. From 61f745b3707abc9296adc1c37ca8ff933f3046a4 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 8 Feb 2016 15:59:51 -0500 Subject: [PATCH 266/344] Allow code in drop_features_where function --- TileStache/Goodies/VecTiles/transform.py | 29 +++++++----------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 92b152d7..0c4f1e8f 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2036,8 +2036,7 @@ def parse_layer_as_float(shape, properties, fid, zoom): def drop_features_where( feature_layers, zoom, source_layer=None, start_zoom=0, - property_name=None, drop_property=True, - geom_types=None): + where=None): """ Drops some features entirely when they have a property named `property_name` and its value is true. Note that it @@ -2052,7 +2051,7 @@ def drop_features_where( """ assert source_layer, 'drop_features_where: missing source layer' - assert property_name, 'drop_features_where: missing property name' + assert where, 'drop_features_where: missing where' if zoom < start_zoom: return None @@ -2061,29 +2060,17 @@ def drop_features_where( if layer is None: return None + where = compile(where, 'queries.yaml', 'eval') + new_features = [] for feature in layer['features']: shape, properties, fid = feature - matches_geom_type = \ - geom_types is None or \ - shape.geom_type in geom_types - - # figure out what to do with the property - do we - # want to drop it, or just fetch it? - func = properties.get - if drop_property: - func = properties.pop - - val = func(property_name, None) + local = properties.copy() + local['properties'] = properties - # skip (i.e: drop) the geometry if the value is - # true and it's the geometry type we want. - if val == True and matches_geom_type: - continue - - # default case is to keep the feature - new_features.append((shape, properties, fid)) + if not eval(where, {}, local): + new_features.append(feature) layer['features'] = new_features return layer From 15708d969e1fb4b047f124947a8e2eb74c0eb6a5 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 9 Feb 2016 11:55:58 -0500 Subject: [PATCH 267/344] Update docstring --- TileStache/Goodies/VecTiles/transform.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 0c4f1e8f..2ce1d6e1 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2038,16 +2038,9 @@ def drop_features_where( feature_layers, zoom, source_layer=None, start_zoom=0, where=None): """ - Drops some features entirely when they have a property - named `property_name` and its value is true. Note that it - must be identically True, not just truthy. Also can - drop the property if `drop_property` is truthy. If - `geom_types` is present and not None, then only types in - that list are considered for dropping. - - This is useful for dropping features which we want to use - earlier in the pipeline (e.g: to generate points), but - that we don't want to appear in the final output. + Drop features entirely that match the particular "where" + condition. Any feature properties are available to use, as well as + the properties dict itself, called "properties" in the scope. """ assert source_layer, 'drop_features_where: missing source layer' From 1c91717c780947dc9ee4e93c6b17daf1ef9f47e6 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 19 Feb 2016 20:19:48 +0000 Subject: [PATCH 268/344] Add bounding box clipping to exterior boundaries transform. --- TileStache/Goodies/VecTiles/transform.py | 67 ++++++++++++++++-------- 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 2ce1d6e1..7aa43550 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -10,6 +10,7 @@ from shapely.geometry import LineString from shapely.geometry import LinearRing from shapely.geometry import Polygon +from shapely.geometry import box as Box from shapely.geometry.multipoint import MultiPoint from shapely.geometry.multilinestring import MultiLineString from shapely.geometry.multipolygon import MultiPolygon @@ -1384,13 +1385,24 @@ def _snap_multi(geoms): raise ValueError("_snap_to_grid: unimplemented for shape type %s" % repr(shape_type)) +# returns a geometry which is the given bounds expanded by `factor`. that is, +# if the original shape was a 1x1 box, the new one will be `factor`x`factor` +# box, with the same centroid as the original box. +def _calculate_padded_bounds(factor, bounds): + min_x, min_y, max_x, max_y = bounds + dx = 0.5 * (max_x - min_x) * (factor - 1.0) + dy = 0.5 * (max_y - min_y) * (factor - 1.0) + return Box(min_x - dx, min_y - dy, max_x + dx, max_y + dy) + + def exterior_boundaries(feature_layers, zoom, base_layer, new_layer_name=None, prop_transform=None, buffer_size=None, start_zoom=0, - snap_tolerance=None): + snap_tolerance=None, + bounds=None): """ create new fetures from the boundaries of polygons in the base layer, subtracting any sections of the @@ -1415,6 +1427,10 @@ def exterior_boundaries(feature_layers, zoom, any features in feature_layers[layer] which aren't polygons will be ignored. + + note that the `bounds` kwarg should be filled out + automatically by tilequeue - it does not have to be + provided from the config. """ layer = None @@ -1422,6 +1438,14 @@ def exterior_boundaries(feature_layers, zoom, if zoom < start_zoom: return layer + # check that the bounds parameter was, in fact, passed. + assert bounds is not None, \ + "Automatic bounds parameter should have been passed." + + # make a bounding box 3x larger than the original tile, but with the same + # centroid. + padded_bbox = _calculate_padded_bounds(3, bounds) + # search through all the layers and extract the one # which has the name of the base layer we were given # as a parameter. @@ -1469,27 +1493,28 @@ def __init__(self, geom, area): indexable_shapes = list() for shape, props, fid in features: if shape.geom_type in ('Polygon', 'MultiPolygon'): - snapped = shape + # clip the feature to the padded bounds of the tile + clipped = shape.intersection(padded_bbox) + + snapped = clipped if snap_tolerance is not None: - snapped = _snap_to_grid(shape, snap_tolerance) - - # snapping coordinates might make the shape - # invalid, so we need a way to clean them. - # one simple, but not foolproof, way is to - # buffer them by 0. - if not snapped.is_valid: - snapped = snapped.buffer(0) - - # that still might not have done the trick, - # so drop any polygons which are still - # invalid so as not to cause errors later. - if not snapped.is_valid: - # TODO: log this as a warning! - continue - - # skip any geometries that may have become empty - if snapped.is_empty: - continue + snapped = _snap_to_grid(clipped, snap_tolerance) + + # snapping coordinates and clipping shapes might make the shape + # invalid, so we need a way to clean them. one simple, but not + # foolproof, way is to buffer them by 0. + if not snapped.is_valid: + snapped = snapped.buffer(0) + + # that still might not have done the trick, so drop any polygons + # which are still invalid so as not to cause errors later. + if not snapped.is_valid: + # TODO: log this as a warning! + continue + + # skip any geometries that may have become empty + if snapped.is_empty: + continue indexable_features.append((snapped, props, fid)) indexable_shapes.append(geom_with_area(snapped, props.get('area'))) From be9458a59461f20a7735bf04cc5adb28e754dba1 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 23 Feb 2016 13:07:13 +0000 Subject: [PATCH 269/344] Add normalisation functions for social facilities and medical places. --- TileStache/Goodies/VecTiles/transform.py | 48 ++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 7aa43550..fefc04bc 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2895,3 +2895,51 @@ def normalize_tourism_kind(shape, properties, fid, zoom): return (shape, properties, fid) return (shape, properties, fid) + + +def normalize_social_kind(shape, properties, fid, zoom): + """ + Social facilities have an `amenity=social_facility` tag, but more + information is generally available in the `social_facility=*` tag, so it + is more informative to put that as the `kind`. We keep the old tag as + well, for disambiguation. + + Additionally, we normalise the `social_facility:for` tag, which is a + semi-colon delimited list, to an actual list under the `for` property. + This should make it easier to consume. + """ + + kind = properties.get('kind') + if kind == 'social_facility': + tags = properties.get('tags', {}) + if tags: + social_facility = tags.get('social_facility') + if social_facility: + properties['kind'] = social_facility + # leave the original tag on for disambiguation + properties['social_facility'] = social_facility + + # normalise the for list to an actual list + for_list = tags.get('social_facility:for') + if for_list: + properties['for'] = for_list.split(';') + + return (shape, properties, fid) + + +def normalize_medical_kind(shape, properties, fid, zoom): + """ + Many medical practices, such as doctors and dentists, have a specialty, + which is indicated through the `healthcare:specialty` tag. This is a + semi-colon delimited list, so we expand it to an actual list. + """ + + kind = properties.get('kind') + if kind in ['clinic', 'doctors', 'dentist']: + tags = properties.get('tags', {}) + if tags: + specialty = tags.get('healthcare:specialty') + if specialty: + properties['specialty'] = specialty.split(';') + + return (shape, properties, fid) From ebc0f62cbca39661aba16772b35b619a4de6c873 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 23 Feb 2016 17:11:47 +0000 Subject: [PATCH 270/344] Fix comment indent. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index fefc04bc..51a77243 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2919,7 +2919,7 @@ def normalize_social_kind(shape, properties, fid, zoom): # leave the original tag on for disambiguation properties['social_facility'] = social_facility - # normalise the for list to an actual list + # normalise the 'for' list to an actual list for_list = tags.get('social_facility:for') if for_list: properties['for'] = for_list.split(';') From 3d80ca96c1bdfd41d653b7425f97377f5832a69e Mon Sep 17 00:00:00 2001 From: Nathaniel Kelso Date: Fri, 26 Feb 2016 13:09:13 -0800 Subject: [PATCH 271/344] fix #364 so all landuse has friendly sort_key orders --- TileStache/Goodies/VecTiles/transform.py | 124 +++++++++++++++++------ 1 file changed, 92 insertions(+), 32 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 51a77243..07ad52fe 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1171,38 +1171,98 @@ def remap_deprecated_landuse_kinds(shape, properties, fid, zoom): # explicit order for some kinds of landuse _landuse_sort_order = { - 'aerodrome': 4, - 'apron': 5, - 'beach': 4, - 'cemetery': 4, - 'commercial': 4, - 'conservation': 2, - 'farm': 3, - 'farmland': 3, - 'forest': 3, - 'generator': 3, - 'golf_course': 4, - 'hospital': 4, - 'nature_reserve': 2, - 'park': 2, - 'parking': 4, - 'pedestrian': 4, - 'place_of_worship': 4, - 'plant': 3, - 'playground': 4, - 'railway': 4, - 'recreation_ground': 4, - 'residential': 1, - 'retail': 4, - 'runway': 5, - 'rural': 1, - 'school': 4, - 'stadium': 3, - 'substation': 4, - 'university': 4, - 'urban': 1, - 'winter_sports': 2, - 'zoo': 4 + 'aerodrome': 48, + 'allotments': 82, + 'amusement_ride': 96, + 'animal': 91, + 'apron': 50, + 'aquarium': 45, + 'artwork': 92, + 'attraction': 93, + 'aviary': 71, + 'beach': 86, + 'breakwater': 221, + 'bridge': 225, + 'carousel': 94, + 'cemetery': 65, + 'cinema': 68, + 'college': 31, + 'commercial': 34, + 'common': 55, + 'conservation': 23, + 'cutline': 222, + 'dike': 223, + 'enclosure': 73, + 'farm': 27, + 'farmland': 28, + 'farmyard': 60, + 'footway': 99, + 'forest': 29, + 'fuel': 67, + 'garden': 87, + 'generator': 83, + 'glacier': 11, + 'golf_course': 46, + 'grass': 77, + 'groyne': 222, + 'hanami': 85, + 'hospital': 44, + 'industrial': 22, + 'land': 220, + 'library': 69, + 'maze': 84, + 'meadow': 78, + 'military': 36, + 'national_park': 18, + 'nature_reserve': 24, + 'park or protected land': 16, + 'park': 25, + 'park': 32, + 'parking': 63, + 'pedestrian': 88, + 'petting_zoo': 72, + 'pier': 224, + 'pitch': 90, + 'place_of_worship': 64, + 'plant': 76, + 'playground': 89, + 'prison': 37, + 'protected_area': 17, + 'quarry': 62, + 'railway': 61, + 'recreation_ground': 47, + 'residential': 21, + 'resort': 42, + 'retail': 35, + 'roller_coaster': 74, + 'runway': 51, + 'rural': 14, + 'school': 66, + 'scrub': 79, + 'sports_centre': 53, + 'stadium': 52, + 'substation': 59, + 'summer_toboggan': 75, + 'taxiway': 49, + 'theatre': 70, + 'theme_park': 38, + 'tower': 98, + 'trail_riding_station': 43, + 'university': 30, + 'urban area': 12, + 'urban': 13, + 'village_green': 54, + 'wastewater_plant': 56, + 'water_slide': 95, + 'water_works': 57, + 'wetland': 80, + 'wilderness_hut': 97, + 'wildlife_park': 39, + 'winery': 40, + 'winter_sports': 26, + 'wood': 81, + 'works': 58, + 'zoo': 41 } From a99300ffdd52718abb735d48a30db140bd2d2942 Mon Sep 17 00:00:00 2001 From: Nathaniel Kelso Date: Fri, 26 Feb 2016 13:23:57 -0800 Subject: [PATCH 272/344] set default landuse order to 11, and adjust everything else by +1 --- TileStache/Goodies/VecTiles/transform.py | 183 ++++++++++++----------- 1 file changed, 96 insertions(+), 87 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 07ad52fe..08da9a57 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1171,98 +1171,98 @@ def remap_deprecated_landuse_kinds(shape, properties, fid, zoom): # explicit order for some kinds of landuse _landuse_sort_order = { - 'aerodrome': 48, - 'allotments': 82, - 'amusement_ride': 96, - 'animal': 91, - 'apron': 50, - 'aquarium': 45, - 'artwork': 92, - 'attraction': 93, - 'aviary': 71, - 'beach': 86, + 'aerodrome': 49, + 'allotments': 83, + 'amusement_ride': 97, + 'animal': 92, + 'apron': 51, + 'aquarium': 46, + 'artwork': 93, + 'attraction': 94, + 'aviary': 72, + 'beach': 87, 'breakwater': 221, 'bridge': 225, - 'carousel': 94, - 'cemetery': 65, - 'cinema': 68, - 'college': 31, - 'commercial': 34, - 'common': 55, - 'conservation': 23, + 'carousel': 95, + 'cemetery': 66, + 'cinema': 69, + 'college': 32, + 'commercial': 35, + 'common': 56, + 'conservation': 24, 'cutline': 222, 'dike': 223, - 'enclosure': 73, - 'farm': 27, - 'farmland': 28, - 'farmyard': 60, - 'footway': 99, - 'forest': 29, - 'fuel': 67, - 'garden': 87, - 'generator': 83, - 'glacier': 11, - 'golf_course': 46, - 'grass': 77, + 'enclosure': 74, + 'farm': 28, + 'farmland': 29, + 'farmyard': 61, + 'footway': 100, + 'forest': 30, + 'fuel': 68, + 'garden': 88, + 'generator': 84, + 'glacier': 12, + 'golf_course': 47, + 'grass': 78, 'groyne': 222, - 'hanami': 85, - 'hospital': 44, - 'industrial': 22, + 'hanami': 86, + 'hospital': 45, + 'industrial': 23, 'land': 220, - 'library': 69, - 'maze': 84, - 'meadow': 78, - 'military': 36, - 'national_park': 18, - 'nature_reserve': 24, - 'park or protected land': 16, - 'park': 25, - 'park': 32, - 'parking': 63, - 'pedestrian': 88, - 'petting_zoo': 72, + 'library': 70, + 'maze': 85, + 'meadow': 79, + 'military': 37, + 'national_park': 19, + 'nature_reserve': 25, + 'park or protected land': 17, + 'park': 26, + 'park': 33, + 'parking': 64, + 'pedestrian': 89, + 'petting_zoo': 73, 'pier': 224, - 'pitch': 90, - 'place_of_worship': 64, - 'plant': 76, - 'playground': 89, - 'prison': 37, - 'protected_area': 17, - 'quarry': 62, - 'railway': 61, - 'recreation_ground': 47, - 'residential': 21, - 'resort': 42, - 'retail': 35, - 'roller_coaster': 74, - 'runway': 51, - 'rural': 14, - 'school': 66, - 'scrub': 79, - 'sports_centre': 53, - 'stadium': 52, - 'substation': 59, - 'summer_toboggan': 75, - 'taxiway': 49, - 'theatre': 70, - 'theme_park': 38, - 'tower': 98, - 'trail_riding_station': 43, - 'university': 30, - 'urban area': 12, - 'urban': 13, - 'village_green': 54, - 'wastewater_plant': 56, - 'water_slide': 95, - 'water_works': 57, - 'wetland': 80, - 'wilderness_hut': 97, - 'wildlife_park': 39, - 'winery': 40, - 'winter_sports': 26, - 'wood': 81, - 'works': 58, - 'zoo': 41 + 'pitch': 91, + 'place_of_worship': 65, + 'plant': 77, + 'playground': 90, + 'prison': 38, + 'protected_area': 18, + 'quarry': 63, + 'railway': 62, + 'recreation_ground': 48, + 'residential': 22, + 'resort': 43, + 'retail': 36, + 'roller_coaster': 75, + 'runway': 52, + 'rural': 15, + 'school': 67, + 'scrub': 80, + 'sports_centre': 54, + 'stadium': 53, + 'substation': 60, + 'summer_toboggan': 76, + 'taxiway': 50, + 'theatre': 71, + 'theme_park': 39, + 'tower': 99, + 'trail_riding_station': 44, + 'university': 31, + 'urban area': 13, + 'urban': 14, + 'village_green': 55, + 'wastewater_plant': 57, + 'water_slide': 96, + 'water_works': 58, + 'wetland': 81, + 'wilderness_hut': 98, + 'wildlife_park': 40, + 'winery': 41, + 'winter_sports': 27, + 'wood': 82, + 'works': 59, + 'zoo': 42 } @@ -1273,12 +1273,21 @@ def remap_deprecated_landuse_kinds(shape, properties, fid, zoom): # steps on the client. def landuse_sort_key(shape, properties, fid, zoom): kind = properties.get('kind') - + + # land is at 10 + # default to 11 for landuse if not in the lookup table + # (landuse lookup table starts at 12) + fallback_sort_key = 11 + if kind is not None: key = _landuse_sort_order.get(kind) if key is not None: properties['sort_key'] = key - + else: + properties['sort_key'] = fallback_sort_key + else: + properties['sort_key'] = fallback_sort_key + return shape, properties, fid From f9e81c63f358581e9d2d70a629c3de91774f8a29 Mon Sep 17 00:00:00 2001 From: Nathaniel Kelso Date: Fri, 26 Feb 2016 14:21:21 -0800 Subject: [PATCH 273/344] things above the water should be uniquely above the water --- TileStache/Goodies/VecTiles/transform.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 08da9a57..58bc5391 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1182,7 +1182,7 @@ def remap_deprecated_landuse_kinds(shape, properties, fid, zoom): 'aviary': 72, 'beach': 87, 'breakwater': 221, - 'bridge': 225, + 'bridge': 226, 'carousel': 95, 'cemetery': 66, 'cinema': 69, @@ -1190,8 +1190,8 @@ def remap_deprecated_landuse_kinds(shape, properties, fid, zoom): 'commercial': 35, 'common': 56, 'conservation': 24, - 'cutline': 222, - 'dike': 223, + 'cutline': 223, + 'dike': 224, 'enclosure': 74, 'farm': 28, 'farmland': 29, @@ -1221,7 +1221,7 @@ def remap_deprecated_landuse_kinds(shape, properties, fid, zoom): 'parking': 64, 'pedestrian': 89, 'petting_zoo': 73, - 'pier': 224, + 'pier': 225, 'pitch': 91, 'place_of_worship': 65, 'plant': 77, From 1aa220d4ff7a5ff62684d8f40337918f131512b5 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 26 Feb 2016 18:08:22 -0500 Subject: [PATCH 274/344] Update road sort key values --- TileStache/Goodies/VecTiles/transform.py | 55 +++++++++++------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 58bc5391..af0feb88 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -291,44 +291,45 @@ def road_classifier(shape, properties, fid, zoom): def road_sort_key(shape, properties, fid, zoom): # Note! parse_layer_as_float must be run before this filter. - # Calculated sort value is in the range 0 to 39 - sort_val = 0 + floor = 300 + ceiling = 385 + sort_val = floor - # Base layer range is 15 to 24 highway = properties.get('highway', '') railway = properties.get('railway', '') aeroway = properties.get('aeroway', '') aerialway = properties.get('aerialway', '') service = properties.get('service') - is_railway = railway in ('rail', 'tram', 'light_rail', 'narrow_guage', 'monorail') + is_railway = railway in ( + 'rail', 'tram', 'light_rail', 'narrow_guage', 'monorail') if highway == 'motorway': - sort_val += 24 + sort_val += 44 elif is_railway: - sort_val += 23 + sort_val += 43 elif highway == 'trunk': - sort_val += 22 + sort_val += 42 elif highway == 'primary': - sort_val += 21 + sort_val += 41 elif highway == 'secondary' or aeroway == 'runway': - sort_val += 20 + sort_val += 40 elif highway == 'tertiary' or aeroway == 'taxiway': - sort_val += 19 + sort_val += 39 elif highway.endswith('_link'): - sort_val += 18 + sort_val += 38 elif highway in ('residential', 'unclassified', 'road', 'living_street'): - sort_val += 17 + sort_val += 37 elif highway in ('unclassified', 'service', 'minor'): - sort_val += 16 + sort_val += 36 elif aerialway in ('gondola', 'cable_car'): - sort_val += 27 + sort_val += 47 elif aerialway == 'chair_lift': - sort_val += 26 - elif aerialway != '': - sort_val += 25 + sort_val += 46 + elif aerialway: + sort_val += 45 else: - sort_val += 15 + sort_val += 25 if is_railway and service is not None: if service in ('spur', 'siding'): @@ -345,20 +346,16 @@ def road_sort_key(shape, properties, fid, zoom): sort_val -= 1 if zoom >= 15: - # Bridges and tunnels add +/- 10 bridge = properties.get('bridge') tunnel = properties.get('tunnel') if bridge in ('yes', 'true'): - sort_val += 10 + sort_val += 40 elif (tunnel in ('yes', 'true') or (railway == 'subway' and tunnel not in ('no', 'false'))): sort_val -= 10 - # Keep aerialways above (almost) everything else, including bridges, - # but make sure it doesn't go beyond the 0-34 range. (still need to - # leave space for explicit layer). - if aerialway != '': - sort_val = min(34, sort_val + 10) + if aerialway: + sort_val += 30 # Explicit layer is clipped to [-5, 5] range. Note that # the layer, if present, will be a Float due to the @@ -366,14 +363,10 @@ def road_sort_key(shape, properties, fid, zoom): layer = properties.get('layer') if layer is not None: layer = max(min(layer, 5), -5) - # The range of values from above is [5, 34] - # For positive layer values, we want the range to be: - # [34, 39] if layer > 0: - sort_val = int(layer + 34) - # For negative layer values, [0, 5] + sort_val = int(layer + ceiling - 5) elif layer < 0: - sort_val = int(layer + 5) + sort_val = int(layer + floor + 5) properties['sort_key'] = sort_val From 1b25770c0aea1a6ba11fea0b92428fbb5580be79 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 29 Feb 2016 11:50:47 -0500 Subject: [PATCH 275/344] First round of updates --- TileStache/Goodies/VecTiles/transform.py | 30 ++++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index af0feb88..7e6b7aec 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -292,33 +292,36 @@ def road_sort_key(shape, properties, fid, zoom): # Note! parse_layer_as_float must be run before this filter. floor = 300 - ceiling = 385 + ceiling = 447 sort_val = floor highway = properties.get('highway', '') railway = properties.get('railway', '') aeroway = properties.get('aeroway', '') aerialway = properties.get('aerialway', '') - service = properties.get('service') + service = properties.get('service', '') + ne_type = properties.get('type', '') is_railway = railway in ( 'rail', 'tram', 'light_rail', 'narrow_guage', 'monorail') - if highway == 'motorway': + if (highway == 'motorway' or + ne_type in ('Major Highway', 'Beltway', 'Bypass')): sort_val += 44 elif is_railway: sort_val += 43 - elif highway == 'trunk': + elif highway == 'trunk' or ne_type == 'Secondary Highway': sort_val += 42 - elif highway == 'primary': + elif highway == 'primary' or ne_type == 'Road': sort_val += 41 elif highway == 'secondary' or aeroway == 'runway': sort_val += 40 - elif highway == 'tertiary' or aeroway == 'taxiway': + elif highway == 'tertiary' or aeroway == 'taxiway' or ne_type == 'Track': sort_val += 39 elif highway.endswith('_link'): sort_val += 38 - elif highway in ('residential', 'unclassified', 'road', 'living_street'): + elif (highway in ('residential', 'unclassified', 'road', 'living_street') + or ne_type == 'Unknown'): sort_val += 37 elif highway in ('unclassified', 'service', 'minor'): sort_val += 36 @@ -329,9 +332,9 @@ def road_sort_key(shape, properties, fid, zoom): elif aerialway: sort_val += 45 else: - sort_val += 25 + sort_val += 35 - if is_railway and service is not None: + if is_railway and service: if service in ('spur', 'siding'): # make sort val more like residential, unclassified which # also come in at zoom 12 @@ -341,21 +344,18 @@ def road_sort_key(shape, properties, fid, zoom): else: sort_val -= 8 - if highway == 'service' and service is not None: + if highway == 'service' and service: # sort alley, driveway, etc... under service sort_val -= 1 if zoom >= 15: bridge = properties.get('bridge') tunnel = properties.get('tunnel') - if bridge in ('yes', 'true'): + if bridge in ('yes', 'true') or aerialway: sort_val += 40 elif (tunnel in ('yes', 'true') or (railway == 'subway' and tunnel not in ('no', 'false'))): - sort_val -= 10 - - if aerialway: - sort_val += 30 + sort_val -= 40 # Explicit layer is clipped to [-5, 5] range. Note that # the layer, if present, will be a Float due to the From 2cfbd98940faf9327a736a683bd924a5dee2c384 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 29 Feb 2016 11:54:00 -0500 Subject: [PATCH 276/344] Remove duplicate unclassified check --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 7e6b7aec..d3065446 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -323,7 +323,7 @@ def road_sort_key(shape, properties, fid, zoom): elif (highway in ('residential', 'unclassified', 'road', 'living_street') or ne_type == 'Unknown'): sort_val += 37 - elif highway in ('unclassified', 'service', 'minor'): + elif highway in ('service', 'minor'): sort_val += 36 elif aerialway in ('gondola', 'cable_car'): sort_val += 47 From 102281336a95d7ee45dba6182df3441525265179 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 29 Feb 2016 14:18:35 -0500 Subject: [PATCH 277/344] Updates round 2 --- TileStache/Goodies/VecTiles/transform.py | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index d3065446..ac27e1f4 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -307,55 +307,55 @@ def road_sort_key(shape, properties, fid, zoom): if (highway == 'motorway' or ne_type in ('Major Highway', 'Beltway', 'Bypass')): - sort_val += 44 + sort_val += 81 elif is_railway: - sort_val += 43 + sort_val += 80 elif highway == 'trunk' or ne_type == 'Secondary Highway': - sort_val += 42 + sort_val += 79 elif highway == 'primary' or ne_type == 'Road': - sort_val += 41 + sort_val += 78 elif highway == 'secondary' or aeroway == 'runway': - sort_val += 40 + sort_val += 77 elif highway == 'tertiary' or aeroway == 'taxiway' or ne_type == 'Track': - sort_val += 39 + sort_val += 76 elif highway.endswith('_link'): - sort_val += 38 + sort_val += 75 elif (highway in ('residential', 'unclassified', 'road', 'living_street') or ne_type == 'Unknown'): - sort_val += 37 + sort_val += 60 elif highway in ('service', 'minor'): - sort_val += 36 + sort_val += 58 elif aerialway in ('gondola', 'cable_car'): - sort_val += 47 + sort_val += 92 elif aerialway == 'chair_lift': - sort_val += 46 + sort_val += 91 elif aerialway: - sort_val += 45 + sort_val += 90 else: - sort_val += 35 + sort_val += 55 if is_railway and service: if service in ('spur', 'siding'): # make sort val more like residential, unclassified which # also come in at zoom 12 - sort_val -= 6 + sort_val -= 19 elif service == 'yard': - sort_val -= 7 + sort_val -= 21 else: - sort_val -= 8 + sort_val -= 23 if highway == 'service' and service: # sort alley, driveway, etc... under service - sort_val -= 1 + sort_val -= 2 if zoom >= 15: bridge = properties.get('bridge') tunnel = properties.get('tunnel') if bridge in ('yes', 'true') or aerialway: - sort_val += 40 + sort_val += 50 elif (tunnel in ('yes', 'true') or (railway == 'subway' and tunnel not in ('no', 'false'))): - sort_val -= 40 + sort_val -= 50 # Explicit layer is clipped to [-5, 5] range. Note that # the layer, if present, will be a Float due to the From 425df4410f9253a683957bce662b39031d5706c7 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 1 Mar 2016 17:20:48 +0000 Subject: [PATCH 278/344] Use context object for parameters. --- TileStache/Goodies/VecTiles/transform.py | 218 ++++++++++++++++------- 1 file changed, 153 insertions(+), 65 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 58bc5391..17148d60 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1053,10 +1053,18 @@ def _intercut_impl(intersect_func, feature_layers, # # returns a feature layer which is the base layer cut by the # cutting layer. -def intercut(feature_layers, zoom, base_layer, cutting_layer, - attribute, target_attribute=None, - cutting_attrs=None, - keep_geom_type=True): +def intercut(ctx): + + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + base_layer = ctx.params.get('base_layer') + assert base_layer, \ + 'Parameter base_layer was missing from intercut config' + cutting_layer = ctx.params.get('cutting_layer') + assert cutting_layer, \ + 'Parameter cutting_layer was missing from intercut ' \ + 'config' + attribute = ctx.params.get('attribute') # sanity check on the availability of the cutting # attribute. assert attribute is not None, \ @@ -1064,6 +1072,11 @@ def intercut(feature_layers, zoom, base_layer, cutting_layer, 'should have been an attribute name. Perhaps check ' + \ 'your configuration file and queries.' + + target_attribute = ctx.params.get('target_attribute') + cutting_attrs = ctx.params.get('cutting_attrs') + keep_geom_type = ctx.params.get('keep_geom_type', True) + return _intercut_impl(_intersect_cut, feature_layers, base_layer, cutting_layer, attribute, target_attribute, cutting_attrs, keep_geom_type) @@ -1084,11 +1097,18 @@ def intercut(feature_layers, zoom, base_layer, cutting_layer, # returns a feature layer which is the base layer with # overlapping features having attributes projected from the # cutting layer. -def overlap(feature_layers, zoom, base_layer, cutting_layer, - attribute, target_attribute=None, - cutting_attrs=None, - keep_geom_type=True, - min_fraction=0.8): +def overlap(ctx): + + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + base_layer = ctx.params.get('base_layer') + assert base_layer, \ + 'Parameter base_layer was missing from overlap config' + cutting_layer = ctx.params.get('cutting_layer') + assert cutting_layer, \ + 'Parameter cutting_layer was missing from overlap ' \ + 'config' + attribute = ctx.params.get('attribute') # sanity check on the availability of the cutting # attribute. assert attribute is not None, \ @@ -1096,6 +1116,11 @@ def overlap(feature_layers, zoom, base_layer, cutting_layer, 'should have been an attribute name. Perhaps check ' + \ 'your configuration file and queries.' + target_attribute = ctx.params.get('target_attribute') + cutting_attrs = ctx.params.get('cutting_attrs') + keep_geom_type = ctx.params.get('keep_geom_type', True) + min_fraction = ctx.params.get('min_fraction', 0.8) + return _intercut_impl(_intersect_overlap(min_fraction), feature_layers, base_layer, cutting_layer, attribute, target_attribute, cutting_attrs, keep_geom_type) @@ -1109,7 +1134,14 @@ def overlap(feature_layers, zoom, base_layer, cutting_layer, # where the `maritime=yes` tag is set. we don't actually want # separate linestrings, we just want the `maritime=yes` attribute # on the first set of linestrings. -def intracut(feature_layers, zoom, base_layer, attribute): +def intracut(ctx): + + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + base_layer = ctx.params.get('base_layer') + assert base_layer, \ + 'Parameter base_layer was missing from intracut config' + attribute = ctx.params.get('attribute') # sanity check on the availability of the cutting # attribute. assert attribute is not None, \ @@ -1464,14 +1496,7 @@ def _calculate_padded_bounds(factor, bounds): return Box(min_x - dx, min_y - dy, max_x + dx, max_y + dy) -def exterior_boundaries(feature_layers, zoom, - base_layer, - new_layer_name=None, - prop_transform=None, - buffer_size=None, - start_zoom=0, - snap_tolerance=None, - bounds=None): +def exterior_boundaries(ctx): """ create new fetures from the boundaries of polygons in the base layer, subtracting any sections of the @@ -1501,6 +1526,18 @@ def exterior_boundaries(feature_layers, zoom, automatically by tilequeue - it does not have to be provided from the config. """ + + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + base_layer = ctx.params.get('base_layer') + assert base_layer, 'Missing base_layer parameter' + new_layer_name = ctx.params.get('new_layer_name') + prop_transform = ctx.params.get('prop_transform') + buffer_size = ctx.params.get('buffer_size') + start_zoom = ctx.params.get('start_zoom', 0) + snap_tolerance = ctx.params.get('snap_tolerance') + bounds = ctx.unpadded_bounds + layer = None # don't start processing until the start zoom @@ -1846,8 +1883,7 @@ def oriented_multi(kind, geom): return geom -def admin_boundaries(feature_layers, zoom, base_layer, - start_zoom=0): +def admin_boundaries(ctx): """ Given a layer with admin boundaries and inclusion polygons for land-based boundaries, attempts to output a set of oriented @@ -1862,6 +1898,12 @@ def admin_boundaries(feature_layers, zoom, base_layer, clockwise if it was an inner). """ + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + base_layer = ctx.params.get('base_layer') + assert base_layer, 'Parameter base_layer missing.' + start_zoom = ctx.params.get('start_zoom', 0) + layer = None # don't start processing until the start zoom @@ -1971,11 +2013,15 @@ def admin_boundaries(feature_layers, zoom, base_layer, return layer -def generate_label_features( - feature_layers, zoom, source_layer=None, label_property_name=None, - label_property_value=None, new_layer_name=None, drop_keys=None): +def generate_label_features(ctx): + feature_layers = ctx.feature_layers + source_layer = ctx.params.get('source_layer') assert source_layer, 'generate_label_features: missing source_layer' + label_property_name = ctx.params.get('label_property_name') + label_property_value = ctx.params.get('label_property_value') + new_layer_name = ctx.params.get('new_layer_name') + drop_keys = ctx.params.get('drop_keys') layer = _find_layer(feature_layers, source_layer) if layer is None: @@ -2036,15 +2082,18 @@ def generate_label_features( return label_feature_layer -def generate_address_points( - feature_layers, zoom, source_layer=None, start_zoom=0): +def generate_address_points(ctx): """ Generates address points from building polygons where there is an addr:housenumber tag on the building. Removes those tags from the building. """ + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') assert source_layer, 'generate_address_points: missing source_layer' + start_zoom = ctx.params.get('start_zoom', 0) if zoom < start_zoom: return None @@ -2128,16 +2177,19 @@ def parse_layer_as_float(shape, properties, fid, zoom): return shape, properties, fid -def drop_features_where( - feature_layers, zoom, source_layer=None, start_zoom=0, - where=None): +def drop_features_where(ctx): """ Drop features entirely that match the particular "where" condition. Any feature properties are available to use, as well as the properties dict itself, called "properties" in the scope. """ + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') assert source_layer, 'drop_features_where: missing source layer' + start_zoom = ctx.params.get('start_zoom', 0) + where = ctx.params.get('where') assert where, 'drop_features_where: missing where' if zoom < start_zoom: @@ -2163,9 +2215,7 @@ def drop_features_where( return layer -def _project_properties( - feature_layers, zoom, where, action, source_layer=None, start_zoom=0, - end_zoom=None): +def _project_properties(ctx, action): """ Project properties down to a subset of the existing properties based on a predicate `where` which returns true when the function `action` should be @@ -2173,7 +2223,13 @@ def _project_properties( feature. """ + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + where = ctx.params.get('where') + source_layer = ctx.params.get('source_layer') assert source_layer, '_project_properties: missing source layer' + start_zoom = ctx.params.get('start_zoom', 0) + end_zoom = ctx.params.get('end_zoom') if zoom < start_zoom: return None @@ -2206,31 +2262,29 @@ def _project_properties( return layer -def drop_properties( - feature_layers, zoom, source_layer=None, start_zoom=0, - properties=None, end_zoom=None, where=None): +def drop_properties(ctx): """ Drop all configured properties for features in source_layer """ + properties = ctx.params.get('properties') assert properties, 'drop_properties: missing properties' def action(p): return _remove_properties(p, *properties) - return _project_properties(feature_layers, zoom, where, action, - source_layer, start_zoom, end_zoom) + return _project_properties(ctx, action) -def keep_properties( - feature_layers, zoom, source_layer=None, start_zoom=0, - properties=None, end_zoom=None, where=None): +def keep_properties(ctx): """ Keep only configured properties for features in source_layer """ + properties = ctx.params.get('properties') assert properties, 'keep_properties: missing properties' + where = ctx.params.get('where') if where is not None: where = compile(where, 'queries.yaml', 'eval') @@ -2242,8 +2296,7 @@ def keep_property(p, props): return p in properties and (where is None or eval(where, {}, local)) - return _project_properties(feature_layers, zoom, keep_property, - source_layer, start_zoom, end_zoom) + return _project_properties(ctx, keep_property) def remove_zero_area(shape, properties, fid, zoom): @@ -2335,10 +2388,7 @@ def keep_feature(self, feature): return False -def remove_duplicate_features( - feature_layers, zoom, source_layer=None, source_layers=None, - start_zoom=0, property_keys=None, geometry_types=None, - min_distance=0.0, none_means_unique=True): +def remove_duplicate_features(ctx): """ Removes duplicate features from a layer, or set of layers. The definition of duplicate is anything which has the same values @@ -2355,6 +2405,16 @@ def remove_duplicate_features( and kind properties would be kept in the output. """ + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') + source_layers = ctx.params.get('source_layers') + start_zoom = ctx.params.get('start_zoom', 0) + property_keys = ctx.params.get('property_keys') + geometry_types = ctx.params.get('geometry_types') + min_distance = ctx.params.get('min_distance', 0.0) + none_means_unique = ctx.params.get('none_means_unique', True) + # can use either a single source layer, or multiple source # layers, but not both. assert bool(source_layer) ^ bool(source_layers), \ @@ -2432,9 +2492,7 @@ def remove_duplicate_features( return None -def normalize_and_merge_duplicate_stations( - feature_layers, zoom, source_layer=None, start_zoom=0, - end_zoom=None): +def normalize_and_merge_duplicate_stations(ctx): """ Normalise station names by removing any parenthetical lines lists at the end (e.g: "Foo St (A, C, E)"). Parse this and @@ -2450,7 +2508,12 @@ def normalize_and_merge_duplicate_stations( the station POIs to be out-of-order. """ + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') assert source_layer, 'normalize_and_merge_duplicate_stations: missing source layer' + start_zoom = ctx.params.get('start_zoom', 0) + end_zoom = ctx.params.get('end_zoom') if zoom < start_zoom: return None @@ -2560,9 +2623,7 @@ def _match_props(props, items_matching): return True -def keep_n_features( - feature_layers, zoom, source_layer=None, start_zoom=0, - end_zoom=None, items_matching=None, max_items=None): +def keep_n_features(ctx): """ Keep only the first N features matching `items_matching` in the layer. This is primarily useful for removing @@ -2576,7 +2637,14 @@ def keep_n_features( count is larger than `max_items`, dropping those features. """ + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') assert source_layer, 'keep_n_features: missing source layer' + start_zoom = ctx.params.get('start_zoom', 0) + end_zoom = ctx.params.get('end_zoom') + items_matching = ctx.params.get('items_matching') + max_items = ctx.params.get('max_items') # leaving items_matching or max_items as None (or zero) # would mean that this filter would do nothing, so assume @@ -2615,9 +2683,7 @@ def keep_n_features( return layer -def rank_features( - feature_layers, zoom, source_layer=None, start_zoom=0, - items_matching=None, rank_key=None): +def rank_features(ctx): """ Enumerate the features matching `items_matching` and insert the rank as a property with the key `rank_key`. This is @@ -2625,7 +2691,13 @@ def rank_features( only the top features, or de-emphasise the later features. """ + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') assert source_layer, 'rank_features: missing source layer' + start_zoom = ctx.params.get('start_zoom', 0) + items_matching = ctx.params.get('items_matching') + rank_key = ctx.params.get('rank_key') # leaving items_matching or rank_key as None would mean # that this filter would do nothing, so assume that this @@ -2666,9 +2738,7 @@ def normalize_aerialways(shape, props, fid, zoom): return shape, props, fid -def numeric_min_filter( - feature_layers, zoom, source_layer=None, filters=None, - mode=None): +def numeric_min_filter(ctx): """ Keep only features which have properties equal or greater than the configured minima. These are in a dict per zoom @@ -2686,7 +2756,12 @@ def numeric_min_filter( match. """ + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') assert source_layer, 'rank_features: missing source layer' + filters = ctx.params.get('filters') + mode = ctx.params.get('mode') # assume missing filter is a config error. assert filters, 'numeric_min_filter: missing or empty filters dict' @@ -2722,15 +2797,20 @@ def numeric_min_filter( return layer -def copy_features( - feature_layers, zoom, source_layer=None, target_layer=None, - where=None, geometry_types=None): +def copy_features(ctx): """ Copy features matching _both_ the `where` selection and the `geometry_types` list to another layer. If the target layer doesn't exist, it is created. """ + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') + target_layer = ctx.params.get('target_layer') + where = ctx.params.get('where') + geometry_types = ctx.params.get('geometry_types') + assert source_layer, 'copy_features: source layer not configured' assert target_layer, 'copy_features: target layer not configured' assert where, 'copy_features: you must specify how to match features in the where parameter' @@ -2779,8 +2859,7 @@ def make_representative_point(shape, properties, fid, zoom): return shape, properties, fid -def remove_abandoned_pistes( - feature_layers, zoom, source_layer=None, start_zoom=0): +def remove_abandoned_pistes(ctx): """ Removes features tagged as abandoned pistes. @@ -2793,6 +2872,11 @@ def remove_abandoned_pistes( information any more. """ + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') + start_zoom = ctx.params.get('start_zoom', 0) + assert source_layer, 'remove_abandoned_pistes: missing source layer' if zoom < start_zoom: @@ -2863,9 +2947,7 @@ def normalize_leisure_kind(shape, properties, fid, zoom): return shape, properties, fid -def merge_features( - feature_layers, zoom, source_layer=None, start_zoom=0, - end_zoom=None): +def merge_features(ctx): """ Merge (linear) features with the same properties together, attempting to make the resulting geometry as large as possible. Note that this will @@ -2875,6 +2957,12 @@ def merge_features( it would be possible to extend to other geometry types. """ + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') + start_zoom = ctx.params.get('start_zoom', 0) + end_zoom = ctx.params.get('end_zoom') + assert source_layer, 'merge_features: missing source layer' if zoom < start_zoom: From 033b6b29f0c8b6a822c9d5197b36982af72719c3 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 1 Mar 2016 18:10:52 +0000 Subject: [PATCH 279/344] Add CSV property matching function. --- TileStache/Goodies/VecTiles/transform.py | 127 +++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 5ae96fa0..12e745f3 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -18,6 +18,8 @@ from util import to_float from sort import pois as sort_pois import re +import csv +import os.path feet_pattern = re.compile('([+-]?[0-9.]+)\'(?: *([+-]?[0-9.]+)")?') @@ -3093,3 +3095,128 @@ def normalize_medical_kind(shape, properties, fid, zoom): properties['specialty'] = specialty.split(';') return (shape, properties, fid) + + +class _AnyMatcher(object): + def match(self, other): + return True + + +class _NoneMatcher(object): + def match(self, other): + return other is None + + +class _SomeMatcher(object): + def match(self, other): + return other is not None + + +class _ExactMatcher(object): + def __init__(self, value): + self.value = value + + def match(self, other): + return other == self.value + + +class _SetMatcher(object): + def __init__(self, values): + self.values = values + + def match(self, other): + return other in self.values + + +class _CSVMatcher(object): + def __init__(self, csv_file): + keys = None + rows = [] + + self.static_any = _AnyMatcher() + self.static_none = _NoneMatcher() + self.static_some = _SomeMatcher() + + with open(csv_file, 'r') as fh: + # CSV - allow whitespace after the comma + reader = csv.reader(fh, skipinitialspace=True) + for row in reader: + if keys is None: + target_key = row.pop(-1) + keys = row + + else: + target_val = row.pop(-1) + row = [self._match_val(v) for v in row] + rows.append((row, target_val)) + + self.keys = keys + self.rows = rows + self.target_key = target_key + + def _match_val(self, v): + if v == '*': + return self.static_any + if v == '-': + return self.static_none + if v == '+': + return self.static_some + if ';' in v: + return _SetMatcher(set(v.split(';'))) + return _ExactMatcher(v) + + def match(self, properties): + vals = [properties.get(k) for k in self.keys] + for row, target_val in self.rows: + if all([a.match(b) for (a, b) in zip(row, vals)]): + return (self.target_key, target_val) + + return None + + +def csv_match_properties(ctx): + """ + Add or update a property on all features which match properties which are + given as headings in a CSV file. + """ + + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') + start_zoom = ctx.params.get('start_zoom', 0) + end_zoom = ctx.params.get('end_zoom') + csv_file = ctx.params.get('csv_file') + target_value_type = ctx.params.get('target_value_type') + + assert source_layer, 'csv_match_properties: missing source layer' + assert csv_file, 'csv_match_properties: missing CSV file' + + if zoom < start_zoom: + return None + + if end_zoom is not None and zoom > end_zoom: + return None + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + # unless the path is absolute, make it relative to the config file + # location. + if not os.path.isabs(csv_file): + csv_file = os.path.join(ctx.config_file_path, csv_file) + + matcher = _CSVMatcher(csv_file) + + def _type_cast(v): + if target_value_type == 'int': + return int(v) + return v + + for shape, props, fid in layer['features']: + m = matcher.match(props) + if m is not None: + k, v = m + props[k] = _type_cast(v) + + return layer From 132250193aa676d84fac461bb393c798b7f6422c Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 1 Mar 2016 18:11:12 +0000 Subject: [PATCH 280/344] Remove deprecated landuse sort key function. --- TileStache/Goodies/VecTiles/transform.py | 122 ----------------------- 1 file changed, 122 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 12e745f3..1df22691 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1196,128 +1196,6 @@ def remap_deprecated_landuse_kinds(shape, properties, fid, zoom): return shape, properties, fid -# explicit order for some kinds of landuse -_landuse_sort_order = { - 'aerodrome': 49, - 'allotments': 83, - 'amusement_ride': 97, - 'animal': 92, - 'apron': 51, - 'aquarium': 46, - 'artwork': 93, - 'attraction': 94, - 'aviary': 72, - 'beach': 87, - 'breakwater': 221, - 'bridge': 226, - 'carousel': 95, - 'cemetery': 66, - 'cinema': 69, - 'college': 32, - 'commercial': 35, - 'common': 56, - 'conservation': 24, - 'cutline': 223, - 'dike': 224, - 'enclosure': 74, - 'farm': 28, - 'farmland': 29, - 'farmyard': 61, - 'footway': 100, - 'forest': 30, - 'fuel': 68, - 'garden': 88, - 'generator': 84, - 'glacier': 12, - 'golf_course': 47, - 'grass': 78, - 'groyne': 222, - 'hanami': 86, - 'hospital': 45, - 'industrial': 23, - 'land': 220, - 'library': 70, - 'maze': 85, - 'meadow': 79, - 'military': 37, - 'national_park': 19, - 'nature_reserve': 25, - 'park or protected land': 17, - 'park': 26, - 'park': 33, - 'parking': 64, - 'pedestrian': 89, - 'petting_zoo': 73, - 'pier': 225, - 'pitch': 91, - 'place_of_worship': 65, - 'plant': 77, - 'playground': 90, - 'prison': 38, - 'protected_area': 18, - 'quarry': 63, - 'railway': 62, - 'recreation_ground': 48, - 'residential': 22, - 'resort': 43, - 'retail': 36, - 'roller_coaster': 75, - 'runway': 52, - 'rural': 15, - 'school': 67, - 'scrub': 80, - 'sports_centre': 54, - 'stadium': 53, - 'substation': 60, - 'summer_toboggan': 76, - 'taxiway': 50, - 'theatre': 71, - 'theme_park': 39, - 'tower': 99, - 'trail_riding_station': 44, - 'university': 31, - 'urban area': 13, - 'urban': 14, - 'village_green': 55, - 'wastewater_plant': 57, - 'water_slide': 96, - 'water_works': 58, - 'wetland': 81, - 'wilderness_hut': 98, - 'wildlife_park': 40, - 'winery': 41, - 'winter_sports': 27, - 'wood': 82, - 'works': 59, - 'zoo': 42 -} - - -# sets a key "order" on anything with a landuse kind -# specified in the landuse sort order above. this is -# to help with maintaining a consistent order across -# post-processing steps in the server and drawing -# steps on the client. -def landuse_sort_key(shape, properties, fid, zoom): - kind = properties.get('kind') - - # land is at 10 - # default to 11 for landuse if not in the lookup table - # (landuse lookup table starts at 12) - fallback_sort_key = 11 - - if kind is not None: - key = _landuse_sort_order.get(kind) - if key is not None: - properties['sort_key'] = key - else: - properties['sort_key'] = fallback_sort_key - else: - properties['sort_key'] = fallback_sort_key - - return shape, properties, fid - - # place kinds, as used by OSM, mapped to their rough # scale_ranks so that we can provide a defaulted, # non-curated scale_rank / min_zoom value. From 37aae605b9ba96663a9897777ab4b64ae1488210 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 1 Mar 2016 18:49:49 +0000 Subject: [PATCH 281/344] Use cache to avoid re-reading the CSV file. --- TileStache/Goodies/VecTiles/transform.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 1df22691..68e81d57 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3084,7 +3084,14 @@ def csv_match_properties(ctx): if not os.path.isabs(csv_file): csv_file = os.path.join(ctx.config_file_path, csv_file) - matcher = _CSVMatcher(csv_file) + # the CSV matcher loads files from disk, which is a pretty expensive + # operation and we'd like that to happen as few times as possble, so + # we cache the matcher object across tile requests. + _cache_key = 'csv_match_properties:matcher' + matcher = ctx.cache.get(_cache_key) + if matcher is None: + matcher = _CSVMatcher(csv_file) + ctx.cache[_cache_key] = matcher def _type_cast(v): if target_value_type == 'int': From fcd69f40edc80285c50645b24f873f875eff6b09 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 2 Mar 2016 12:11:30 +0000 Subject: [PATCH 282/344] Use new 'resources' section for creating and storing the 'matcher' object. Also, matcher is now a callable. --- TileStache/Goodies/VecTiles/transform.py | 48 ++++++++---------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 68e81d57..34a079f4 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -19,7 +19,6 @@ from sort import pois as sort_pois import re import csv -import os.path feet_pattern = re.compile('([+-]?[0-9.]+)\'(?: *([+-]?[0-9.]+)")?') @@ -3006,8 +3005,8 @@ def match(self, other): return other in self.values -class _CSVMatcher(object): - def __init__(self, csv_file): +class CSVMatcher(object): + def __init__(self, fh): keys = None rows = [] @@ -3015,18 +3014,17 @@ def __init__(self, csv_file): self.static_none = _NoneMatcher() self.static_some = _SomeMatcher() - with open(csv_file, 'r') as fh: - # CSV - allow whitespace after the comma - reader = csv.reader(fh, skipinitialspace=True) - for row in reader: - if keys is None: - target_key = row.pop(-1) - keys = row + # CSV - allow whitespace after the comma + reader = csv.reader(fh, skipinitialspace=True) + for row in reader: + if keys is None: + target_key = row.pop(-1) + keys = row - else: - target_val = row.pop(-1) - row = [self._match_val(v) for v in row] - rows.append((row, target_val)) + else: + target_val = row.pop(-1) + row = [self._match_val(v) for v in row] + rows.append((row, target_val)) self.keys = keys self.rows = rows @@ -3043,7 +3041,7 @@ def _match_val(self, v): return _SetMatcher(set(v.split(';'))) return _ExactMatcher(v) - def match(self, properties): + def __call__(self, properties): vals = [properties.get(k) for k in self.keys] for row, target_val in self.rows: if all([a.match(b) for (a, b) in zip(row, vals)]): @@ -3063,11 +3061,11 @@ def csv_match_properties(ctx): source_layer = ctx.params.get('source_layer') start_zoom = ctx.params.get('start_zoom', 0) end_zoom = ctx.params.get('end_zoom') - csv_file = ctx.params.get('csv_file') target_value_type = ctx.params.get('target_value_type') + matcher = ctx.resources.get('matcher') assert source_layer, 'csv_match_properties: missing source layer' - assert csv_file, 'csv_match_properties: missing CSV file' + assert matcher, 'csv_match_properties: missing matcher resource' if zoom < start_zoom: return None @@ -3079,27 +3077,13 @@ def csv_match_properties(ctx): if layer is None: return None - # unless the path is absolute, make it relative to the config file - # location. - if not os.path.isabs(csv_file): - csv_file = os.path.join(ctx.config_file_path, csv_file) - - # the CSV matcher loads files from disk, which is a pretty expensive - # operation and we'd like that to happen as few times as possble, so - # we cache the matcher object across tile requests. - _cache_key = 'csv_match_properties:matcher' - matcher = ctx.cache.get(_cache_key) - if matcher is None: - matcher = _CSVMatcher(csv_file) - ctx.cache[_cache_key] = matcher - def _type_cast(v): if target_value_type == 'int': return int(v) return v for shape, props, fid in layer['features']: - m = matcher.match(props) + m = matcher(props) if m is not None: k, v = m props[k] = _type_cast(v) From 4f304d8074a0d8be4c0ec4aa085df6c91f00db8a Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 2 Mar 2016 20:02:52 +0000 Subject: [PATCH 283/344] Fix typo 'guage' -> 'gauge'. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 34a079f4..c0ea68f8 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -304,7 +304,7 @@ def road_sort_key(shape, properties, fid, zoom): ne_type = properties.get('type', '') is_railway = railway in ( - 'rail', 'tram', 'light_rail', 'narrow_guage', 'monorail') + 'rail', 'tram', 'light_rail', 'narrow_gauge', 'monorail') if (highway == 'motorway' or ne_type in ('Major Highway', 'Beltway', 'Bypass')): From eac03250315e468926ac9373aa1d9ce28e81a33f Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 2 Mar 2016 22:53:12 +0000 Subject: [PATCH 284/344] Need layer for sort key - will have to delete it as a post-process step. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index c0ea68f8..9f650094 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -375,7 +375,7 @@ def road_sort_key(shape, properties, fid, zoom): def road_trim_properties(shape, properties, fid, zoom): - properties = _remove_properties(properties, 'bridge', 'layer', 'tunnel') + properties = _remove_properties(properties, 'bridge', 'tunnel') return shape, properties, fid From ec66eec7a14b24149bd98f621ee3cf7839e6c948 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 2 Mar 2016 22:53:35 +0000 Subject: [PATCH 285/344] Added functionality to allow the column headings to specify what type they are. --- TileStache/Goodies/VecTiles/transform.py | 45 +++++++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 9f650094..e8caaa2a 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2978,16 +2978,25 @@ class _AnyMatcher(object): def match(self, other): return True + def __repr__(self): + return "*" + class _NoneMatcher(object): def match(self, other): return other is None + def __repr__(self): + return "-" + class _SomeMatcher(object): def match(self, other): return other is not None + def __repr__(self): + return "+" + class _ExactMatcher(object): def __init__(self, value): @@ -2996,6 +3005,9 @@ def __init__(self, value): def match(self, other): return other == self.value + def __repr__(self): + return repr(self.value) + class _SetMatcher(object): def __init__(self, values): @@ -3004,10 +3016,28 @@ def __init__(self, values): def match(self, other): return other in self.values + def __repr__(self): + return repr(self.value) + + +_KEY_TYPE_LOOKUP = { + 'int': int, + 'float': float, +} + +def _parse_kt(key_type): + kt = key_type.split("::") + + type_key = kt[1] if len(kt) > 1 else None + fn = _KEY_TYPE_LOOKUP.get(type_key, str) + + return (kt[0], fn) + class CSVMatcher(object): def __init__(self, fh): keys = None + types = [] rows = [] self.static_any = _AnyMatcher() @@ -3019,27 +3049,32 @@ def __init__(self, fh): for row in reader: if keys is None: target_key = row.pop(-1) - keys = row + keys = [] + for key_type in row: + key, typ = _parse_kt(key_type) + keys.append(key) + types.append(typ) else: target_val = row.pop(-1) - row = [self._match_val(v) for v in row] + for i in range(0, len(row)): + row[i] = self._match_val(row[i], types[i]) rows.append((row, target_val)) self.keys = keys self.rows = rows self.target_key = target_key - def _match_val(self, v): + def _match_val(self, v, typ): if v == '*': return self.static_any if v == '-': return self.static_none if v == '+': return self.static_some - if ';' in v: + if isinstance(v, str) and ';' in v: return _SetMatcher(set(v.split(';'))) - return _ExactMatcher(v) + return _ExactMatcher(typ(v)) def __call__(self, properties): vals = [properties.get(k) for k in self.keys] From 8cf93d910155f4cc0ff7639b82dbadd74e47d0b8 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 3 Mar 2016 18:16:56 -0500 Subject: [PATCH 286/344] Support matching on zoom levels --- TileStache/Goodies/VecTiles/transform.py | 69 ++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index e8caaa2a..e0ab1d28 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3020,11 +3020,56 @@ def __repr__(self): return repr(self.value) +class _GreaterThanEqualMatcher(object): + def __init__(self, value): + self.value = value + + def match(self, other): + return other >= self.value + + def __repr__(self): + return '>=%r' % self.value + + +class _GreaterThanMatcher(object): + def __init__(self, value): + self.value = value + + def match(self, other): + return other > self.value + + def __repr__(self): + return '>%r' % self.value + + +class _LessThanEqualMatcher(object): + def __init__(self, value): + self.value = value + + def match(self, other): + return other <= self.value + + def __repr__(self): + return '<=%r' % self.value + + +class _LessThanMatcher(object): + def __init__(self, value): + self.value = value + + def match(self, other): + return other < self.value + + def __repr__(self): + return '<%r' % self.value + + _KEY_TYPE_LOOKUP = { 'int': int, 'float': float, } + def _parse_kt(key_type): kt = key_type.split("::") @@ -3074,10 +3119,28 @@ def _match_val(self, v, typ): return self.static_some if isinstance(v, str) and ';' in v: return _SetMatcher(set(v.split(';'))) + if v.startswith('>='): + assert len(v) > 2, 'Invalid >= matcher' + return _GreaterThanEqualMatcher(typ(v[2:])) + if v.startswith('<='): + assert len(v) > 2, 'Invalid <= matcher' + return _LessThanEqualMatcher(typ(v[2:])) + if v.startswith('>'): + assert len(v) > 1, 'Invalid > matcher' + return _GreaterThanMatcher(typ(v[1:])) + if v.startswith('<'): + assert len(v) > 1, 'Invalid > matcher' + return _LessThanMatcher(typ(v[1:])) return _ExactMatcher(typ(v)) - def __call__(self, properties): - vals = [properties.get(k) for k in self.keys] + def __call__(self, properties, zoom): + vals = [] + for key in self.keys: + # NOTE zoom is special cased + if key == 'zoom': + vals.append(zoom) + else: + vals.append(key) for row, target_val in self.rows: if all([a.match(b) for (a, b) in zip(row, vals)]): return (self.target_key, target_val) @@ -3118,7 +3181,7 @@ def _type_cast(v): return v for shape, props, fid in layer['features']: - m = matcher(props) + m = matcher(props, zoom) if m is not None: k, v = m props[k] = _type_cast(v) From 4816bd2f581404a5b5e2db544c5542119a2bf2ff Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 4 Mar 2016 07:25:27 -0500 Subject: [PATCH 287/344] Add tag value, not key --- TileStache/Goodies/VecTiles/transform.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index e0ab1d28..6a8ee2cd 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3138,9 +3138,10 @@ def __call__(self, properties, zoom): for key in self.keys: # NOTE zoom is special cased if key == 'zoom': - vals.append(zoom) + val = zoom else: - vals.append(key) + val = properties.get('key') + vals.append(val) for row, target_val in self.rows: if all([a.match(b) for (a, b) in zip(row, vals)]): return (self.target_key, target_val) From a3939f64cb3f8e9e5f0670a18bea2e8211aae240 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 4 Mar 2016 11:28:52 -0500 Subject: [PATCH 288/344] Add more precision to json formatter at z16 --- TileStache/Goodies/VecTiles/geojson.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/geojson.py b/TileStache/Goodies/VecTiles/geojson.py index afcffdd7..5d18ff35 100644 --- a/TileStache/Goodies/VecTiles/geojson.py +++ b/TileStache/Goodies/VecTiles/geojson.py @@ -12,7 +12,9 @@ charfloat_pat = compile(r'^[\[,\,]-?\d+\.\d+(e-?\d+)?$') # floating point lat/lon precision for each zoom level, good to ~1/4 pixel. -precisions = [int(ceil(log(1< Date: Fri, 4 Mar 2016 16:32:18 +0000 Subject: [PATCH 289/344] Fix typo - needs to use the key, rather than the string 'key'. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 6a8ee2cd..4ddcab30 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3140,7 +3140,7 @@ def __call__(self, properties, zoom): if key == 'zoom': val = zoom else: - val = properties.get('key') + val = properties.get(key) vals.append(val) for row, target_val in self.rows: if all([a.match(b) for (a, b) in zip(row, vals)]): From 578d85fa6989de0984b84ff950569588f929486c Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 4 Mar 2016 17:57:24 -0500 Subject: [PATCH 290/344] Add changelog for v0.8.0 --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 173f5ded..57a7d99f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +v0.8.0 +------ +* Allow code in `drop_features_where` function +* Add bounding box clipping to exterior boundaries transform. +* Add normalisation functions for social facilities and medical places. +* Set default landuse order to 11, and adjust everything else by +1 +* Correct water features such that things above the water should be uniquely above the water +* Update road sort key values +* Add csv property matching functions +* Use more precision for json formatter on z16 and higher + v0.7.0 ------ From 80f142f9b977b317f14dea8676ea8eced78766e6 Mon Sep 17 00:00:00 2001 From: Nathaniel Kelso Date: Fri, 4 Mar 2016 16:36:53 -0800 Subject: [PATCH 291/344] update changelog --- .DS_Store | Bin 6148 -> 6148 bytes CHANGELOG.md | 14 ++++++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.DS_Store b/.DS_Store index ec04ea62620ebc16ad3bdb29fcf97fc89373ccb3..68b60cab42fb739179b80063df6a544205c2f5fe 100644 GIT binary patch delta 31 ncmZoMXfc@J&nU1lU^g?Pz+@hl*v+$9a+oJJ7;R?f_{$Ffo$3ku delta 64 zcmZoMXfc@J&nUPtU^g?P;A9?_SY=KIX9f=jM+QFzcLrAm9|nI0ch8*s Date: Mon, 7 Mar 2016 13:49:58 +0000 Subject: [PATCH 292/344] After merging, simplify with a very small tolerance to remove duplicate and almost-colinear points. --- TileStache/Goodies/VecTiles/transform.py | 25 ++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4ddcab30..3b1a7345 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -17,6 +17,7 @@ from shapely.geometry.collection import GeometryCollection from util import to_float from sort import pois as sort_pois +from sys import float_info import re import csv @@ -1692,6 +1693,7 @@ def _linemerge(geom): function. """ geom_type = geom.type + result_geom = None if geom_type == 'GeometryCollection': # collect together everything line-like from the geometry @@ -1702,16 +1704,31 @@ def _linemerge(geom): if not line.is_empty: lines.append(line) - return linemerge(lines) if lines else MultiLineString([]) + result_geom = linemerge(lines) if lines else None elif geom_type == 'LineString': - return geom + result_geom = geom elif geom_type == 'MultiLineString': - return linemerge(geom) + result_geom = linemerge(geom) else: - return MultiLineString([]) + result_geom = None + + if result_geom is not None: + # simplify with very small tolerance to remove duplicate points. + # almost duplicate or nearly colinear points can occur due to + # numerical round-off or precision in the intersection algorithm, and + # this should help get rid of those. see also: + # http://lists.gispython.org/pipermail/community/2014-January/003236.html + # + # the tolerance here is hard-coded to a fraction of the coordinate + # magnitude. there isn't a perfect way to figure out what this tolerance + # should be, so this may require some tweaking. + epsilon = max(map(abs, result_geom.bounds)) * float_info.epsilon * 1000 + result_geom = result_geom.simplify(epsilon, True) + + return result_geom if result_geom else MultiLineString([]) def _orient(geom): From 027755d113ae8860f7883efc1d38917bcdc6ba5b Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 7 Mar 2016 15:49:51 -0500 Subject: [PATCH 293/344] Add funicular to road kind rail --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 3b1a7345..b4d83927 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -131,7 +131,7 @@ def _building_calc_height(height_val, levels_val, levels_calc_fn): road_kind_path = set(('footpath', 'track', 'footway', 'steps', 'pedestrian', 'path', 'cycleway')) road_kind_rail = set(('rail', 'tram', 'light_rail', 'narrow_gauge', - 'monorail', 'subway')) + 'monorail', 'subway', 'funicular')) road_kind_aerialway = set(('gondola', 'cable_car', 'chair_lift', 'drag_lift', 'platter', 't-bar', 'goods', 'magic_carpet', 'rope_tow', 'yes', 'zip_line', 'j-bar', 'unknown', From 329028ca79ac0306a4487b215d8d19f0db0a456e Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 7 Mar 2016 18:15:40 -0500 Subject: [PATCH 294/344] Add transform to update parenthetical properties --- TileStache/Goodies/VecTiles/transform.py | 41 ++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 3b1a7345..39c9465c 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3205,3 +3205,44 @@ def _type_cast(v): props[k] = _type_cast(v) return layer + + +def update_parenthetical_properties(ctx): + """ + If a feature's name ends with a set of values in parens, update + its kind and increase the min_zoom appropriately. + """ + + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') + start_zoom = ctx.params.get('start_zoom', 0) + end_zoom = ctx.params.get('end_zoom') + parenthetical_values = ctx.params.get('values') + target_min_zoom = ctx.params.get('target_min_zoom') + + assert parenthetical_values is not None, \ + 'update_parenthetical_properties: missing values' + assert target_min_zoom is not None, \ + 'update_parenthetical_properties: missing target_min_zoom' + + if zoom < start_zoom: + return None + + if end_zoom is not None and zoom > end_zoom: + return None + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + for shape, props, fid in layer['features']: + name = props.get('name', '') + if not name: + continue + for value in parenthetical_values: + if name.endswith('(%s)' % value): + props['kind'] = value + props['min_zoom'] = target_min_zoom + + return layer From 7aab27de85cd577a1d20817d95b472787bf5c7df Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 8 Mar 2016 16:17:19 +0000 Subject: [PATCH 295/344] Added ability to drop parenthetical features below some given zoom level. --- TileStache/Goodies/VecTiles/transform.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 39c9465c..1fe50ebc 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3220,6 +3220,7 @@ def update_parenthetical_properties(ctx): end_zoom = ctx.params.get('end_zoom') parenthetical_values = ctx.params.get('values') target_min_zoom = ctx.params.get('target_min_zoom') + drop_below_zoom = ctx.params.get('drop_below_zoom') assert parenthetical_values is not None, \ 'update_parenthetical_properties: missing values' @@ -3236,13 +3237,22 @@ def update_parenthetical_properties(ctx): if layer is None: return None + new_features = [] for shape, props, fid in layer['features']: name = props.get('name', '') if not name: + new_features.append((shape, props, fid)) continue + + keep = True for value in parenthetical_values: if name.endswith('(%s)' % value): props['kind'] = value props['min_zoom'] = target_min_zoom + if drop_below_zoom and zoom < drop_below_zoom: + keep = False + if keep: + new_features.append((shape, props, fid)) + layer['features'] = new_features return layer From 72bc17b6bbff17511ba73ddccb03c30f7b6cf35b Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 8 Mar 2016 17:49:47 +0000 Subject: [PATCH 296/344] Add function to add construction state to stations. --- TileStache/Goodies/VecTiles/transform.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 97d5eb73..d7e3fec1 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3256,3 +3256,26 @@ def update_parenthetical_properties(ctx): layer['features'] = new_features return layer + + +def add_construction_state_to_stations(shape, properties, fid, zoom): + """ + If the feature is a station, and it has a state tag, then move that + tag to its properties. + """ + + kind = properties.get('kind') + if kind not in ('station'): + return shape, properties, fid + + tags = properties.get('tags') + if not tags: + return shape, properties, fid + + state = tags.get('state') + if not state: + return shape, properties, fid + + properties['state'] = state + + return shape, properties, fid From a38c41c5a9e6eca9b1ebdf97217ccb6227e2fff6 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 8 Mar 2016 17:57:43 +0000 Subject: [PATCH 297/344] Update function name - 'state' doesn't seem to be just about construction status. --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index d7e3fec1..679b0da7 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3258,7 +3258,7 @@ def update_parenthetical_properties(ctx): return layer -def add_construction_state_to_stations(shape, properties, fid, zoom): +def add_state_to_stations(shape, properties, fid, zoom): """ If the feature is a station, and it has a state tag, then move that tag to its properties. From 3c6880812601c279bf2557ad0ebc43ee07f51643 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 14 Mar 2016 17:59:16 +0000 Subject: [PATCH 298/344] Adjust tile rank score for stations to take into accound the different types of routes. --- TileStache/Goodies/VecTiles/sort.py | 16 ++-- TileStache/Goodies/VecTiles/transform.py | 97 +++++++++++++++++------- 2 files changed, 76 insertions(+), 37 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index 5e04faca..3a65d138 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -46,20 +46,14 @@ def _by_population(feature): return default_value if population is None else population -def _by_transit_routes(feature): +def _by_transit_score(feature): wkb, props, fid = feature + return props.get('mz_transit_score', 0) - num_lines = 0 - transit_routes = props.get('transit_routes') - if transit_routes is not None: - num_lines = len(transit_routes) - return num_lines - - -def _sort_by_transit_routes_then_feature_id(features): +def _sort_by_transit_score_then_feature_id(features): features.sort(key=_by_feature_id) - features.sort(key=_by_transit_routes, reverse=True) + features.sort(key=_by_transit_score, reverse=True) return features @@ -89,7 +83,7 @@ def places(features, zoom): def pois(features, zoom): - return _sort_by_transit_routes_then_feature_id(features) + return _sort_by_transit_score_then_feature_id(features) def roads(features, zoom): diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 679b0da7..822b5505 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2381,17 +2381,16 @@ def remove_duplicate_features(ctx): return None -def normalize_and_merge_duplicate_stations(ctx): +def merge_duplicate_stations(ctx): """ Normalise station names by removing any parenthetical lines lists at the end (e.g: "Foo St (A, C, E)"). Parse this and - use it to replace the `transit_routes` list if that is empty + use it to replace the `subway_routes` list if that is empty or isn't present. - Use the name, now appropriately trimmed, to merge station - POIs together, unioning their transit routes. - - Stations with empty transit_routes have that property removed. + Use the root relation ID, calculated as part of the exploration of the + transit relations, plus the name, now appropriately trimmed, to merge + station POIs together, unioning their subway routes. Finally, re-sort the features in case the merging has caused the station POIs to be out-of-order. @@ -2432,28 +2431,33 @@ def normalize_and_merge_duplicate_stations(ctx): # list of lines if we haven't already got that info. m = station_pattern.match(name) - transit_routes = props.get('transit_routes', []) + subway_routes = props.get('subway_routes', []) + transit_route_relation_id = props.get('mz_transit_root_relation_id') if m: # if the lines aren't present or are empty - if not transit_routes: + if not subway_routes: lines = m.group(2).split(',') - transit_routes = [x.strip() for x in lines] - props['transit_routes'] = transit_routes + subway_routes = [x.strip() for x in lines] + props['subway_routes'] = subway_routes # update name so that it doesn't contain all the # lines. name = m.group(1).strip() props['name'] = name - seen_idx = seen_stations.get(name) + # if the root relation ID is available, then use that for + # identifying duplicates. otherwise, use the name. + key = transit_route_relation_id or name + + seen_idx = seen_stations.get(key) if seen_idx is None: - seen_stations[name] = len(new_features) + seen_stations[key] = len(new_features) # ensure that transit routes is present and is of # list type for when we append to it later if we # find a duplicate. - props['transit_routes'] = transit_routes + props['subway_routes'] = subway_routes new_features.append(feature) else: @@ -2463,25 +2467,15 @@ def normalize_and_merge_duplicate_stations(ctx): seen_props = new_features[seen_idx][1] # make sure routes are unique - unique_transit_routes = set(transit_routes) & \ - set(seen_props['transit_routes']) - seen_props['transit_routes'] = list(unique_transit_routes) + unique_subway_routes = set(subway_routes) | \ + set(seen_props['subway_routes']) + seen_props['subway_routes'] = list(unique_subway_routes) else: # not a station, or name is missing - we can't # de-dup these. new_features.append(feature) - # remove anything that has an empty transit_routes - # list, as this most likely indicates that we were - # not able to _detect_ what lines it's part of, as - # it seems unlikely that a station would be part of - # _zero_ routes. - for shape, props, fid in new_features: - transit_routes = props.pop('transit_routes', []) - if transit_routes: - props['transit_routes'] = transit_routes - # might need to re-sort, if we merged any stations: # removing duplicates would have changed the number # of routes for each station. @@ -2492,6 +2486,57 @@ def normalize_and_merge_duplicate_stations(ctx): return layer +def normalize_station_properties(ctx): + """ + Normalise station properties by removing some which are only used during + importance calculation. Stations may also have route information, which may + appear as empty lists. These are removed. Also, flags are put on the station + to indicate what kind(s) of station it might be. + """ + + feature_layers = ctx.feature_layers + zoom = ctx.tile_coord.zoom + source_layer = ctx.params.get('source_layer') + assert source_layer, 'normalize_and_merge_duplicate_stations: missing source layer' + start_zoom = ctx.params.get('start_zoom', 0) + end_zoom = ctx.params.get('end_zoom') + + if zoom < start_zoom: + return None + + # we probably don't want to do this at higher zooms (e.g: 17 & + # 18), even if there are a bunch of stations very close + # together. + if end_zoom is not None and zoom > end_zoom: + return None + + layer = _find_layer(feature_layers, source_layer) + if layer is None: + return None + + for shape, props, fid in layer['features']: + kind = props.get('kind') + + # get rid of temporaries + #props.pop('mz_transit_root_relation_id', None) + #props.pop('mz_transit_score', None) + + if kind == 'station': + # remove anything that has an empty *_routes + # list, as this most likely indicates that we were + # not able to _detect_ what lines it's part of, as + # it seems unlikely that a station would be part of + # _zero_ routes. + for typ in ['train', 'subway', 'light_rail', 'tram']: + prop_name = '%s_routes' % typ + routes = props.pop(prop_name, []) + if routes: + props[prop_name] = routes + props['is_%s' % typ] = True + + return layer + + def _match_props(props, items_matching): """ Checks if all the items in `items_matching` are also From 747c6351df1f887d4fec077a44549d10426a04b8 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 15 Mar 2016 12:38:31 +0000 Subject: [PATCH 299/344] Remove temporary properties which shouldn't be public. --- TileStache/Goodies/VecTiles/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 822b5505..33cdcd4e 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2518,8 +2518,8 @@ def normalize_station_properties(ctx): kind = props.get('kind') # get rid of temporaries - #props.pop('mz_transit_root_relation_id', None) - #props.pop('mz_transit_score', None) + props.pop('mz_transit_root_relation_id', None) + props.pop('mz_transit_score', None) if kind == 'station': # remove anything that has an empty *_routes From ca50c689bbea44162732f3c18b6dfa4618b825e4 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 15 Mar 2016 12:41:50 +0000 Subject: [PATCH 300/344] Add 'root_relation_id' for linking related features together in a site or public transport 'stop area' or 'stop area group'. --- TileStache/Goodies/VecTiles/transform.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 33cdcd4e..4dcbe4c2 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2518,7 +2518,7 @@ def normalize_station_properties(ctx): kind = props.get('kind') # get rid of temporaries - props.pop('mz_transit_root_relation_id', None) + root_relation_id = props.pop('mz_transit_root_relation_id', None) props.pop('mz_transit_score', None) if kind == 'station': @@ -2534,6 +2534,12 @@ def normalize_station_properties(ctx): props[prop_name] = routes props['is_%s' % typ] = True + # if the station has a root relation ID then include + # that as a way for the client to link together related + # features. + if root_relation_id: + props['root_relation_id'] = root_relation_id + return layer From e2ed241a9b7a7268a8171b23a7b7639f8ab0c05f Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 15 Mar 2016 15:30:41 -0400 Subject: [PATCH 301/344] Remove now unused landuse kind mapping transform --- TileStache/Goodies/VecTiles/transform.py | 26 ------------------------ 1 file changed, 26 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4dcbe4c2..107c548c 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1170,32 +1170,6 @@ def intracut(ctx): return base -# map from old or deprecated kind value to the value that we want -# it to be. -_deprecated_landuse_kinds = { - 'station': 'substation', - 'sub_station': 'substation' -} - - -def remap_deprecated_landuse_kinds(shape, properties, fid, zoom): - """ - some landuse kinds are deprecated, or can be coalesced down to - a single value. this filter implements that by remapping kind - values. - """ - - original_kind = properties.get('kind') - - if original_kind is not None: - remapped_kind = _deprecated_landuse_kinds.get(original_kind) - - if remapped_kind is not None: - properties['kind'] = remapped_kind - - return shape, properties, fid - - # place kinds, as used by OSM, mapped to their rough # scale_ranks so that we can provide a defaulted, # non-curated scale_rank / min_zoom value. From 472b9430963239f7d22789b42e8b7c545f10b736 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 22 Mar 2016 13:46:19 -0400 Subject: [PATCH 302/344] Add uic_ref transform --- TileStache/Goodies/VecTiles/transform.py | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 107c548c..bb3258cf 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2842,6 +2842,33 @@ def add_iata_code_to_airports(shape, properties, fid, zoom): return shape, properties, fid +def add_uic_ref(shape, properties, fid, zoom): + """ + If the feature has a valid uic_ref tag (7 integers), then move it + to its properties. + """ + + tags = properties.get('tags') + if not tags: + return shape, properties, fid + + uic_ref = tags.get('uic_ref') + if not uic_ref: + return shape, properties, fid + + uic_ref = uic_ref.strip() + if len(uic_ref) != 7: + return shape, properties, fid + + try: + uic_ref_int = int(uic_ref) + except ValueError: + return shape, properties, fid + else: + properties['uic_ref'] = uic_ref_int + return shape, properties, fid + + def normalize_leisure_kind(shape, properties, fid, zoom): """ Normalise the various ways of representing fitness POIs to a From 8fab34ff9b829f08783564db5874f7e0e373a941 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 24 Mar 2016 16:05:44 -0400 Subject: [PATCH 303/344] Initial changelog for v0.9.0 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c721bec..13f1cc9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +v0.9.0 +------ +* After merging, simplify with a very small tolerance to remove duplicate and almost-colinear points +* Add transform to update parenthetical properties +* Add ability to drop parenthetical features below some given zoom level +* Add function to add construction state to stations +* Adjust tile rank score for stations to take into account the different types of routes +* Remove temporary properties which shouldn't be public +* Add 'root_relation_id' for linking related features together in a site or public transport 'stop area' or 'stop area group' +* Remove now unused landuse kind mapping transform +* Add uic_ref transform + v0.8.0 ------ * Allow code in `drop_features_where` function. From 36784d3aa30a8dccf26fecfd7806b292a0f6eebc Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 30 Mar 2016 14:18:54 +0100 Subject: [PATCH 304/344] Shapely STRtree now uses the is_empty property to filter out empty geometries when building the tree. --- TileStache/Goodies/VecTiles/transform.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index bb3258cf..aaa9fd92 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1436,6 +1436,9 @@ def __init__(self, geom, area): self.geom = geom self.area = area self._geom = geom._geom + # STRtree started filtering out empty geoms at some version, so + # we need to proxy the is_empty property. + self.is_empty = geom.is_empty # create an index so that we can efficiently find the # polygons intersecting the 'current' one. Note that From 9143f49910a320b1da736888f45c2e81775e71df Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 31 Mar 2016 11:01:07 -0400 Subject: [PATCH 305/344] Add transform to convert admin_level to an int --- TileStache/Goodies/VecTiles/transform.py | 28 ++++++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index aaa9fd92..f1fa86b5 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -446,6 +446,18 @@ def water_tunnel(shape, properties, fid, zoom): return shape, properties, fid +def admin_level_as_int(shape, properties, fid, zoom): + admin_level_str = properties.pop('admin_level', None) + if admin_level_str is None: + return shape, properties, fid + try: + admin_level_int = int(admin_level_str) + except ValueError: + return shape, properties, fid + properties['admin_level'] = admin_level_int + return shape, properties, fid + + boundary_admin_level_mapping = { 2: 'country', 4: 'state', @@ -455,6 +467,8 @@ def water_tunnel(shape, properties, fid, zoom): def boundary_kind(shape, properties, fid, zoom): + # assumes that the admin_level_int transform is run first + kind = properties.get('kind') if kind: return shape, properties, fid @@ -467,14 +481,14 @@ def boundary_kind(shape, properties, fid, zoom): properties['kind'] = 'aboriginal_lands' return shape, properties, fid - admin_level_str = properties.get('admin_level') - if admin_level_str is None: - return shape, properties, fid - try: - admin_level_int = int(admin_level_str) - except ValueError: + admin_level = properties.get('admin_level', None) + if admin_level is None: return shape, properties, fid - kind = boundary_admin_level_mapping.get(admin_level_int) + + assert isinstance(admin_level, int), \ + 'boundary_kind: expected admin_level to already be an int' + + kind = boundary_admin_level_mapping.get(admin_level) if kind: properties['kind'] = kind return shape, properties, fid From d38c632932866c68e9b67c2e888bb2bdaa661013 Mon Sep 17 00:00:00 2001 From: Nathaniel Kelso Date: Tue, 5 Apr 2016 17:15:31 -0400 Subject: [PATCH 306/344] connects to vector-datasource #680 (remove junk highway=minor and footpath) --- TileStache/Goodies/VecTiles/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index f1fa86b5..b26f5bc7 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -128,7 +128,7 @@ def _building_calc_height(height_val, levels_val, levels_calc_fn): road_kind_major_road = set(('trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link')) -road_kind_path = set(('footpath', 'track', 'footway', 'steps', 'pedestrian', +road_kind_path = set(('track', 'footway', 'steps', 'pedestrian', 'path', 'cycleway')) road_kind_rail = set(('rail', 'tram', 'light_rail', 'narrow_gauge', 'monorail', 'subway', 'funicular')) @@ -325,7 +325,7 @@ def road_sort_key(shape, properties, fid, zoom): elif (highway in ('residential', 'unclassified', 'road', 'living_street') or ne_type == 'Unknown'): sort_val += 60 - elif highway in ('service', 'minor'): + elif highway in ('service'): sort_val += 58 elif aerialway in ('gondola', 'cable_car'): sort_val += 92 From 0a5e7fbc0ed5424531d835cb1594f5a504e0328f Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 7 Apr 2016 22:31:54 +0100 Subject: [PATCH 307/344] Remove unused boundary transforms. --- TileStache/Goodies/VecTiles/transform.py | 43 ------------------------ 1 file changed, 43 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index f1fa86b5..03d27f8d 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -458,49 +458,6 @@ def admin_level_as_int(shape, properties, fid, zoom): return shape, properties, fid -boundary_admin_level_mapping = { - 2: 'country', - 4: 'state', - 6: 'county', - 8: 'municipality', -} - - -def boundary_kind(shape, properties, fid, zoom): - # assumes that the admin_level_int transform is run first - - kind = properties.get('kind') - if kind: - return shape, properties, fid - - # if the boundary is tagged as being that of a first nations - # state then skip the rest of the kind logic, regardless of - # the admin_level (see mapzen/vector-datasource#284). - boundary_type = properties.get('boundary_type') - if boundary_type == 'aboriginal_lands': - properties['kind'] = 'aboriginal_lands' - return shape, properties, fid - - admin_level = properties.get('admin_level', None) - if admin_level is None: - return shape, properties, fid - - assert isinstance(admin_level, int), \ - 'boundary_kind: expected admin_level to already be an int' - - kind = boundary_admin_level_mapping.get(admin_level) - if kind: - properties['kind'] = kind - return shape, properties, fid - - -def boundary_trim_properties(shape, properties, fid, zoom): - properties = _remove_properties( - properties, - 'boundary_type') - return shape, properties, fid - - def tags_create_dict(shape, properties, fid, zoom): tags_hstore = properties.get('tags') if tags_hstore: From c57e204afe312b00115179244c52b43139916257 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 8 Apr 2016 21:07:33 +0100 Subject: [PATCH 308/344] Removed unused building kind calculation. --- TileStache/Goodies/VecTiles/transform.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4c60b618..7e48c952 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -193,21 +193,6 @@ def remove_feature_id(shape, properties, fid, zoom): return shape, properties, None -def building_kind(shape, properties, fid, zoom): - kind = properties.get('kind') - if kind: - return shape, properties, fid - building = _coalesce(properties, 'building:part', 'building') - if building: - if building != 'yes': - kind = building - else: - kind = 'building' - if kind: - properties['kind'] = kind - return shape, properties, fid - - def building_height(shape, properties, fid, zoom): height = _building_calc_height( properties.get('height'), properties.get('building:levels'), From dd96b13650c12eb7108aa46b92db2ed6d306079e Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 11 Apr 2016 16:03:50 -0400 Subject: [PATCH 309/344] kind=portage_way when whitewater=portage_way --- TileStache/Goodies/VecTiles/transform.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 7e48c952..a116f6f7 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -171,6 +171,11 @@ def _road_kind(properties): man_made = properties.get('man_made') if man_made == 'pier': return 'path' + tags = properties.get('tags') + if tags: + whitewater = tags.get('whitewater') + if whitewater == 'portage_way': + return 'portage_way' return 'minor_road' From 73af54bd19a38cf429591c88cc8dbd64b122936e Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 11 Apr 2016 17:07:15 -0400 Subject: [PATCH 310/344] Delegate quantization to mapbox-vector-tile --- TileStache/Goodies/VecTiles/mvt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/mvt.py b/TileStache/Goodies/VecTiles/mvt.py index f6025d3d..6acb4507 100644 --- a/TileStache/Goodies/VecTiles/mvt.py +++ b/TileStache/Goodies/VecTiles/mvt.py @@ -13,16 +13,16 @@ def decode(file): return data # print data or write to file? -def encode(file, features, coord, layer_name=''): +def encode(file, features, coord, bounds, layer_name=''): layers = [] layers.append(get_feature_layer(layer_name, features)) - data = mapbox_vector_tile.encode(layers) + data = mapbox_vector_tile.encode(layers, quantize_bounds=bounds) file.write(data) -def merge(file, feature_layers, coord): +def merge(file, feature_layers, coord, bounds): ''' Retrieve a list of mapbox vector tile responses and merge them into one. @@ -33,7 +33,7 @@ def merge(file, feature_layers, coord): for layer in feature_layers: layers.append(get_feature_layer(layer['name'], layer['features'])) - data = mapbox_vector_tile.encode(layers) + data = mapbox_vector_tile.encode(layers, quantize_bounds=bounds) file.write(data) From 9e15acc8d3e272aa9b98be1548b9f16e65e97621 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 11 Apr 2016 17:18:19 -0400 Subject: [PATCH 311/344] Remove the unused road_sort_key transform --- TileStache/Goodies/VecTiles/transform.py | 85 ------------------------ 1 file changed, 85 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 7e48c952..3e2d023d 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -275,91 +275,6 @@ def road_classifier(shape, properties, fid, zoom): return shape, properties, fid -def road_sort_key(shape, properties, fid, zoom): - # Note! parse_layer_as_float must be run before this filter. - - floor = 300 - ceiling = 447 - sort_val = floor - - highway = properties.get('highway', '') - railway = properties.get('railway', '') - aeroway = properties.get('aeroway', '') - aerialway = properties.get('aerialway', '') - service = properties.get('service', '') - ne_type = properties.get('type', '') - - is_railway = railway in ( - 'rail', 'tram', 'light_rail', 'narrow_gauge', 'monorail') - - if (highway == 'motorway' or - ne_type in ('Major Highway', 'Beltway', 'Bypass')): - sort_val += 81 - elif is_railway: - sort_val += 80 - elif highway == 'trunk' or ne_type == 'Secondary Highway': - sort_val += 79 - elif highway == 'primary' or ne_type == 'Road': - sort_val += 78 - elif highway == 'secondary' or aeroway == 'runway': - sort_val += 77 - elif highway == 'tertiary' or aeroway == 'taxiway' or ne_type == 'Track': - sort_val += 76 - elif highway.endswith('_link'): - sort_val += 75 - elif (highway in ('residential', 'unclassified', 'road', 'living_street') - or ne_type == 'Unknown'): - sort_val += 60 - elif highway in ('service'): - sort_val += 58 - elif aerialway in ('gondola', 'cable_car'): - sort_val += 92 - elif aerialway == 'chair_lift': - sort_val += 91 - elif aerialway: - sort_val += 90 - else: - sort_val += 55 - - if is_railway and service: - if service in ('spur', 'siding'): - # make sort val more like residential, unclassified which - # also come in at zoom 12 - sort_val -= 19 - elif service == 'yard': - sort_val -= 21 - else: - sort_val -= 23 - - if highway == 'service' and service: - # sort alley, driveway, etc... under service - sort_val -= 2 - - if zoom >= 15: - bridge = properties.get('bridge') - tunnel = properties.get('tunnel') - if bridge in ('yes', 'true') or aerialway: - sort_val += 50 - elif (tunnel in ('yes', 'true') or - (railway == 'subway' and tunnel not in ('no', 'false'))): - sort_val -= 50 - - # Explicit layer is clipped to [-5, 5] range. Note that - # the layer, if present, will be a Float due to the - # parse_layer_as_float filter. - layer = properties.get('layer') - if layer is not None: - layer = max(min(layer, 5), -5) - if layer > 0: - sort_val = int(layer + ceiling - 5) - elif layer < 0: - sort_val = int(layer + floor + 5) - - properties['sort_key'] = sort_val - - return shape, properties, fid - - def road_trim_properties(shape, properties, fid, zoom): properties = _remove_properties(properties, 'bridge', 'tunnel') return shape, properties, fid From 121a0bae548f1f322f9af46c478f45c811fb7e49 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 15 Apr 2016 20:07:41 +0100 Subject: [PATCH 312/344] Fixed another source of duplicate points - not sure why this one appeared now? --- TileStache/Goodies/VecTiles/transform.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index ec539130..581a4cad 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1581,6 +1581,21 @@ def _linemerge(geom): epsilon = max(map(abs, result_geom.bounds)) * float_info.epsilon * 1000 result_geom = result_geom.simplify(epsilon, True) + result_geom_type = result_geom.type + # the geometry may still have invalid or repeated points if it has zero + # length segments, so remove anything where the length is less than + # epsilon. + if result_geom_type == 'LineString': + if result_geom.length < epsilon: + result_geom = None + + elif result_geom_type == 'MultiLineString': + parts = [] + for line in result_geom.geoms: + if line.length >= epsilon: + parts.append(line) + result_geom = MultiLineString(parts) + return result_geom if result_geom else MultiLineString([]) From f43cb03986bd121f3503f39a829b9dfdab870517 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 15 Apr 2016 20:08:34 +0100 Subject: [PATCH 313/344] Remove unused kind function and abandoned pistes filter. --- TileStache/Goodies/VecTiles/transform.py | 95 ------------------------ 1 file changed, 95 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 581a4cad..c9a80325 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -124,61 +124,6 @@ def _building_calc_height(height_val, levels_val, levels_calc_fn): return levels -road_kind_highway = set(('motorway', 'motorway_link')) -road_kind_major_road = set(('trunk', 'trunk_link', 'primary', 'primary_link', - 'secondary', 'secondary_link', - 'tertiary', 'tertiary_link')) -road_kind_path = set(('track', 'footway', 'steps', 'pedestrian', - 'path', 'cycleway')) -road_kind_rail = set(('rail', 'tram', 'light_rail', 'narrow_gauge', - 'monorail', 'subway', 'funicular')) -road_kind_aerialway = set(('gondola', 'cable_car', 'chair_lift', 'drag_lift', - 'platter', 't-bar', 'goods', 'magic_carpet', - 'rope_tow', 'yes', 'zip_line', 'j-bar', 'unknown', - 'mixed_lift', 'canopy', 'cableway')) -# top 10 values for piste:type from taginfo -road_kind_piste = set(('nordic', 'downhill', 'sleigh', 'skitour', 'hike', - 'sled', 'yes', 'snow_park', 'playground', 'ski_jump')) - - -def _road_kind(properties): - highway = properties.get('highway') - if highway in road_kind_highway: - return 'highway' - if highway in road_kind_major_road: - return 'major_road' - piste_type = properties.get('piste_type') - if piste_type in road_kind_piste: - return 'piste' - if highway in road_kind_path: - return 'path' - railway = properties.get('railway') - if railway in road_kind_rail: - return 'rail' - route = properties.get('route') - if route == 'ferry': - return 'ferry' - aerialway = properties.get('aerialway') - if aerialway in road_kind_aerialway: - return 'aerialway' - if highway == 'motorway_junction': - return 'exit' - leisure = properties.get('leisure') - if leisure == 'track': - # note: racetrack rather than track, as track might be confusing - # between a track for racing and a track as in a faint trail. - return 'racetrack' - man_made = properties.get('man_made') - if man_made == 'pier': - return 'path' - tags = properties.get('tags') - if tags: - whitewater = tags.get('whitewater') - if whitewater == 'portage_way': - return 'portage_way' - return 'minor_road' - - def add_id_to_properties(shape, properties, fid, zoom): properties['id'] = fid return shape, properties, fid @@ -2667,46 +2612,6 @@ def make_representative_point(shape, properties, fid, zoom): return shape, properties, fid -def remove_abandoned_pistes(ctx): - """ - Removes features tagged as abandoned pistes. - - It checks the kind, because it doesn't matter if the piste is abandoned if - the kind was detected as a road or track. It also checks the value, as it - appears that 'piste:abandoned = no' accounts for 30% of the instances. - - Finally, the piste_abandoned property is removed from the feature, as we - have filtered out all the 'yes' values, meaning that it conveys no useful - information any more. - """ - - feature_layers = ctx.feature_layers - zoom = ctx.tile_coord.zoom - source_layer = ctx.params.get('source_layer') - start_zoom = ctx.params.get('start_zoom', 0) - - assert source_layer, 'remove_abandoned_pistes: missing source layer' - - if zoom < start_zoom: - return None - - layer = _find_layer(feature_layers, source_layer) - if layer is None: - return None - - new_features = [] - for feature in layer['features']: - shape, props, fid = feature - - piste_abandoned = props.pop('piste_abandoned', None) - kind = props.get('kind') - if piste_abandoned != 'yes' or kind != 'piste': - new_features.append(feature) - - layer['features'] = new_features - return layer - - def add_iata_code_to_airports(shape, properties, fid, zoom): """ If the feature is an airport, and it has a 3-character From a5b0cd1e87777dbda892696a90dac14590a4940e Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 20 Apr 2016 15:42:43 +0100 Subject: [PATCH 314/344] Add function to convert any height present into meters. --- TileStache/Goodies/VecTiles/transform.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index c9a80325..f11959c7 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3130,3 +3130,16 @@ def add_state_to_stations(shape, properties, fid, zoom): properties['state'] = state return shape, properties, fid + + +def height_to_meters(shape, props, fid, zoom): + """ + If the properties has a "height" entry, then convert that to meters. + """ + + height = props.get('height') + if not height: + return shape, props, fid + + props['height'] = _to_float_meters(height) + return shape, props, fid From fb174cd6d2c37138a904dfd9c5491db964265546 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 20 Apr 2016 19:20:15 +0100 Subject: [PATCH 315/344] Update conversion of lengths / heights to meters and add more units. --- TileStache/Goodies/VecTiles/transform.py | 29 +++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index f11959c7..f7fee3f8 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -24,6 +24,19 @@ feet_pattern = re.compile('([+-]?[0-9.]+)\'(?: *([+-]?[0-9.]+)")?') number_pattern = re.compile('([+-]?[0-9.]+)') +# pattern to detect numbers with units. +# PLEASE: keep this in sync with the conversion factors below. +unit_pattern = re.compile('([+-]?[0-9.]+) *(mi|km|m|nmi|ft)') + +# multiplicative conversion factor from the unit into meters. +# PLEASE: keep this in sync with the unit_pattern above. +unit_conversion_factor = { + 'mi': 1609.3440, + 'km': 1000.0000, + 'm': 1.0000, + 'nmi': 1852.0000, + 'ft': 0.3048 +} # used to detect if the "name" of a building is # actually a house number. @@ -51,11 +64,14 @@ def _to_float_meters(x): # trim whitespace to simplify further matching x = x.strip() - # try explicit meters suffix - if x.endswith(' m'): - meters_as_float = to_float(x[:-2]) - if meters_as_float is not None: - return meters_as_float + # try looking for a unit + unit_match = unit_pattern.match(x) + if unit_match is not None: + value = unit_match.group(1) + units = unit_match.group(2) + value_as_float = to_float(value) + if value_as_float is not None: + return value_as_float * unit_conversion_factor[units] # try if it looks like an expression in feet via ' " feet_match = feet_pattern.match(x) @@ -74,7 +90,8 @@ def _to_float_meters(x): total_inches += inches_as_float parsed_feet_or_inches = True if parsed_feet_or_inches: - meters = total_inches * 0.02544 + # international inch is exactly 25.4mm + meters = total_inches * 0.0254 return meters # try and match the first number that can be parsed From 9aa91cce550a1d51209d27583122e7d88936d95f Mon Sep 17 00:00:00 2001 From: Nathaniel Kelso Date: Mon, 25 Apr 2016 13:12:40 +0100 Subject: [PATCH 316/344] add transform for pois to convert capacity to int --- TileStache/Goodies/VecTiles/transform.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index f7fee3f8..4f380cc5 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -304,6 +304,14 @@ def place_population_int(shape, properties, fid, zoom): return shape, properties, fid +def pois_capacity_int(shape, properties, fid, zoom): + pois_capacity_str = properties.pop('capacity', None) + capacity = to_float(pois_capacity_str) + if capacity is not None: + properties['capacity'] = int(capacity) + return shape, properties, fid + + def water_tunnel(shape, properties, fid, zoom): tunnel = properties.pop('tunnel', None) if tunnel in (None, 'no', 'false', '0'): From c5a6170bd51dbc7cd40a1009807f97bc2bacce1a Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 26 Apr 2016 11:06:01 +0100 Subject: [PATCH 317/344] Add function to convert elevations to meters. --- TileStache/Goodies/VecTiles/transform.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4f380cc5..edb6e4c1 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3141,6 +3141,7 @@ def add_state_to_stations(shape, properties, fid, zoom): """ kind = properties.get('kind') + assert kind, "WTF: kind should be set on [%r]: %r" % (fid, properties) if kind not in ('station'): return shape, properties, fid @@ -3168,3 +3169,15 @@ def height_to_meters(shape, props, fid, zoom): props['height'] = _to_float_meters(height) return shape, props, fid + +def elevation_to_meters(shape, props, fid, zoom): + """ + If the properties has an "elevation" entry, then convert that to meters. + """ + + elevation = props.get('elevation') + if not elevation: + return shape, props, fid + + props['elevation'] = _to_float_meters(elevation) + return shape, props, fid From 42bf243fedf039a887a6b8b9cc983c1faa7df0af Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 28 Apr 2016 12:54:18 +0100 Subject: [PATCH 318/344] Sort POIs by elevation when they are peaks. --- TileStache/Goodies/VecTiles/sort.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index 3a65d138..66f75cb3 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -51,8 +51,16 @@ def _by_transit_score(feature): return props.get('mz_transit_score', 0) -def _sort_by_transit_score_then_feature_id(features): +def _by_peak_elevation(feature): + wkb, props, fid = feature + if props.get('kind') != 'peak': + return 0 + return props.get('elevation', 0) + + +def _sort_by_transit_score_then_elevation_then_feature_id(features): features.sort(key=_by_feature_id) + features.sort(key=_by_peak_elevation, reverse=True) features.sort(key=_by_transit_score, reverse=True) return features @@ -83,7 +91,7 @@ def places(features, zoom): def pois(features, zoom): - return _sort_by_transit_score_then_feature_id(features) + return _sort_by_transit_score_then_elevation_then_feature_id(features) def roads(features, zoom): From 2881e4a6080828604e7c34cbb419b2ebec097c92 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 28 Apr 2016 14:36:49 +0100 Subject: [PATCH 319/344] Extend peaks to include volcanos. --- TileStache/Goodies/VecTiles/sort.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/sort.py b/TileStache/Goodies/VecTiles/sort.py index 66f75cb3..62fd21fe 100644 --- a/TileStache/Goodies/VecTiles/sort.py +++ b/TileStache/Goodies/VecTiles/sort.py @@ -53,7 +53,8 @@ def _by_transit_score(feature): def _by_peak_elevation(feature): wkb, props, fid = feature - if props.get('kind') != 'peak': + kind = props.get('kind') + if kind != 'peak' and kind != 'volcano': return 0 return props.get('elevation', 0) From bd7201559686b72e1c20ff6ff239fa89b61a4d14 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Fri, 29 Apr 2016 18:44:32 +0100 Subject: [PATCH 320/344] Add an end_zoom parameter to control the behaviour of remove_duplicate_features. --- TileStache/Goodies/VecTiles/transform.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index edb6e4c1..4aef2f87 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -2141,6 +2141,7 @@ def remove_duplicate_features(ctx): geometry_types = ctx.params.get('geometry_types') min_distance = ctx.params.get('min_distance', 0.0) none_means_unique = ctx.params.get('none_means_unique', True) + end_zoom = ctx.params.get('end_zoom') # can use either a single source layer, or multiple source # layers, but not both. @@ -2157,6 +2158,9 @@ def remove_duplicate_features(ctx): if zoom < start_zoom: return None + if end_zoom is not None and zoom > end_zoom: + return None + # allow either a single or multiple layers to be used. if source_layer: source_layers = [source_layer] From 2faef8c23f9863b4c283f920453aab0df405ebf8 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 2 May 2016 11:39:20 -0400 Subject: [PATCH 321/344] Add transform to add is_bicycle_route property --- TileStache/Goodies/VecTiles/transform.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4aef2f87..1b8e254c 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3185,3 +3185,14 @@ def elevation_to_meters(shape, props, fid, zoom): props['elevation'] = _to_float_meters(elevation) return shape, props, fid + + +def add_is_bicycle_route(shape, props, fid, zoom): + """ + If the props contain a bicycle_network tag or cycleway, it should + have an is_bicycle_route boolean. + """ + props.pop('is_bicycle_route', None) + if 'bicycle_network' in props or 'cycleway' in props: + props['is_bicycle_route'] = 'yes' + return shape, props, fid From 46a3e927e6cf431bd529105e819a780fc2c46e29 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 2 May 2016 14:29:23 -0400 Subject: [PATCH 322/344] Expand is_bicycle_route to include left/right/both --- TileStache/Goodies/VecTiles/transform.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 1b8e254c..dd3fcc87 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3193,6 +3193,10 @@ def add_is_bicycle_route(shape, props, fid, zoom): have an is_bicycle_route boolean. """ props.pop('is_bicycle_route', None) - if 'bicycle_network' in props or 'cycleway' in props: + if ('bicycle_network' in props or + 'cycleway' in props or + 'cycleway_left' in props or + 'cycleway_right' in props or + 'cycleway_both' in props): props['is_bicycle_route'] = 'yes' return shape, props, fid From e7f5842b1a1a02dba2550c59bc381a4a749275fe Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 2 May 2016 14:29:56 -0400 Subject: [PATCH 323/344] Add transform to normalize bicycle left/right vals --- TileStache/Goodies/VecTiles/transform.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index dd3fcc87..c9181d6c 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3200,3 +3200,20 @@ def add_is_bicycle_route(shape, props, fid, zoom): 'cycleway_both' in props): props['is_bicycle_route'] = 'yes' return shape, props, fid + + +def normalize_cycle_left_right(shape, props, fid, zoom): + """ + If the properties contain both a cycleway:left and cycleway:right + with the same values, those should be removed and replaced with a + single cycleway property. + """ + cycleway = props.get('cycleway') + cycleway_left = props.get('cycleway_left') + cycleway_right = props.get('cycleway_right') + if (cycleway_left and cycleway_right and cycleway_left == cycleway_right + and (not cycleway or cycleway_left == cycleway)): + props['cycleway'] = cycleway_left + del props['cycleway_left'] + del props['cycleway_right'] + return shape, props, fid From 8968b8bbaf8122d4cc91740ec49ce67c94e4cf52 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 2 May 2016 18:15:16 -0400 Subject: [PATCH 324/344] Normalize cycleway_both -> cycleway --- TileStache/Goodies/VecTiles/transform.py | 40 ++++++++++++++---------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index c9181d6c..5fee8460 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3187,33 +3187,39 @@ def elevation_to_meters(shape, props, fid, zoom): return shape, props, fid -def add_is_bicycle_route(shape, props, fid, zoom): - """ - If the props contain a bicycle_network tag or cycleway, it should - have an is_bicycle_route boolean. - """ - props.pop('is_bicycle_route', None) - if ('bicycle_network' in props or - 'cycleway' in props or - 'cycleway_left' in props or - 'cycleway_right' in props or - 'cycleway_both' in props): - props['is_bicycle_route'] = 'yes' - return shape, props, fid - - -def normalize_cycle_left_right(shape, props, fid, zoom): +def normalize_cycleway(shape, props, fid, zoom): """ If the properties contain both a cycleway:left and cycleway:right with the same values, those should be removed and replaced with a - single cycleway property. + single cycleway property. Additionally, if a cycleway_both tag is + present, normalize that to the cycleway tag. """ cycleway = props.get('cycleway') cycleway_left = props.get('cycleway_left') cycleway_right = props.get('cycleway_right') + + cycleway_both = props.pop('cycleway_both', None) + if cycleway_both and not cycleway: + props['cycleway'] = cycleway = cycleway_both + if (cycleway_left and cycleway_right and cycleway_left == cycleway_right and (not cycleway or cycleway_left == cycleway)): props['cycleway'] = cycleway_left del props['cycleway_left'] del props['cycleway_right'] return shape, props, fid + + +def add_is_bicycle_route(shape, props, fid, zoom): + """ + If the props contain a bicycle_network tag or cycleway, it should + have an is_bicycle_route boolean. Depends on the + normalize_cycleway transform to have been run first. + """ + props.pop('is_bicycle_route', None) + if ('bicycle_network' in props or + 'cycleway' in props or + 'cycleway_left' in props or + 'cycleway_right' in props): + props['is_bicycle_route'] = 'yes' + return shape, props, fid From abfbaca8044b627d46fbbe209f7450af3ced1286 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 2 May 2016 18:30:36 -0400 Subject: [PATCH 325/344] Don't label kind=rock|stone features --- TileStache/Goodies/VecTiles/transform.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4aef2f87..1ac9eb07 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1777,6 +1777,11 @@ def generate_label_features(ctx): sport = properties.get('sport') if not sport: continue + # if we have a sport tag but no name, we only want it + # included if it's not a rock or stone + kind = properties.get('kind') + if kind in ('rock', 'stone'): + continue label_point = shape.representative_point() From 03b021e51c7533067f7dd3629349e53b12bad616 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 3 May 2016 13:45:53 -0400 Subject: [PATCH 326/344] Update bicycle related transform behavior --- TileStache/Goodies/VecTiles/transform.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 8ed4ee0f..6be7a952 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3215,16 +3215,18 @@ def normalize_cycleway(shape, props, fid, zoom): return shape, props, fid -def add_is_bicycle_route(shape, props, fid, zoom): +def add_is_bicycle_related(shape, props, fid, zoom): """ - If the props contain a bicycle_network tag or cycleway, it should - have an is_bicycle_route boolean. Depends on the - normalize_cycleway transform to have been run first. + If the props contain a bicycle_network tag, cycleway, or + highway=cycleway, it should have an is_bicycle_related + boolean. Depends on the normalize_cycleway transform to have been + run first. """ - props.pop('is_bicycle_route', None) + props.pop('is_bicycle_related', None) if ('bicycle_network' in props or 'cycleway' in props or 'cycleway_left' in props or - 'cycleway_right' in props): - props['is_bicycle_route'] = 'yes' + 'cycleway_right' in props or + props.get('highway') == 'cycleway'): + props['is_bicycle_related'] = True return shape, props, fid From cd73f28d157fc72a9848a669b8a195d0054dde6c Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 3 May 2016 22:16:57 +0100 Subject: [PATCH 327/344] Make geometry types for label generation configurable. --- TileStache/Goodies/VecTiles/transform.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 6be7a952..0f0c8096 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1749,6 +1749,7 @@ def generate_label_features(ctx): label_property_value = ctx.params.get('label_property_value') new_layer_name = ctx.params.get('new_layer_name') drop_keys = ctx.params.get('drop_keys') + geom_types = ctx.params.get('geom_types', ['Polygon', 'MultiPolygon']) layer = _find_layer(feature_layers, source_layer) if layer is None: @@ -1765,9 +1766,7 @@ def generate_label_features(ctx): if new_layer_name is None: new_features.append(feature) - # We only want to create label features for polygonal - # geometries - if shape.geom_type not in ('Polygon', 'MultiPolygon'): + if shape.geom_type not in geom_types: continue # Additionally, the feature needs to have a name or a sport From cb210e1f084df548bc5415b41097c9929e643000 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 3 May 2016 22:35:56 +0100 Subject: [PATCH 328/344] Don't convert lines or points to points for labelling - leave them as they were. --- TileStache/Goodies/VecTiles/transform.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 0f0c8096..014764e6 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1782,7 +1782,12 @@ def generate_label_features(ctx): if kind in ('rock', 'stone'): continue - label_point = shape.representative_point() + # need to generate a single point for (multi)polygons, but lines and + # points can be labelled directly. + if shape.geom_type in ('Polygon', 'MultiPolygon'): + label_geom = shape.representative_point() + else: + label_geom = shape label_properties = properties.copy() @@ -1795,7 +1800,7 @@ def generate_label_features(ctx): if label_property_name: label_properties[label_property_name] = label_property_value - label_feature = label_point, label_properties, fid + label_feature = label_geom, label_properties, fid new_features.append(label_feature) From 85a28d03c04ea8787af3aac4ed3b5abf6c54e4cd Mon Sep 17 00:00:00 2001 From: Nathaniel Kelso Date: Wed, 4 May 2016 10:44:55 -0700 Subject: [PATCH 329/344] v0.10 changelog --- CHANGELOG.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13f1cc9d..fd2956bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,24 @@ +v0.10.0 +------ +* Add `is_bicycle_related` transform. See [#152](https://github.com/mapzen/TileStache/pull/152). +* Normalize cycling related properties so shared properties for `cycleway:left` and `cycleway:right` are deleted and projected into `cycleway` directly, and if `cycleway:both` is included but `cycleway` is not, project that onto `cycleway`. [#150](https://github.com/mapzen/TileStache/pull/150). +* Don't label generate label placements for unnamed stone and rock features. See [#151](https://github.com/mapzen/TileStache/pull/151). +* Don't generate label placements for point or line features (for islands), make it configurable. See [#153](https://github.com/mapzen/TileStache/pull/153) and [#154](https://github.com/mapzen/TileStache/pull/154). +* Update road kind for `whitewater=portage_way`. See [#140](https://github.com/mapzen/TileStache/pull/140). +* Remove junk `highway=minor` and `highway=footpath` from logic. See [#137](https://github.com/mapzen/TileStache/pull/137). +* Remove outdated building kind calculation. See [#139](https://github.com/mapzen/TileStache/pull/139). +* Remove outdated `road_sort_key` transform. See [#142](https://github.com/mapzen/TileStache/pull/142). +* Remove outdated roads functions (logic is now carried in YAML queries), and fix duplicate points bug. See [#143](https://github.com/mapzen/TileStache/pull/143). +* Remove outdated boundary transforms (logic is now carried in YAML queries). See [#138](https://github.com/mapzen/TileStache/pull/138). +* Add transform to convert `admin_level` to an int. See [#136](https://github.com/mapzen/TileStache/pull/136). +* Add transform for convert `capacity` in pois layer to int. See [#146](https://github.com/mapzen/TileStache/pull/146). +* Add function to convert `height` values to meters. See [#145](https://github.com/mapzen/TileStache/pull/145). +* Add function to convert `elevation` values meters. See [#147](https://github.com/mapzen/TileStache/pull/147). +* Add sort order for `peak` and `volcano` features. See [#148](https://github.com/mapzen/TileStache/pull/148). +* Add end zoom for remove duplicates function. See [#149](https://github.com/mapzen/TileStache/pull/149). +* Add `is_empty` property for geometry proxy for Shapely's STRtree. See [#135](https://github.com/mapzen/TileStache/pull/135). +* Delegate quantization to mapbox-vector-tile. See [#141](https://github.com/mapzen/TileStache/pull/141). + v0.9.0 ------ * After merging, simplify with a very small tolerance to remove duplicate and almost-colinear points From 6c75a67ae8a39cbc4926cd87c4596f27a0b04141 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 5 May 2016 14:29:03 -0400 Subject: [PATCH 330/344] Remove no longer used transform --- TileStache/Goodies/VecTiles/transform.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 014764e6..6f44a0a0 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -285,17 +285,6 @@ def route_name(shape, properties, fid, zoom): return shape, properties, fid -def place_ne_capital(shape, properties, fid, zoom): - source = properties.get('source', '') - if source == 'naturalearthdata.com': - kind = properties.get('kind', '') - if kind == 'Admin-0 capital': - properties['capital'] = 'yes' - elif kind == 'Admin-1 capital': - properties['state_capital'] = 'yes' - return shape, properties, fid - - def place_population_int(shape, properties, fid, zoom): population_str = properties.pop('population', None) population = to_float(population_str) From 69869fdb234086e446f266b532a5ba50d2afde47 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 10 May 2016 15:29:56 -0400 Subject: [PATCH 331/344] Update building properties to use _ separator The queries now return the building properties with _ as the separator. --- CHANGELOG.md | 6 +++++- TileStache/Goodies/VecTiles/transform.py | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd2956bf..70b93a1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ +v0.10.1 +------- +* Update building transforms to work with `_` separated properties. The queries upstream changed to return `_` as the separator instead of `:`. See [#806](https://github.com/mapzen/vector-datasource/issues/806). + v0.10.0 ------- +------- * Add `is_bicycle_related` transform. See [#152](https://github.com/mapzen/TileStache/pull/152). * Normalize cycling related properties so shared properties for `cycleway:left` and `cycleway:right` are deleted and projected into `cycleway` directly, and if `cycleway:both` is included but `cycleway` is not, project that onto `cycleway`. [#150](https://github.com/mapzen/TileStache/pull/150). * Don't label generate label placements for unnamed stone and rock features. See [#151](https://github.com/mapzen/TileStache/pull/151). diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 014764e6..129dea69 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -162,7 +162,7 @@ def remove_feature_id(shape, properties, fid, zoom): def building_height(shape, properties, fid, zoom): height = _building_calc_height( - properties.get('height'), properties.get('building:levels'), + properties.get('height'), properties.get('building_levels'), _building_calc_levels) if height is not None: properties['height'] = height @@ -173,7 +173,7 @@ def building_height(shape, properties, fid, zoom): def building_min_height(shape, properties, fid, zoom): min_height = _building_calc_height( - properties.get('min_height'), properties.get('building:min_levels'), + properties.get('min_height'), properties.get('building_min_levels'), _building_calc_min_levels) if min_height is not None: properties['min_height'] = min_height @@ -193,8 +193,8 @@ def synthesize_volume(shape, props, fid, zoom): def building_trim_properties(shape, properties, fid, zoom): properties = _remove_properties( properties, - 'building', 'building:part', - 'building:levels', 'building:min_levels') + 'building', 'building_part', + 'building_levels', 'building_min_levels') return shape, properties, fid From 231484161d7e0d70c26bbe18f22913108b58434a Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Mon, 16 May 2016 11:07:18 -0400 Subject: [PATCH 332/344] Update behavior for route_name transform --- TileStache/Goodies/VecTiles/transform.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 44ab6861..ebec4bc3 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -277,10 +277,13 @@ def road_abbreviate_name(shape, properties, fid, zoom): def route_name(shape, properties, fid, zoom): - route_name = properties.get('route_name', '') - if route_name: - name = properties.get('name', '') - if route_name == name: + rn = properties.get('route_name') + if rn: + name = properties.get('name') + if not name: + properties['name'] = rn + del properties['route_name'] + elif rn == name: del properties['route_name'] return shape, properties, fid From e1f1ea054707998f4786d2f0f0cf494d94a034dd Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 17 May 2016 18:49:37 +0100 Subject: [PATCH 333/344] Start using true for boolean values rather than 'yes'. --- TileStache/Goodies/VecTiles/transform.py | 39 +++++++++++++++++------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index ebec4bc3..9e5b6fe3 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -233,11 +233,11 @@ def road_classifier(shape, properties, fid, zoom): bridge = properties.get('bridge', '') if highway.endswith('_link'): - properties['is_link'] = 'yes' + properties['is_link'] = True if tunnel in ('yes', 'true'): - properties['is_tunnel'] = 'yes' + properties['is_tunnel'] = True if bridge in ('yes', 'true'): - properties['is_bridge'] = 'yes' + properties['is_bridge'] = True return shape, properties, fid @@ -309,7 +309,7 @@ def water_tunnel(shape, properties, fid, zoom): if tunnel in (None, 'no', 'false', '0'): properties.pop('is_tunnel', None) else: - properties['is_tunnel'] = 'yes' + properties['is_tunnel'] = True return shape, properties, fid @@ -1058,9 +1058,9 @@ def calculate_default_place_scalerank(shape, properties, fid, zoom): # adjust scalerank for state / country capitals if kind in ('city', 'town'): - if properties.get('state_capital') == 'yes': + if properties.get('state_capital'): scalerank -= 1 - elif properties.get('capital') == 'yes': + elif properties.get('capital'): scalerank -= 2 properties['scalerank'] = scalerank @@ -1093,6 +1093,11 @@ def _make_new_properties(props, props_instructions): original_v = props.get(k) if original_v in v: new_props[k] = v[original_v] + elif isinstance(v, list) and len(v) == 1: + # this is a hack to implement escaping for when the output value + # should be a value, but that value (e.g: True, or a dict) is + # used for some other purpose above. + new_props[k] = v[0] else: new_props[k] = v @@ -1608,7 +1613,7 @@ def admin_boundaries(ctx): land-based boundaries, attempts to output a set of oriented boundaries with properties from both the left and right admin boundary, and also cut with the maritime information to provide - a `maritime_boundary=yes` value where there's overlap between + a `maritime_boundary: True` value where there's overlap between the maritime lines and the admin boundaries. Note that admin boundaries must alread be correctly oriented. @@ -1653,8 +1658,8 @@ def admin_boundaries(ctx): if dims == _LINE_DIMENSION and kind is not None: admin_features[kind].append((shape, props, fid)) - elif dims == _POLYGON_DIMENSION and maritime_boundary == 'yes': - maritime_features.append((shape, {'maritime_boundary':'no'}, 0)) + elif dims == _POLYGON_DIMENSION and maritime_boundary: + maritime_features.append((shape, {'maritime_boundary':False}, 0)) # there are separate polygons for each admin level, and # we only want to intersect like with like because it @@ -1726,7 +1731,7 @@ def admin_boundaries(ctx): for shape, props, fid in cutter.new_features: maritime_boundary = props.pop('maritime_boundary', None) if maritime_boundary is None: - props['maritime_boundary'] = 'yes' + props['maritime_boundary'] = True layer['features'] = cutter.new_features return layer @@ -2622,7 +2627,8 @@ def copy_features(ctx): # need to deep copy, otherwise we could have some # unintended side effects if either layer is # mutated later on. - new_features.append((shape.copy(), props.copy(), fid)) + shape_copy = shape.__class__(shape) + new_features.append((shape_copy, props.copy(), fid)) tgt_layer['features'].extend(new_features) return tgt_layer @@ -2896,6 +2902,14 @@ def __repr__(self): return "+" +class _TrueMatcher(object): + def match(self, other): + return other is True + + def __repr__(self): + return "true" + + class _ExactMatcher(object): def __init__(self, value): self.value = value @@ -2986,6 +3000,7 @@ def __init__(self, fh): self.static_any = _AnyMatcher() self.static_none = _NoneMatcher() self.static_some = _SomeMatcher() + self.static_true = _TrueMatcher() # CSV - allow whitespace after the comma reader = csv.reader(fh, skipinitialspace=True) @@ -3015,6 +3030,8 @@ def _match_val(self, v, typ): return self.static_none if v == '+': return self.static_some + if v == 'true': + return self.static_true if isinstance(v, str) and ';' in v: return _SetMatcher(set(v.split(';'))) if v.startswith('>='): From 865f7b04e351a027d6648b5109fc99adfa885cb3 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 13 May 2016 12:05:36 -0400 Subject: [PATCH 334/344] Update tags transform to convert wof l10n names --- TileStache/Goodies/VecTiles/transform.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 9e5b6fe3..487366c1 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -349,6 +349,14 @@ def tags_remove(shape, properties, fid, zoom): ) +def _convert_wof_l10n_name(x): + assert x.startswith('name:') + name_val = x[len('name:'):] + two_letter_lang = name_val[:2] + result = 'name:%s' % two_letter_lang + return result + + def tags_name_i18n(shape, properties, fid, zoom): tags = properties.get('tags') if not tags: @@ -358,6 +366,8 @@ def tags_name_i18n(shape, properties, fid, zoom): if not name: return shape, properties, fid + is_wof = properties.get('source') == 'whosonfirst.mapzen.com' + for k, v in tags.items(): if (k.startswith('name:') and v != name or k.startswith('alt_name:') and v != name or @@ -365,6 +375,8 @@ def tags_name_i18n(shape, properties, fid, zoom): k.startswith('old_name:') and v != name or k.startswith('left:name:') and v != name or k.startswith('right:name:') and v != name): + if is_wof: + k = _convert_wof_l10n_name(k) properties[k] = v for alt_tag_name_candidate in tag_name_alternates: From 026826d366e9068c32f6e6d106b97901b7b030ce Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 13 May 2016 16:25:53 -0400 Subject: [PATCH 335/344] Use pycountry to convert language codes --- TileStache/Goodies/VecTiles/transform.py | 14 ++++++++++++-- setup.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 487366c1..1f83e25c 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -18,6 +18,7 @@ from util import to_float from sort import pois as sort_pois from sys import float_info +import pycountry import re import csv @@ -352,8 +353,15 @@ def tags_remove(shape, properties, fid, zoom): def _convert_wof_l10n_name(x): assert x.startswith('name:') name_val = x[len('name:'):] - two_letter_lang = name_val[:2] - result = 'name:%s' % two_letter_lang + lang_str_iso_639_3 = name_val[:3] + if len(lang_str_iso_639_3) != 3: + return None + try: + lang = pycountry.languages.get(iso639_3_code=lang_str_iso_639_3) + except KeyError: + return None + lang_str_iso_639_1 = lang.iso639_1_code.encode('utf-8') + result = 'name:%s' % lang_str_iso_639_1 return result @@ -377,6 +385,8 @@ def tags_name_i18n(shape, properties, fid, zoom): k.startswith('right:name:') and v != name): if is_wof: k = _convert_wof_l10n_name(k) + if k is None: + continue properties[k] = v for alt_tag_name_candidate in tag_name_alternates: diff --git a/setup.py b/setup.py index 8dcfbdc7..0deefdb7 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ def is_installed(name): requires = ['ModestMaps >=1.3.0', 'simplejson', 'Werkzeug', - 'mapbox-vector-tile', 'StreetNames', 'Pillow'] + 'mapbox-vector-tile', 'StreetNames', 'Pillow', 'pycountry'] setup(name='TileStache', From a1cf7af251392a7bcfd4b83455355863e26dd315 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Tue, 17 May 2016 17:08:36 -0400 Subject: [PATCH 336/344] Normalize l10n tags to use iso-639-3 code --- TileStache/Goodies/VecTiles/transform.py | 65 +++++++++++++++++------- 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 1f83e25c..c0cfb622 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -347,22 +347,38 @@ def tags_remove(shape, properties, fid, zoom): 'old_name', 'reg_name', 'short_name', + 'name_left', + 'name_right', ) def _convert_wof_l10n_name(x): - assert x.startswith('name:') - name_val = x[len('name:'):] - lang_str_iso_639_3 = name_val[:3] + lang_str_iso_639_3 = x[:3] if len(lang_str_iso_639_3) != 3: return None try: - lang = pycountry.languages.get(iso639_3_code=lang_str_iso_639_3) + pycountry.languages.get(iso639_3_code=lang_str_iso_639_3) except KeyError: return None - lang_str_iso_639_1 = lang.iso639_1_code.encode('utf-8') - result = 'name:%s' % lang_str_iso_639_1 - return result + return lang_str_iso_639_3 + + +def _convert_osm_l10n_name(x): + # first try a 639-1 code + try: + lang = pycountry.languages.get(iso639_1_code=x) + except KeyError: + # next, try a 639-2 code + try: + lang = pycountry.languages.get(iso639_2_code=x) + except KeyError: + # finally, try a 639-3 code + try: + lang = pycountry.languages.get(iso639_3_code=x) + except KeyError: + return None + iso639_3_code = lang.iso639_3_code.encode('utf-8') + return iso639_3_code def tags_name_i18n(shape, properties, fid, zoom): @@ -374,20 +390,31 @@ def tags_name_i18n(shape, properties, fid, zoom): if not name: return shape, properties, fid - is_wof = properties.get('source') == 'whosonfirst.mapzen.com' + source = properties.get('source') + is_wof = source == 'whosonfirst.mapzen.com' + is_osm = source == 'openstreetmap.org' + + alt_name_prefix_candidates = [] + convert_fn = lambda x: None + if is_osm: + alt_name_prefix_candidates = [ + 'name:left:', 'name:right:', 'name:', 'alt_name:', 'old_name:' + ] + convert_fn = _convert_osm_l10n_name + elif is_wof: + alt_name_prefix_candidates = ['name:'] + convert_fn = _convert_wof_l10n_name for k, v in tags.items(): - if (k.startswith('name:') and v != name or - k.startswith('alt_name:') and v != name or - k.startswith('alt_name_') and v != name or - k.startswith('old_name:') and v != name or - k.startswith('left:name:') and v != name or - k.startswith('right:name:') and v != name): - if is_wof: - k = _convert_wof_l10n_name(k) - if k is None: - continue - properties[k] = v + if v == name: + continue + for candidate in alt_name_prefix_candidates: + if k.startswith(candidate): + lang_code = k[len(candidate):] + normalized_lang_code = convert_fn(lang_code) + if normalized_lang_code: + lang_key = '%s%s' % (candidate, normalized_lang_code) + properties[lang_key] = v for alt_tag_name_candidate in tag_name_alternates: alt_tag_name_value = tags.get(alt_tag_name_candidate) From 1ffe42094f7aff6ba91883fda024a3b999cd0072 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Thu, 19 May 2016 09:55:19 -0400 Subject: [PATCH 337/344] Correct iso-639-2 lookup --- TileStache/Goodies/VecTiles/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index c0cfb622..b7486a56 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -370,7 +370,7 @@ def _convert_osm_l10n_name(x): except KeyError: # next, try a 639-2 code try: - lang = pycountry.languages.get(iso639_2_code=x) + lang = pycountry.languages.get(iso639_2T_code=x) except KeyError: # finally, try a 639-3 code try: From aaf5dfc30960a517c27cea3fade773b82f2133c9 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Fri, 20 May 2016 11:08:03 -0400 Subject: [PATCH 338/344] Support l10n lookups and country suffixes --- TileStache/Goodies/VecTiles/transform.py | 53 +++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index b7486a56..545069b8 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -363,7 +363,7 @@ def _convert_wof_l10n_name(x): return lang_str_iso_639_3 -def _convert_osm_l10n_name(x): +def _normalize_osm_lang_code(x): # first try a 639-1 code try: lang = pycountry.languages.get(iso639_1_code=x) @@ -381,6 +381,57 @@ def _convert_osm_l10n_name(x): return iso639_3_code +def _normalize_country_code(x): + x = x.upper() + try: + c = pycountry.countries.get(alpha2=x) + except KeyError: + try: + c = pycountry.countries.get(alpha3=x) + except KeyError: + try: + c = pycountry.countries.get(numeric_code=x) + except KeyError: + return None + alpha2_code = c.alpha2 + return alpha2_code + + +osm_l10n_lookup = { + 'zh-min-nan': 'nan', + 'zh-yue': 'yue', +} + + +def osm_l10n_name_lookup(x): + lookup = osm_l10n_lookup.get(x) + if lookup is not None: + return lookup + else: + return x + + +def _convert_osm_l10n_name(x): + x = osm_l10n_name_lookup(x) + + if '_' not in x: + return _normalize_osm_lang_code(x) + + fields_by_underscore = x.split('_', 1) + lang_code_candidate, country_candidate = fields_by_underscore + + lang_code_result = _normalize_osm_lang_code(lang_code_candidate) + if lang_code_result is None: + return None + + country_result = _normalize_country_code(country_candidate) + if country_result is None: + return None + + result = '%s_%s' % (lang_code_result, country_result) + return result + + def tags_name_i18n(shape, properties, fid, zoom): tags = properties.get('tags') if not tags: From d9083676d6844dccd81ed5c2e90065a892761070 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 18 May 2016 13:22:44 -0400 Subject: [PATCH 339/344] Removed pois transforms transitioned to yaml --- TileStache/Goodies/VecTiles/transform.py | 53 ------------------------ 1 file changed, 53 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 9e5b6fe3..af979035 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -198,16 +198,6 @@ def building_trim_properties(shape, properties, fid, zoom): return shape, properties, fid -def pois_kind_aeroway_gate(shape, properties, fid, zoom): - aeroway = properties.pop('aeroway', None) - if aeroway is None: - return shape, properties, fid - kind = properties.get('kind') - if kind == 'gate' and aeroway: - properties['aeroway'] = aeroway - return shape, properties, fid - - def road_kind(shape, properties, fid, zoom): source = properties.get('source') assert source, 'Missing source in road query' @@ -2704,25 +2694,6 @@ def add_uic_ref(shape, properties, fid, zoom): return shape, properties, fid -def normalize_leisure_kind(shape, properties, fid, zoom): - """ - Normalise the various ways of representing fitness POIs to a - single kind=fitness. - """ - - kind = properties.get('kind') - if kind in ('fitness_centre', 'gym'): - properties['kind'] = 'fitness' - - elif kind == 'sports_centre': - sport = properties.get('sport') - if sport in ('fitness', 'gym'): - properties.pop('sport') - properties['kind'] = 'fitness' - - return shape, properties, fid - - def merge_features(ctx): """ Merge (linear) features with the same properties together, attempting to @@ -3156,30 +3127,6 @@ def update_parenthetical_properties(ctx): return layer -def add_state_to_stations(shape, properties, fid, zoom): - """ - If the feature is a station, and it has a state tag, then move that - tag to its properties. - """ - - kind = properties.get('kind') - assert kind, "WTF: kind should be set on [%r]: %r" % (fid, properties) - if kind not in ('station'): - return shape, properties, fid - - tags = properties.get('tags') - if not tags: - return shape, properties, fid - - state = tags.get('state') - if not state: - return shape, properties, fid - - properties['state'] = state - - return shape, properties, fid - - def height_to_meters(shape, props, fid, zoom): """ If the properties has a "height" entry, then convert that to meters. From 286fe07c0043ad9f82853002aa59baad4c3f945f Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 18 May 2016 13:23:26 -0400 Subject: [PATCH 340/344] Remove unused road_kind transform --- TileStache/Goodies/VecTiles/transform.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index af979035..d4ecbaad 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -198,16 +198,6 @@ def building_trim_properties(shape, properties, fid, zoom): return shape, properties, fid -def road_kind(shape, properties, fid, zoom): - source = properties.get('source') - assert source, 'Missing source in road query' - if source == 'naturalearthdata.com': - return shape, properties, fid - - properties['kind'] = _road_kind(properties) - return shape, properties, fid - - def road_classifier(shape, properties, fid, zoom): source = properties.get('source') assert source, 'Missing source in road query' From 8d6c8e71d40019a92d01c4a84b1de542fa531cc5 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 25 May 2016 11:36:04 -0400 Subject: [PATCH 341/344] Add transform to drop properties with prefix --- TileStache/Goodies/VecTiles/transform.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 4657626a..7e69b54a 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -3280,3 +3280,20 @@ def add_is_bicycle_related(shape, props, fid, zoom): props.get('highway') == 'cycleway'): props['is_bicycle_related'] = True return shape, props, fid + + +def drop_properties_with_prefix(ctx): + """ + Iterate through all features, dropping all properties that start + with prefix. + """ + + prefix = ctx.params.get('prefix') + assert prefix, 'drop_properties_with_prefix: missing prefix param' + + feature_layers = ctx.feature_layers + for feature_layer in feature_layers: + for shape, props, fid in feature_layer['features']: + for k in props.keys(): + if k.startswith(prefix): + del props[k] From 5d2f029acc812dd8d577ec5d40b8b3b49bda17b1 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 25 May 2016 13:18:31 -0400 Subject: [PATCH 342/344] Remove no longer necessary drop_keys --- TileStache/Goodies/VecTiles/transform.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/TileStache/Goodies/VecTiles/transform.py b/TileStache/Goodies/VecTiles/transform.py index 7e69b54a..004515fb 100644 --- a/TileStache/Goodies/VecTiles/transform.py +++ b/TileStache/Goodies/VecTiles/transform.py @@ -1825,16 +1825,12 @@ def generate_label_features(ctx): label_property_name = ctx.params.get('label_property_name') label_property_value = ctx.params.get('label_property_value') new_layer_name = ctx.params.get('new_layer_name') - drop_keys = ctx.params.get('drop_keys') geom_types = ctx.params.get('geom_types', ['Polygon', 'MultiPolygon']) layer = _find_layer(feature_layers, source_layer) if layer is None: return None - if drop_keys is None: - drop_keys = [] - new_features = [] for feature in layer['features']: shape, properties, fid = feature @@ -1868,12 +1864,6 @@ def generate_label_features(ctx): label_properties = properties.copy() - # drop particular keys which might not be relevant any more. - # for example, mz_is_building, which is used by a later - # polygon processing stage, but irrelevant to label processing. - for k in drop_keys: - label_properties.pop(k, None) - if label_property_name: label_properties[label_property_name] = label_property_value From 03f311dfc5a7e1f00b229b1e298c235c93a878c0 Mon Sep 17 00:00:00 2001 From: Robert Marianski Date: Wed, 3 Aug 2016 16:00:28 -0400 Subject: [PATCH 343/344] Add note that this repo is deprecated now closes #166 --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 376b9646..96310d7a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,8 @@ -#TileStache +# Note: This repository is deprecated, and no longer supported. + +To serve tiles, please have a look at our [documentation for getting started](https://github.com/tilezen/vector-datasource/wiki/Mapzen-Vector-Tile-Service). + +##TileStache _a stylish alternative for caching your map tiles_ From bf00ae2bbbf5904af5bc741eb6082216151aab90 Mon Sep 17 00:00:00 2001 From: ThomasG77 Date: Sat, 24 Sep 2016 14:25:30 +0200 Subject: [PATCH 344/344] Fix ValueError: too many values to unpack when producing MVT tiles --- TileStache/Goodies/VecTiles/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TileStache/Goodies/VecTiles/server.py b/TileStache/Goodies/VecTiles/server.py index b5aed535..3072aa62 100644 --- a/TileStache/Goodies/VecTiles/server.py +++ b/TileStache/Goodies/VecTiles/server.py @@ -339,7 +339,7 @@ def save(self, out, format): features = get_features(self.dbinfo, self.query[format], self.geometry_types, self.transform_fn, self.sort_fn, self.coord.zoom) if format == 'MVT': - mvt.encode(out, features, self.coord, self.layer_name) + mvt.encode(out, features, self.coord, self.bounds, self.layer_name) elif format == 'JSON': geojson.encode(out, features, self.zoom)