diff --git a/src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java b/src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java index d4aef50489..f2e67c6b5b 100644 --- a/src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java +++ b/src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java @@ -302,8 +302,10 @@ else if (i == lst.size() - 1) { for (String mapDefTag : list) { String[] tags = mapDefTag.split(":"); String mapDef = tags[0]; - if (mapActiveHolderList.containsKey(mapDef)) + if (mapActiveHolderList.containsKey(mapDef)) { mapActiveHolderList.put(mapDef, true); + Tile.setCache(mapDef,true); + } if (mapLayerPrioriryHolderList.containsKey(mapDef) && tags.length > 1) { try { short prio = Short.parseShort(tags[1]); @@ -409,6 +411,7 @@ public void run() { public void run() { clearMemCache(); + Tile.cleanup(); ttask.cancel(); ttask1.cancel(); timer.cancel(); @@ -488,6 +491,7 @@ public void onSelectedChange(boolean selected) { for (String key : mapStyle) { if (mapActiveHolderList.containsKey(key)) { + Tile.setCache(key, true); mapActiveHolderList.put(key, true); } } @@ -664,10 +668,12 @@ public void setMapStyle(boolean exclusive, boolean activate, String... mapStyleN for (String mapKey : mapActiveHolderList.keySet()) { if (mapKey.equalsIgnoreCase(mapStyle)) { mapActiveHolderList.put(mapKey, activate); + Tile.setCache(mapKey, activate); } else { if (exclusive && !mapStyleList.contains(mapKey)) { mapActiveHolderList.put(mapKey, !activate); + Tile.setCache(mapKey, !activate); } } } diff --git a/src/pt/lsts/neptus/renderer2d/tiles/Tile.java b/src/pt/lsts/neptus/renderer2d/tiles/Tile.java index c0417c2b4e..b1a5e4743e 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/Tile.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/Tile.java @@ -32,6 +32,7 @@ */ package pt.lsts.neptus.renderer2d.tiles; +import com.sun.imageio.plugins.png.PNGMetadata; import java.awt.AlphaComposite; import java.awt.Color; import java.awt.Font; @@ -39,17 +40,38 @@ import java.awt.Image; import java.awt.geom.Point2D; import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.io.Serializable; +import java.util.Date; +import java.util.HashMap; import java.util.Map; import java.util.Timer; import java.util.TimerTask; import java.util.Vector; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; +import javax.imageio.IIOImage; import javax.imageio.ImageIO; +import javax.imageio.ImageReader; +import javax.imageio.ImageTypeSpecifier; +import javax.imageio.ImageWriteParam; +import javax.imageio.ImageWriter; +import javax.imageio.metadata.IIOMetadata; +import javax.imageio.metadata.IIOMetadataNode; +import javax.imageio.stream.ImageOutputStream; +import org.apache.commons.io.FileUtils; + +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; import pt.lsts.neptus.NeptusLog; import pt.lsts.neptus.gui.PropertiesEditor; import pt.lsts.neptus.i18n.I18n; @@ -78,7 +100,7 @@ public abstract class Tile implements /*Renderer2DPainter,*/ Serializable { protected static String TILE_BASE_CACHE_DIR; - { + static { if (new File("../" + ".cache/wmcache").exists()) TILE_BASE_CACHE_DIR = "../" + ".cache/wmcache"; else @@ -86,6 +108,8 @@ public abstract class Tile implements /*Renderer2DPainter,*/ Serializable { } protected static final String TILE_FX_EXTENSION = "png"; + + static HashMap> cacheExpiration = new HashMap<>(); public static final long MILISECONDS_TO_TILE_MEM_REMOVAL = 20000; private static final int MILLIS_TO_NOT_TRY_LOAD_LOW_LEVEL_IMAGE = 30000; @@ -103,6 +127,7 @@ public enum TileState { LOADING, RETRYING, LOADED, ERROR, FATAL_ERROR, DISPOSING public final int levelOfDetail; public final int tileX, tileY; public final int worldX, worldY; + public long expiration; protected BufferedImage image = null; protected boolean temporaryTransparencyDetectedOnImageOnDisk = false; //only for base layers private boolean showTileId = false; @@ -114,7 +139,11 @@ public enum TileState { LOADING, RETRYING, LOADED, ERROR, FATAL_ERROR, DISPOSING private Timer timer = null; // new Timer(this.getClass().getSimpleName() + " [" + Integer.toHexString(this.hashCode()) + "]"); private TimerTask timerTask = null; - + + private static Timer saveTimer = new Timer("TileExpirationMapSaveTimer"); + private static AtomicBoolean hasSaveTimer = new AtomicBoolean(false); + private static final long SAVE_INTERVAL = 120000; //2 minutes + /** * @param levelOfDetail * @param tileX @@ -391,8 +420,62 @@ public boolean saveTile() { tileCacheDiskClearOrTileSaveLock.readLock().lock(); try { File outFile = new File(getTileFilePath()); - outFile.mkdirs(); - return ImageIO.write(image, TILE_FX_EXTENSION.toUpperCase(), outFile); + outFile.getParentFile().mkdirs(); + outFile.createNewFile(); + NeptusLog.pub().debug("Saving expiration date for tile: " + getId() + " from map: " + getClass().getSimpleName()); + NeptusLog.pub().debug("expiration = " + new Date(expiration)); + + + // https://docs.oracle.com/javase/8/docs/api/javax/imageio/metadata/doc-files/png_metadata.html + ImageWriter writer = ImageIO.getImageWritersByFormatName("png").next(); + + ImageWriteParam writeParam = writer.getDefaultWriteParam(); + ImageTypeSpecifier typeSpecifier = ImageTypeSpecifier.createFromBufferedImageType(BufferedImage.TYPE_INT_RGB); + + //adding metadata + IIOMetadata metadata = writer.getDefaultImageMetadata(typeSpecifier, writeParam); + + IIOMetadataNode textEntry = new IIOMetadataNode("tEXtEntry"); + textEntry.setAttribute("keyword", "expiration"); + textEntry.setAttribute("value", Long.toString(expiration)); + + IIOMetadataNode text = new IIOMetadataNode("tEXt"); + text.appendChild(textEntry); + + IIOMetadataNode root = new IIOMetadataNode("javax_imageio_png_1.0"); + root.appendChild(text); + + metadata.mergeTree("javax_imageio_png_1.0", root); + + //writing the data + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ImageOutputStream stream = ImageIO.createImageOutputStream(baos); + writer.setOutput(stream); + writer.write(metadata, new IIOImage(image, null, metadata), writeParam); + stream.close(); + + FileUtils.writeByteArrayToFile(outFile, baos.toByteArray()); + + HashMap currMapStyleCache = cacheExpiration.get(getClass().getAnnotation(MapTileProvider.class).name()); + if(currMapStyleCache != null){ + currMapStyleCache.put(id, expiration); + } else { + HashMap newMap = new HashMap<>(); + newMap.put(id, expiration); + cacheExpiration.put(getClass().getAnnotation(MapTileProvider.class).name(), newMap); + } + if(!hasSaveTimer.get()) { + saveTimer.schedule(new TimerTask() { + @Override + public void run() { + saveCacheExpiration(); + hasSaveTimer.set(false); + } + }, SAVE_INTERVAL); + hasSaveTimer.set(true); + } + + return true; } catch (Exception e) { e.printStackTrace(); @@ -415,12 +498,21 @@ public boolean loadTile() { if (image == null) state = TileState.LOADING; File inFile = new File(getTileFilePath()); - if (!inFile.exists()) { - lasErrorMessage = "Error loading tile from file not existing!"; - if (image == null) - state = TileState.ERROR; -// scheduleLoadImageFromLowerLevelOfDetail(); - return false; + + if(hasExpired()){ + if (!inFile.exists()) { + lasErrorMessage = "Error loading tile from file not existing!"; + if (image == null) + state = TileState.ERROR; + // scheduleLoadImageFromLowerLevelOfDetail(); + return false; + } else { + NeptusLog.pub().debug("Checking file expiration"); + if(hasExpired(inFile)){ + state = TileState.ERROR; + return false; + } + } } BufferedImage img; @@ -449,6 +541,115 @@ public boolean loadTile() { } } + private boolean hasExpired() { + HashMap currMapStyleCache = cacheExpiration.get(getClass().getAnnotation(MapTileProvider.class).name()); + Long expiration = currMapStyleCache.get(id); + if(expiration != null) { + return expiration <= System.currentTimeMillis(); + } else { + return true; + } + } + + private boolean hasExpired(File inFile) throws IOException { + // https://docs.oracle.com/javase/8/docs/api/javax/imageio/metadata/doc-files/png_metadata.html + ImageReader imageReader = ImageIO.getImageReadersByFormatName("png").next(); + + imageReader.setInput(ImageIO.createImageInputStream(inFile), true); + + // read metadata of first image + IIOMetadata metadata = imageReader.getImageMetadata(0); + + // the PNG image reader already create a PNGMetadata Object + PNGMetadata pngmeta = (PNGMetadata) metadata; + NodeList childNodes = pngmeta.getStandardTextNode().getChildNodes(); + + for (int i = 0; i < childNodes.getLength(); i++) { + Node node = childNodes.item(i); + String keyword = node.getAttributes().getNamedItem("keyword").getNodeValue(); + String value = node.getAttributes().getNamedItem("value").getNodeValue(); + if("expiration".equals(keyword)){ + try { + expiration = Long.valueOf(value); + + // add new entry to cache map + HashMap currMapStyleCache = cacheExpiration.get(getClass().getAnnotation(MapTileProvider.class).name()); + if(currMapStyleCache != null){ + currMapStyleCache.put(id, expiration); + } else { + HashMap newMap = new HashMap<>(); + newMap.put(id, expiration); + cacheExpiration.put(getClass().getAnnotation(MapTileProvider.class).name(), newMap); + } + + return expiration <= System.currentTimeMillis(); + } catch (NumberFormatException e) { + NeptusLog.pub().info(String.format("Could not load expiration metadata for map tile %s of style '%s'", + id, + getClass().getSimpleName())); + return true; + } + } + } + return true; + } + + public static void setCache(String mapKey, boolean state) { + if(state) { + cacheExpiration.put(mapKey,loadCacheExpiration(mapKey)); + } else { + cacheExpiration.remove(mapKey); + } + } + + private static HashMap loadCacheExpiration(String mapKey) { + NeptusLog.pub().info("Loading cache file for: " + mapKey); + File serFile = new File(TILE_BASE_CACHE_DIR + "/serializedCaches/" + mapKey); + if(!serFile.exists()) { + NeptusLog.pub().error(String.format("No cache expiration found at '%s'", serFile.getPath())); + return new HashMap<>(); + } + + try (FileInputStream fis = new FileInputStream(serFile); + ObjectInputStream ois = new ObjectInputStream(fis)) { + Object savedObject = ois.readObject(); + if (savedObject instanceof HashMap){ + return ((HashMap) savedObject); + } else { + throw new Exception("Saved Object is not instance of HashMap"); + } + } catch(Exception e) { + NeptusLog.pub().error("An error occurred while saving cache expiration data"); + e.printStackTrace(); + return new HashMap<>(); + } + } + + private static void saveCacheExpiration() { + NeptusLog.pub().debug("Saving tile cache"); + for (Map.Entry> mapEntry : cacheExpiration.entrySet()) { + NeptusLog.pub().debug("Saving Map Style: " + mapEntry.getKey()); + try { + File serFile = new File(TILE_BASE_CACHE_DIR + "/serializedCaches/" + mapEntry.getKey()); + serFile.getParentFile().mkdirs(); + serFile.createNewFile(); + FileOutputStream fos = new FileOutputStream(serFile); + ObjectOutputStream oos = new ObjectOutputStream(fos); + oos.writeObject(mapEntry.getValue()); + oos.close(); + fos.close(); + } catch(IOException ioe) { + NeptusLog.pub().error("An error occurred while saving cache expiration data for map style: " + mapEntry.getKey()); + ioe.printStackTrace(); + } + } + } + + public static void cleanup() { + saveTimer.cancel(); + saveCacheExpiration(); + } + /** * @param img */ diff --git a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java index 3a8c7f23e4..04210cf93c 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/TileHttpFetcher.java @@ -35,6 +35,8 @@ import java.awt.image.BufferedImage; import java.io.InputStream; import java.net.UnknownHostException; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Random; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -44,6 +46,8 @@ import javax.imageio.ImageIO; import javax.imageio.stream.ImageInputStream; +import org.apache.http.Header; +import org.apache.http.HeaderElement; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.methods.HttpGet; @@ -287,7 +291,10 @@ else if (resp.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { } retries = 0; isInStateForbidden = false; - + + NeptusLog.pub().debug("Fetched tile from: " + urlGet); + parseExpirationHeader(resp, urlGet); + InputStream is = resp.getEntity().getContent(); ImageInputStream iis = ImageIO.createImageInputStream(is); @@ -340,7 +347,7 @@ public void run() { } finally { get.abort(); - + // Mostly for GoogleMaps wait time between connections if (waitTime > 0) { if (getWaitTimeLock() != null) { @@ -408,6 +415,33 @@ public void run() { } } + private void parseExpirationHeader(HttpResponse resp, String urlGet) { + Header cacheControlHeader = resp.getFirstHeader("cache-control"); + if(cacheControlHeader != null) { + for (HeaderElement cacheInfo : cacheControlHeader.getElements()){ + if(cacheInfo.getName().equalsIgnoreCase("max-age")){ + try { + expiration = System.currentTimeMillis() + 1000 * Long.parseLong(cacheInfo.getValue()); + return; + } catch (NumberFormatException e) { + NeptusLog.pub().info("An error occurred while parsing cache control:max-age from request url: " + urlGet); + } + } + } + } + + Header expiresHeader = resp.getFirstHeader("Expires"); + if(expiration < System.currentTimeMillis() && expiresHeader != null) { + try { + expiration = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz").parse(expiresHeader.getValue()).getTime(); + return; + } catch (ParseException e){ + NeptusLog.pub().info("An error occurred while parsing expires header from request url: " + urlGet); + } + } + expiration = System.currentTimeMillis() + 86400000;// one day + } + // --------------- Less threads for loading tiles code (test) ------------------------ private static final boolean DEBUG = false; private static boolean useThreadPool = true; @@ -636,6 +670,9 @@ else if (resp.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { retries = 0; isInStateForbidden = false; + NeptusLog.pub().debug("Fetched tile from: " + urlGet); + parseExpirationHeader(resp, urlGet); + InputStream is = resp.getEntity().getContent(); ImageInputStream iis = ImageIO.createImageInputStream(is); diff --git a/src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java b/src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java index f92d26e3d7..b1b1f782a7 100644 --- a/src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java +++ b/src/pt/lsts/neptus/renderer2d/tiles/TileOpenStreetMap.java @@ -121,7 +121,7 @@ protected String createTileRequestURL() { // urlGet = "http://" + sv + "." + "tile.opencyclemap.org/cycle/" + levelOfDetail + "/" // + tileX + "/" + tileY + ".png"; // else - String urlGet = "http://" + sv + "." + "tile.openstreetmap.org/" + levelOfDetail + "/" + String urlGet = "http://" + sv + ".tile.openstreetmap.org/" + levelOfDetail + "/" + tileX + "/" + tileY + ".png"; return urlGet;