Skip to content

Feature/tiles cache expiration #108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
edramos-97 wants to merge 8 commits into
base: develop
Choose a base branch
from
Open

Feature/tiles cache expiration #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
defd878
renderer2d/tiles: Change openstreetmap url and reduce sleep for testing
Jul 23, 2019
990b117
renderer2d/tiles: Add parsing of cache-control header to tile expiration
Jul 23, 2019
e70402c
renderer2d/tiles: Add expiration to file metadata and dynamic build o…
Jul 24, 2019
b335075
renderer2d/tiles: Add expiration reading from file
Jul 25, 2019
64658bc
renderer2d/tiles: Add parsing for "Expires" header
Jul 25, 2019
7b0abcd
renderer2d/tiles: Add permanent storage and loading of cache data
Jul 25, 2019
7ad5cee
renderer2d/tiles: Improve debug information format to write to NeptusLog
Jul 28, 2019
55a014a
renderer2d/tiles: Restore original sleep time in TileHttpFetcher
Jul 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,8 +302,10 @@ else if (i == lst.size() - 1) {
for (String mapDefTag : list) {
String[] tags = mapDefTag.split(":");
String mapDef = tags[0];
if (mapActiveHolderList.containsKey(mapDef))
if (mapActiveHolderList.containsKey(mapDef)) {
mapActiveHolderList.put(mapDef, true);
Tile.setCache(mapDef,true);
}
if (mapLayerPrioriryHolderList.containsKey(mapDef) && tags.length > 1) {
try {
short prio = Short.parseShort(tags[1]);
Expand Down Expand Up @@ -409,6 +411,7 @@ public void run() {
public void run() {
clearMemCache();

Tile.cleanup();
ttask.cancel();
ttask1.cancel();
timer.cancel();
Expand Down Expand Up @@ -488,6 +491,7 @@ public void onSelectedChange(boolean selected) {

for (String key : mapStyle) {
if (mapActiveHolderList.containsKey(key)) {
Tile.setCache(key, true);
mapActiveHolderList.put(key, true);
}
}
Expand Down Expand Up @@ -664,10 +668,12 @@ public void setMapStyle(boolean exclusive, boolean activate, String... mapStyleN
for (String mapKey : mapActiveHolderList.keySet()) {
if (mapKey.equalsIgnoreCase(mapStyle)) {
mapActiveHolderList.put(mapKey, activate);
Tile.setCache(mapKey, activate);
}
else {
if (exclusive && !mapStyleList.contains(mapKey)) {
mapActiveHolderList.put(mapKey, !activate);
Tile.setCache(mapKey, !activate);
}
}
}
Expand Down
221 changes: 211 additions & 10 deletions src/pt/lsts/neptus/renderer2d/tiles/Tile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,24 +32,46 @@
*/
package pt.lsts.neptus.renderer2d.tiles;

import com.sun.imageio.plugins.png.PNGMetadata;
import java.awt.AlphaComposite;
import java.awt.Color;
import java.awt.Font;
import java.awt.Graphics2D;
import java.awt.Image;
import java.awt.geom.Point2D;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.Vector;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import javax.imageio.IIOImage;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.ImageTypeSpecifier;
import javax.imageio.ImageWriteParam;
import javax.imageio.ImageWriter;
import javax.imageio.metadata.IIOMetadata;
import javax.imageio.metadata.IIOMetadataNode;
import javax.imageio.stream.ImageOutputStream;

import org.apache.commons.io.FileUtils;

import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import pt.lsts.neptus.NeptusLog;
import pt.lsts.neptus.gui.PropertiesEditor;
import pt.lsts.neptus.i18n.I18n;
Expand Down Expand Up @@ -78,14 +100,16 @@ public abstract class Tile implements /*Renderer2DPainter,*/ Serializable {

protected static String TILE_BASE_CACHE_DIR;

{
static {
if (new File("../" + ".cache/wmcache").exists())
TILE_BASE_CACHE_DIR = "../" + ".cache/wmcache";
else
TILE_BASE_CACHE_DIR = ".cache/wmcache";
}

protected static final String TILE_FX_EXTENSION = "png";

static HashMap<String, HashMap<String, Long>> cacheExpiration = new HashMap<>();

public static final long MILISECONDS_TO_TILE_MEM_REMOVAL = 20000;
private static final int MILLIS_TO_NOT_TRY_LOAD_LOW_LEVEL_IMAGE = 30000;
Expand All @@ -103,6 +127,7 @@ public enum TileState { LOADING, RETRYING, LOADED, ERROR, FATAL_ERROR, DISPOSING
public final int levelOfDetail;
public final int tileX, tileY;
public final int worldX, worldY;
public long expiration;
protected BufferedImage image = null;
protected boolean temporaryTransparencyDetectedOnImageOnDisk = false; //only for base layers
private boolean showTileId = false;
Expand All @@ -114,7 +139,11 @@ public enum TileState { LOADING, RETRYING, LOADED, ERROR, FATAL_ERROR, DISPOSING

private Timer timer = null; // new Timer(this.getClass().getSimpleName() + " [" + Integer.toHexString(this.hashCode()) + "]");
private TimerTask timerTask = null;


private static Timer saveTimer = new Timer("TileExpirationMapSaveTimer");
private static AtomicBoolean hasSaveTimer = new AtomicBoolean(false);
private static final long SAVE_INTERVAL = 120000; //2 minutes

/**
* @param levelOfDetail
* @param tileX
Expand Down Expand Up @@ -391,8 +420,62 @@ public boolean saveTile() {
tileCacheDiskClearOrTileSaveLock.readLock().lock();
try {
File outFile = new File(getTileFilePath());
outFile.mkdirs();
return ImageIO.write(image, TILE_FX_EXTENSION.toUpperCase(), outFile);
outFile.getParentFile().mkdirs();
outFile.createNewFile();
NeptusLog.pub().debug("Saving expiration date for tile: " + getId() + " from map: " + getClass().getSimpleName());
NeptusLog.pub().debug("expiration = " + new Date(expiration));


// https://docs.oracle.com/javase/8/docs/api/javax/imageio/metadata/doc-files/png_metadata.html
ImageWriter writer = ImageIO.getImageWritersByFormatName("png").next();

ImageWriteParam writeParam = writer.getDefaultWriteParam();
ImageTypeSpecifier typeSpecifier = ImageTypeSpecifier.createFromBufferedImageType(BufferedImage.TYPE_INT_RGB);

//adding metadata
IIOMetadata metadata = writer.getDefaultImageMetadata(typeSpecifier, writeParam);

IIOMetadataNode textEntry = new IIOMetadataNode("tEXtEntry");
textEntry.setAttribute("keyword", "expiration");
textEntry.setAttribute("value", Long.toString(expiration));

IIOMetadataNode text = new IIOMetadataNode("tEXt");
text.appendChild(textEntry);

IIOMetadataNode root = new IIOMetadataNode("javax_imageio_png_1.0");
root.appendChild(text);

metadata.mergeTree("javax_imageio_png_1.0", root);

//writing the data
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ImageOutputStream stream = ImageIO.createImageOutputStream(baos);
writer.setOutput(stream);
writer.write(metadata, new IIOImage(image, null, metadata), writeParam);
stream.close();

FileUtils.writeByteArrayToFile(outFile, baos.toByteArray());

HashMap<String, Long> currMapStyleCache = cacheExpiration.get(getClass().getAnnotation(MapTileProvider.class).name());
if(currMapStyleCache != null){
currMapStyleCache.put(id, expiration);
} else {
HashMap<String, Long> newMap = new HashMap<>();
newMap.put(id, expiration);
cacheExpiration.put(getClass().getAnnotation(MapTileProvider.class).name(), newMap);
}
if(!hasSaveTimer.get()) {
saveTimer.schedule(new TimerTask() {
@Override
public void run() {
saveCacheExpiration();
hasSaveTimer.set(false);
}
}, SAVE_INTERVAL);
hasSaveTimer.set(true);
}

return true;
}
catch (Exception e) {
e.printStackTrace();
Expand All @@ -415,12 +498,21 @@ public boolean loadTile() {
if (image == null)
state = TileState.LOADING;
File inFile = new File(getTileFilePath());
if (!inFile.exists()) {
lasErrorMessage = "Error loading tile from file not existing!";
if (image == null)
state = TileState.ERROR;
// scheduleLoadImageFromLowerLevelOfDetail();
return false;

if(hasExpired()){
if (!inFile.exists()) {
lasErrorMessage = "Error loading tile from file not existing!";
if (image == null)
state = TileState.ERROR;
// scheduleLoadImageFromLowerLevelOfDetail();
return false;
} else {
NeptusLog.pub().debug("Checking file expiration");
if(hasExpired(inFile)){
state = TileState.ERROR;
return false;
}
}
}

BufferedImage img;
Expand Down Expand Up @@ -449,6 +541,115 @@ public boolean loadTile() {
}
}

private boolean hasExpired() {
HashMap<String, Long> currMapStyleCache = cacheExpiration.get(getClass().getAnnotation(MapTileProvider.class).name());
Long expiration = currMapStyleCache.get(id);
if(expiration != null) {
return expiration <= System.currentTimeMillis();
} else {
return true;
}
}

private boolean hasExpired(File inFile) throws IOException {
// https://docs.oracle.com/javase/8/docs/api/javax/imageio/metadata/doc-files/png_metadata.html
ImageReader imageReader = ImageIO.getImageReadersByFormatName("png").next();

imageReader.setInput(ImageIO.createImageInputStream(inFile), true);

// read metadata of first image
IIOMetadata metadata = imageReader.getImageMetadata(0);

// the PNG image reader already create a PNGMetadata Object
PNGMetadata pngmeta = (PNGMetadata) metadata;
NodeList childNodes = pngmeta.getStandardTextNode().getChildNodes();

for (int i = 0; i < childNodes.getLength(); i++) {
Node node = childNodes.item(i);
String keyword = node.getAttributes().getNamedItem("keyword").getNodeValue();
String value = node.getAttributes().getNamedItem("value").getNodeValue();
if("expiration".equals(keyword)){
try {
expiration = Long.valueOf(value);

// add new entry to cache map
HashMap<String, Long> currMapStyleCache = cacheExpiration.get(getClass().getAnnotation(MapTileProvider.class).name());
if(currMapStyleCache != null){
currMapStyleCache.put(id, expiration);
} else {
HashMap<String, Long> newMap = new HashMap<>();
newMap.put(id, expiration);
cacheExpiration.put(getClass().getAnnotation(MapTileProvider.class).name(), newMap);
}

return expiration <= System.currentTimeMillis();
} catch (NumberFormatException e) {
NeptusLog.pub().info(String.format("Could not load expiration metadata for map tile %s of style '%s'",
id,
getClass().getSimpleName()));
return true;
}
}
}
return true;
}

public static void setCache(String mapKey, boolean state) {
if(state) {
cacheExpiration.put(mapKey,loadCacheExpiration(mapKey));
} else {
cacheExpiration.remove(mapKey);
}
}

private static HashMap<String, Long> loadCacheExpiration(String mapKey) {
NeptusLog.pub().info("Loading cache file for: " + mapKey);
File serFile = new File(TILE_BASE_CACHE_DIR + "/serializedCaches/" + mapKey);
if(!serFile.exists()) {
NeptusLog.pub().error(String.format("No cache expiration found at '%s'", serFile.getPath()));
return new HashMap<>();
}

try (FileInputStream fis = new FileInputStream(serFile);
ObjectInputStream ois = new ObjectInputStream(fis)) {
Object savedObject = ois.readObject();
if (savedObject instanceof HashMap){
return ((HashMap) savedObject);
} else {
throw new Exception("Saved Object is not instance of HashMap");
}
} catch(Exception e) {
NeptusLog.pub().error("An error occurred while saving cache expiration data");
e.printStackTrace();
return new HashMap<>();
}
}

private static void saveCacheExpiration() {
NeptusLog.pub().debug("Saving tile cache");
for (Map.Entry<String, HashMap<String, Long>> mapEntry : cacheExpiration.entrySet()) {
NeptusLog.pub().debug("Saving Map Style: " + mapEntry.getKey());
try {
File serFile = new File(TILE_BASE_CACHE_DIR + "/serializedCaches/" + mapEntry.getKey());
serFile.getParentFile().mkdirs();
serFile.createNewFile();
FileOutputStream fos = new FileOutputStream(serFile);
ObjectOutputStream oos = new ObjectOutputStream(fos);
oos.writeObject(mapEntry.getValue());
oos.close();
fos.close();
} catch(IOException ioe) {
NeptusLog.pub().error("An error occurred while saving cache expiration data for map style: " + mapEntry.getKey());
ioe.printStackTrace();
}
}
}

public static void cleanup() {
saveTimer.cancel();
saveCacheExpiration();
}

/**
* @param img
*/
Expand Down
Loading