Skip to content

Feature/tiles cache expiration #108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
edramos-97 wants to merge 8 commits into
base: develop
Choose a base branch
from
Open

Feature/tiles cache expiration #108

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
defd878
renderer2d/tiles: Change openstreetmap url and reduce sleep for testing
Jul 23, 2019
990b117
renderer2d/tiles: Add parsing of cache-control header to tile expiration
Jul 23, 2019
e70402c
renderer2d/tiles: Add expiration to file metadata and dynamic build o…
Jul 24, 2019
b335075
renderer2d/tiles: Add expiration reading from file
Jul 25, 2019
64658bc
renderer2d/tiles: Add parsing for "Expires" header
Jul 25, 2019
7b0abcd
renderer2d/tiles: Add permanent storage and loading of cache data
Jul 25, 2019
7ad5cee
renderer2d/tiles: Improve debug information format to write to NeptusLog
Jul 28, 2019
55a014a
renderer2d/tiles: Restore original sleep time in TileHttpFetcher
Jul 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
renderer2d/tiles: Add expiration to file metadata and dynamic build o… 
…f expiration map
  • Loading branch information
edramos-97 committed Jul 24, 2019
commit e70402c85c81c0e8600da48d147dcfa3ae492e17
7 changes: 6 additions & 1 deletion src/pt/lsts/neptus/renderer2d/WorldRenderPainter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,8 +302,10 @@ else if (i == lst.size() - 1) {
for (String mapDefTag : list) {
String[] tags = mapDefTag.split(":");
String mapDef = tags[0];
if (mapActiveHolderList.containsKey(mapDef))
if (mapActiveHolderList.containsKey(mapDef)) {
mapActiveHolderList.put(mapDef, true);
Tile.setCache(mapDef,true);
}
if (mapLayerPrioriryHolderList.containsKey(mapDef) && tags.length > 1) {
try {
short prio = Short.parseShort(tags[1]);
Expand Down Expand Up @@ -488,6 +490,7 @@ public void onSelectedChange(boolean selected) {

for (String key : mapStyle) {
if (mapActiveHolderList.containsKey(key)) {
Tile.setCache(key, true);
mapActiveHolderList.put(key, true);
}
}
Expand Down Expand Up @@ -664,10 +667,12 @@ public void setMapStyle(boolean exclusive, boolean activate, String... mapStyleN
for (String mapKey : mapActiveHolderList.keySet()) {
if (mapKey.equalsIgnoreCase(mapStyle)) {
mapActiveHolderList.put(mapKey, activate);
Tile.setCache(mapKey, activate);
}
else {
if (exclusive && !mapStyleList.contains(mapKey)) {
mapActiveHolderList.put(mapKey, !activate);
Tile.setCache(mapKey, !activate);
}
}
}
Expand Down
107 changes: 99 additions & 8 deletions src/pt/lsts/neptus/renderer2d/tiles/Tile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,27 @@
import java.awt.Image;
import java.awt.geom.Point2D;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.Vector;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import javax.imageio.IIOImage;
import javax.imageio.ImageIO;
import javax.imageio.ImageTypeSpecifier;
import javax.imageio.ImageWriteParam;
import javax.imageio.ImageWriter;
import javax.imageio.metadata.IIOMetadata;
import javax.imageio.metadata.IIOMetadataNode;
import javax.imageio.stream.ImageOutputStream;

import org.apache.commons.io.FileUtils;

import pt.lsts.neptus.NeptusLog;
import pt.lsts.neptus.gui.PropertiesEditor;
Expand Down Expand Up @@ -86,6 +97,8 @@ public abstract class Tile implements /*Renderer2DPainter,*/ Serializable {
}

protected static final String TILE_FX_EXTENSION = "png";

static HashMap<String, HashMap<String, Long>> cacheExpiration = new HashMap<>();

public static final long MILISECONDS_TO_TILE_MEM_REMOVAL = 20000;
private static final int MILLIS_TO_NOT_TRY_LOAD_LOW_LEVEL_IMAGE = 30000;
Expand Down Expand Up @@ -393,9 +406,51 @@ public boolean saveTile() {
try {
File outFile = new File(getTileFilePath());
outFile.mkdirs();
System.out.println("Saving expiration date for tile: " + getId());
outFile.delete();
outFile.createNewFile();
System.out.println("Saving expiration date for tile: " + getId() + " from map: " + getClass().getSimpleName());
System.out.println("expiration = " + expiration);
return ImageIO.write(image, TILE_FX_EXTENSION.toUpperCase(), outFile);


ImageWriter writer = ImageIO.getImageWritersByFormatName("png").next();

ImageWriteParam writeParam = writer.getDefaultWriteParam();
ImageTypeSpecifier typeSpecifier = ImageTypeSpecifier.createFromBufferedImageType(BufferedImage.TYPE_INT_RGB);

//adding metadata
IIOMetadata metadata = writer.getDefaultImageMetadata(typeSpecifier, writeParam);

IIOMetadataNode textEntry = new IIOMetadataNode("tEXtEntry");
textEntry.setAttribute("keyword", "expiration");
textEntry.setAttribute("value", Long.toString(expiration));

IIOMetadataNode text = new IIOMetadataNode("tEXt");
text.appendChild(textEntry);

IIOMetadataNode root = new IIOMetadataNode("javax_imageio_png_1.0");
root.appendChild(text);

metadata.mergeTree("javax_imageio_png_1.0", root);

//writing the data
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ImageOutputStream stream = ImageIO.createImageOutputStream(baos);
writer.setOutput(stream);
writer.write(metadata, new IIOImage(image, null, metadata), writeParam);
stream.close();

FileUtils.writeByteArrayToFile(outFile, baos.toByteArray());

HashMap<String, Long> currMapStyleCache = cacheExpiration.get(getClass().getAnnotation(MapTileProvider.class).name());
if(currMapStyleCache != null){
currMapStyleCache.put(id, expiration);
} else {
HashMap<String, Long> newMap = new HashMap<>();
newMap.put(id, expiration);
cacheExpiration.put(getClass().getAnnotation(MapTileProvider.class).name(), newMap);
}

return true;
}
catch (Exception e) {
e.printStackTrace();
Expand All @@ -418,12 +473,20 @@ public boolean loadTile() {
if (image == null)
state = TileState.LOADING;
File inFile = new File(getTileFilePath());
if (!inFile.exists()) {
lasErrorMessage = "Error loading tile from file not existing!";
if (image == null)
state = TileState.ERROR;
// scheduleLoadImageFromLowerLevelOfDetail();
return false;

if(hasExpired()){
if (!inFile.exists()) {
lasErrorMessage = "Error loading tile from file not existing!";
if (image == null)
state = TileState.ERROR;
// scheduleLoadImageFromLowerLevelOfDetail();
return false;
} else {
if(hasExpired(inFile)){
state = TileState.ERROR;
return false;
}
}
}

BufferedImage img;
Expand Down Expand Up @@ -452,6 +515,34 @@ public boolean loadTile() {
}
}

private boolean hasExpired() {
HashMap<String, Long> currMapStyleCache = cacheExpiration.get(getClass().getAnnotation(MapTileProvider.class).name());
Long expiration = currMapStyleCache.get(id);
if(expiration != null) {
return expiration <= System.currentTimeMillis();
} else {
return true;
}
}

private boolean hasExpired(File inFile) {
// TODO: 24/07/2019 read file metadata field with name expiration
return true;
}

public static void setCache(String mapKey, boolean state) {
if(state) {
cacheExpiration.put(mapKey,loadCacheExpiration(mapKey));
} else {
cacheExpiration.remove(mapKey);
}
}

private static HashMap<String, Long> loadCacheExpiration(String mapKey) {
// TODO: 24/07/2019 load previously saved cache from disk
return new HashMap<>();
}

/**
* @param img
*/
Expand Down