[mkgmap-dev] [PATCH 01/14] Excise the disk cache code.
From Jeffrey C. Ollie jeff at ocjtech.us on Thu Sep 9 21:12:01 BST 2010
From: Scott Crosby <scrosby at cs.rice.edu> --- src/uk/me/parabola/splitter/BinaryMapLoader.java | 160 ----------- .../me/parabola/splitter/CachingMapProcessor.java | 242 ----------------- src/uk/me/parabola/splitter/Main.java | 103 +------- .../splitter/disk/AbstractStoreReader.java | 75 ------ .../splitter/disk/AbstractStoreWriter.java | 58 ---- .../me/parabola/splitter/disk/CacheVerifier.java | 183 ------------- .../me/parabola/splitter/disk/KeyLookupReader.java | 58 ---- .../me/parabola/splitter/disk/KeyLookupWriter.java | 50 ---- .../splitter/disk/LengthPrefixInputStream.java | 278 -------------------- .../splitter/disk/LengthPrefixOutputStream.java | 194 -------------- src/uk/me/parabola/splitter/disk/Member.java | 40 --- src/uk/me/parabola/splitter/disk/MemberType.java | 21 -- .../me/parabola/splitter/disk/NodeStoreReader.java | 48 ---- .../me/parabola/splitter/disk/NodeStoreWriter.java | 37 --- .../splitter/disk/RelationStoreReader.java | 54 ---- .../splitter/disk/RelationStoreWriter.java | 53 ---- .../me/parabola/splitter/disk/WayStoreReader.java | 54 ---- .../me/parabola/splitter/disk/WayStoreWriter.java | 43 --- .../me/parabola/splitter/disk/TestKeyLookups.java | 51 ---- .../splitter/disk/TestLengthPrefixStreams.java | 87 ------ test/uk/me/parabola/splitter/disk/TestStores.java | 216 --------------- 21 files changed, 12 insertions(+), 2093 deletions(-) delete mode 100644 src/uk/me/parabola/splitter/BinaryMapLoader.java delete mode 100644 src/uk/me/parabola/splitter/CachingMapProcessor.java delete mode 100644 src/uk/me/parabola/splitter/disk/AbstractStoreReader.java delete mode 100644 src/uk/me/parabola/splitter/disk/AbstractStoreWriter.java delete mode 100644 src/uk/me/parabola/splitter/disk/CacheVerifier.java delete mode 100644 src/uk/me/parabola/splitter/disk/KeyLookupReader.java delete mode 100644 src/uk/me/parabola/splitter/disk/KeyLookupWriter.java delete mode 100644 src/uk/me/parabola/splitter/disk/LengthPrefixInputStream.java delete mode 100644 src/uk/me/parabola/splitter/disk/LengthPrefixOutputStream.java delete mode 100644 src/uk/me/parabola/splitter/disk/Member.java delete mode 100644 src/uk/me/parabola/splitter/disk/MemberType.java delete mode 100644 src/uk/me/parabola/splitter/disk/NodeStoreReader.java delete mode 100644 src/uk/me/parabola/splitter/disk/NodeStoreWriter.java delete mode 100644 src/uk/me/parabola/splitter/disk/RelationStoreReader.java delete mode 100644 src/uk/me/parabola/splitter/disk/RelationStoreWriter.java delete mode 100644 src/uk/me/parabola/splitter/disk/WayStoreReader.java delete mode 100644 src/uk/me/parabola/splitter/disk/WayStoreWriter.java delete mode 100644 test/uk/me/parabola/splitter/disk/TestKeyLookups.java delete mode 100644 test/uk/me/parabola/splitter/disk/TestLengthPrefixStreams.java delete mode 100644 test/uk/me/parabola/splitter/disk/TestStores.java diff --git a/src/uk/me/parabola/splitter/BinaryMapLoader.java b/src/uk/me/parabola/splitter/BinaryMapLoader.java deleted file mode 100644 index c7b5717..0000000 --- a/src/uk/me/parabola/splitter/BinaryMapLoader.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter; - -import java.io.File; -import java.io.IOException; -import java.util.Map; - -import uk.me.parabola.splitter.disk.Member; -import uk.me.parabola.splitter.disk.NodeStoreReader; -import uk.me.parabola.splitter.disk.RelationStoreReader; -import uk.me.parabola.splitter.disk.WayStoreReader; - -/** - * Loads binary map files, calling the appropriate methods on a - * {@code MapProcessor} as it progresses. - */ -class BinaryMapLoader implements MapReader { - - // How many elements to process before displaying a status update - private static final int NODE_STATUS_UPDATE_THRESHOLD = 2500000; - private static final int WAY_STATUS_UPDATE_THRESHOLD = 500000; - private static final int RELATION_STATUS_UPDATE_THRESHOLD = 50000; - - private final String path; - private final MapProcessor processor; - - private boolean startNodeOnly; - - private long nodeCount; - private long wayCount; - private long relationCount; - private int minNodeId = Integer.MAX_VALUE; - private int maxNodeId = Integer.MIN_VALUE; - - BinaryMapLoader(String path, MapProcessor processor) { - this.path = path; - this.processor = processor; - this.startNodeOnly = processor.isStartNodeOnly(); - } - - @Override - public long getNodeCount() { - return nodeCount; - } - - @Override - public long getWayCount() { - return wayCount; - } - - @Override - public long getRelationCount() { - return relationCount; - } - - @Override - public int getMinNodeId() { - return minNodeId; - } - - @Override - public int getMaxNodeId() { - return maxNodeId; - } - - public void load() throws IOException { - processNodes(); - if (!startNodeOnly) { - processWays(); - processRelations(); - } - processor.endMap(); - } - - private void processNodes() throws IOException { - System.out.println("Loading and processing nodes"); - NodeStoreReader reader = new NodeStoreReader(path + File.separatorChar + "nodes.bin"); - while (reader.next()) { - int id = reader.getId(); - processor.startNode(id, reader.getLat(), reader.getLon()); - if (!startNodeOnly) { - for (Map.Entry<String, String> entry : reader.getTags().entrySet()) { - processor.nodeTag(entry.getKey(), entry.getValue()); - } - processor.endNode(); - } - - if (id < minNodeId) { - minNodeId = id; - } - if (id > maxNodeId) { - maxNodeId = id; - } - - nodeCount++; - if (nodeCount % NODE_STATUS_UPDATE_THRESHOLD == 0) { - System.out.println(Utils.format(nodeCount) + " nodes processed..."); - } - } - reader.close(); - } - - private void processWays() throws IOException { - System.out.println("Loading and processing ways"); - WayStoreReader reader = new WayStoreReader(path + File.separatorChar + "ways.bin"); - while (reader.next()) { - processor.startWay(reader.getId()); - for (int nodeId : reader.getNodeIds()) { - processor.wayNode(nodeId); - } - for (Map.Entry<String, String> entry : reader.getTags().entrySet()) { - processor.wayTag(entry.getKey(), entry.getValue()); - } - processor.endWay(); - wayCount++; - if (wayCount % WAY_STATUS_UPDATE_THRESHOLD == 0) { - System.out.println(Utils.format(wayCount) + " ways processed..."); - } - } - reader.close(); - } - - private void processRelations() throws IOException { - System.out.println("Loading and processing relations"); - RelationStoreReader reader = new RelationStoreReader(path + File.separatorChar + "relations.bin"); - while (reader.next()) { - processor.startRelation(reader.getId()); - for (Map.Entry<String, String> entry : reader.getTags().entrySet()) { - processor.relationTag(entry.getKey(), entry.getValue()); - } - for (Member member : reader.getMembers()) { - switch (member.getType()) { - case Node: - processor.relationNode(member.getId(), member.getRole()); - break; - case Way: - processor.relationWay(member.getId(), member.getRole()); - break; - } - } - processor.endRelation(); - relationCount++; - if (relationCount % RELATION_STATUS_UPDATE_THRESHOLD == 0) { - System.out.println(Utils.format(relationCount) + " relations processed..."); - } - } - reader.close(); - } -} \ No newline at end of file diff --git a/src/uk/me/parabola/splitter/CachingMapProcessor.java b/src/uk/me/parabola/splitter/CachingMapProcessor.java deleted file mode 100644 index 5b8b799..0000000 --- a/src/uk/me/parabola/splitter/CachingMapProcessor.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ - -package uk.me.parabola.splitter; - -import java.io.File; -import java.io.IOException; - -import uk.me.parabola.splitter.disk.CacheVerifier; -import uk.me.parabola.splitter.disk.MemberType; -import uk.me.parabola.splitter.disk.NodeStoreWriter; -import uk.me.parabola.splitter.disk.RelationStoreWriter; -import uk.me.parabola.splitter.disk.WayStoreWriter; - -/** - * Writes the map out in a binary format to a disk cache and then - * delegates calls on to another {@link MapProcessor}. - * - * @author Chris Miller - */ -public class CachingMapProcessor implements MapProcessor { - - private NodeStoreWriter nodeWriter; - private WayStoreWriter wayWriter; - private RelationStoreWriter relationWriter; - - private MapProcessor delegate; - CacheVerifier verifier; - private int currentNode; - private int currentWay; - private int currentRel; - private boolean startedWayTags; - private boolean startedRelTags; - - public CachingMapProcessor(String outputDir, CacheVerifier verifier, MapProcessor delegate) throws IOException { - this.delegate = delegate; - this.verifier = verifier; - verifier.clearEntries(); - nodeWriter = new NodeStoreWriter(outputDir + File.separatorChar + "nodes.bin"); - wayWriter = new WayStoreWriter(outputDir + File.separatorChar + "ways.bin"); - relationWriter = new RelationStoreWriter(outputDir + File.separatorChar + "relations.bin"); - } - - @Override - public boolean isStartNodeOnly() { - return false; - } - - @Override - public void boundTag(Area bounds) { - // todo: write the bounds out to the cache - delegate.boundTag(bounds); - } - - @Override - public void startNode(int id, double lat, double lon) { - currentNode = id; - try { - nodeWriter.write(id, lat, lon); - } catch (IOException e) { - System.out.println("Unable to write node " + id + ". Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - delegate.startNode(id, lat, lon); - } - - @Override - public void startWay(int id) { - currentWay = id; - startedWayTags = false; - try { - wayWriter.write(id); - } catch (IOException e) { - System.out.println("Unable to write way " + id + ". Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - if (!delegate.isStartNodeOnly()) - delegate.startWay(id); - } - - @Override - public void startRelation(int id) { - currentRel = id; - startedRelTags = false; - try { - relationWriter.write(id); - } catch (IOException e) { - System.out.println("Unable to write relation " + id + ". Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - if (!delegate.isStartNodeOnly()) - delegate.startRelation(id); - } - - @Override - public void nodeTag(String key, String value) { - try { - nodeWriter.writeTag(key, value); - } catch (IOException e) { - System.out.println("Unable to write tag for node " + currentNode + ". key=" + key + ", value=" + value + ". Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - if (!delegate.isStartNodeOnly()) - delegate.nodeTag(key, value); - } - - @Override - public void wayTag(String key, String value) { - try { - if (!startedWayTags) { - startedWayTags = true; - wayWriter.closeNodeRefs(); - } - wayWriter.writeTag(key, value); - } catch (IOException e) { - System.out.println("Unable to write tag for way " + currentWay + ". key=" + key + ", value=" + value + ". Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - if (!delegate.isStartNodeOnly()) - delegate.wayTag(key, value); - } - - @Override - public void relationTag(String key, String value) { - try { - if (!startedRelTags) { - startedRelTags = true; - relationWriter.closeMembers(); - } - relationWriter.writeTag(key, value); - } catch (IOException e) { - System.out.println("Unable to write tag for relation " + currentRel + ". key=" + key + ", value=" + value + ". Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - if (!delegate.isStartNodeOnly()) - delegate.relationTag(key, value); - } - - @Override - public void wayNode(int nodeId) { - try { - wayWriter.writeNodeRef(nodeId); - } catch (IOException e) { - System.out.println("Unable to write node reference for way " + currentWay + ", nodeId=" + nodeId + ". Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - if (!delegate.isStartNodeOnly()) - delegate.wayNode(nodeId); - } - - @Override - public void relationNode(int nodeId, String role) { - try { - relationWriter.writeMember(MemberType.Node, nodeId, role); - } catch (IOException e) { - System.out.println("Unable to write node member for relation " + currentRel + ", nodeId=" + nodeId + ", role='" + role + "'. Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - if (!delegate.isStartNodeOnly()) - delegate.relationNode(nodeId, role); - } - - @Override - public void relationWay(int wayId, String role) { - try { - relationWriter.writeMember(MemberType.Way, wayId, role); - } catch (IOException e) { - System.out.println("Unable to write way member for relation " + currentRel + ", wayId=" + wayId + ", role='" + role + "'. Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - if (!delegate.isStartNodeOnly()) - delegate.relationWay(wayId, role); - } - - @Override - public void endNode() { - try { - nodeWriter.closeTags(); - nodeWriter.next(); - } catch (IOException e) { - System.out.println("Unable to finish writing node " + currentNode + ". Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - if (!delegate.isStartNodeOnly()) - delegate.endNode(); - } - - @Override - public void endWay() { - try { - if (!startedWayTags) - wayWriter.closeNodeRefs(); - wayWriter.closeTags(); - wayWriter.next(); - } catch (IOException e) { - System.out.println("Unable to finish writing way " + currentWay + ". Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - if (!delegate.isStartNodeOnly()) - delegate.endWay(); - } - - @Override - public void endRelation() { - try { - if (!startedRelTags) - relationWriter.closeMembers(); - relationWriter.closeTags(); - relationWriter.next(); - } catch (IOException e) { - System.out.println("Unable to finish writing relation " + currentRel + ". Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - if (!delegate.isStartNodeOnly()) - delegate.endRelation(); - } - - @Override - public void endMap() { - try { - nodeWriter.close(); - wayWriter.close(); - relationWriter.close(); - verifier.saveEntries(); - } catch (IOException e) { - System.out.println("Unable to close node/way/relation writer(s). Reason: " + e.getMessage()); - throw new RuntimeException(e); - } - if (!delegate.isStartNodeOnly()) - delegate.endMap(); - } -} diff --git a/src/uk/me/parabola/splitter/Main.java b/src/uk/me/parabola/splitter/Main.java index 37dfc77..84e5dd7 100644 --- a/src/uk/me/parabola/splitter/Main.java +++ b/src/uk/me/parabola/splitter/Main.java @@ -13,7 +13,6 @@ package uk.me.parabola.splitter; -import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; @@ -27,7 +26,6 @@ import java.util.Set; import uk.me.parabola.splitter.args.ParamParser; import uk.me.parabola.splitter.args.SplitterParams; -import uk.me.parabola.splitter.disk.CacheVerifier; import uk.me.parabola.splitter.geo.City; import uk.me.parabola.splitter.geo.CityFinder; import uk.me.parabola.splitter.geo.CityLoader; @@ -84,12 +82,6 @@ public class Main { // Whether or not the source OSM file(s) contain strictly nodes first, then ways, then rels, // or they're all mixed up. Running with mixed enabled takes longer. private boolean mixed; - // The path to the disk cache. If this is null, no cache will be generated or used. - private String diskCachePath; - // Whether or not a new cache needs to be generated. - private boolean generateCache; - // Used to verify whether an existing cache is valid or not. - private CacheVerifier verifier; // The path where the results are written out to. private String fileOutputDir; // A GeoNames file to use for naming the tiles. @@ -147,53 +139,15 @@ public class Main { fileOutputDir = DEFAULT_DIR; } - if (diskCachePath != null) { - File cacheDir = new File(diskCachePath); - if (!cacheDir.exists()) { - System.out.println("Cache directory not found. Creating directory '" + cacheDir + "' and generating cache"); - if (cacheDir.mkdirs()) { - generateCache = true; - } else { - System.err.println("Unable to create cache directory! Disk cache disabled"); - diskCachePath = null; - } - } else if (!cacheDir.isDirectory()) { - System.err.println("The --cache parameter must specify a directory. The --cache parameter is being ignored, disk cache is disabled."); - diskCachePath = null; - } - if (diskCachePath != null) { - verifier = new CacheVerifier(diskCachePath, filenames); - try { - if (!generateCache) { - System.out.println("Checking for an existing cache and verifying contents..."); - } - if (verifier.validateCache()) { - System.out.println("A suitable cache was found. All data will be loaded from cache rather than any .osm file(s) or stdin"); - } else if (filenames.isEmpty()) { - System.out.println("No .osm files were supplied and the --cache isn't populated so osm data will be read from stdin"); - useStdIn = true; - generateCache = true; - } else { - System.out.println("No suitable cache was found. A new cache will be created to speed up the splitting stage"); - generateCache = true; - } - } catch (IOException e) { - System.err.println("Unable to verify cache content - regenerating cache. Reason: " + e.getMessage()); - e.printStackTrace(); - generateCache = true; - } - } - } - - if (diskCachePath == null && filenames.isEmpty()) { + if (filenames.isEmpty()) { if (areaList == null) { - throw new IllegalArgumentException("No .osm files were supplied so at least one of --cache or --split-file must be specified"); + throw new IllegalArgumentException("No .osm files were supplied so --split-file must be specified"); } else { int areaCount = areaList.getAreas().size(); int passes = getAreasPerPass(areaCount); if (passes > 1) { - throw new IllegalArgumentException("No .osm files or --cache parameter were supplied, but stdin cannot be used because " + passes - + " passes are required to write out the areas. Either provide --cache or increase --max-areas to match the number of areas (" + areaCount + ')'); + throw new IllegalArgumentException("No .osm files supplied, but stdin cannot be used because " + passes + + " passes are required to write out the areas. Increase --max-areas to match the number of areas (" + areaCount + ')'); } useStdIn = true; } @@ -270,7 +224,6 @@ public class Main { } mixed = params.isMixed(); statusFreq = params.getStatusFreq(); - diskCachePath = params.getCache(); fileOutputDir = params.getOutputDir(); if (fileOutputDir == null) { fileOutputDir = DEFAULT_DIR; @@ -311,27 +264,12 @@ public class Main { MapCollector nodes = densityMap ? new DensityMapCollector(trim, resolution) : new NodeCollector(); MapProcessor processor = nodes; - boolean loadFromCache = false; - if (diskCachePath == null) { - System.out.println("The input osm file(s) will be re-parsed during the split (slower) because no --cache parameter was specified"); - } else { - if (generateCache) { - processor = new CachingMapProcessor(diskCachePath, verifier, processor); - generateCache = false; - } else { - loadFromCache = true; - } - } - - MapReader mapReader = processMap(processor, loadFromCache); + MapReader mapReader = processMap(processor); System.out.print("A total of " + Utils.format(mapReader.getNodeCount()) + " nodes, " + Utils.format(mapReader.getWayCount()) + " ways and " + Utils.format(mapReader.getRelationCount()) + " relations were processed "); - if (loadFromCache) { - System.out.println("from the disk cache"); - } else { - System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files")); - } + + System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files")); System.out.println("Min node ID = " + mapReader.getMinNodeId()); System.out.println("Max node ID = " + mapReader.getMaxNodeId()); @@ -404,18 +342,7 @@ public class Main { areas.get(i * areasPerPass + currentWriters.length - 1).getMapId() + ')'); MapProcessor processor = new SplitProcessor(currentWriters, maxThreads); - if (generateCache) { - if (numPasses == 1) { - System.out.println("No cache will be generated since only one pass is required"); - generateCache = false; - diskCachePath = null; - } else { - System.out.println("No valid existing cache found. A cache will be generated on this pass"); - processor = new CachingMapProcessor(diskCachePath, verifier, processor); - } - } - MapReader mapReader = processMap(processor, !generateCache && diskCachePath != null); - generateCache = false; // Make sure the cache isn't generated more than once! + MapReader mapReader = processMap(processor); System.out.println("Wrote " + Utils.format(mapReader.getNodeCount()) + " nodes, " + Utils.format(mapReader.getWayCount()) + " ways, " + Utils.format(mapReader.getRelationCount()) + " relations"); @@ -426,16 +353,10 @@ public class Main { return (int) Math.ceil((double) areaCount / (double) maxAreasPerPass); } - private MapReader processMap(MapProcessor processor, boolean useCache) throws XmlPullParserException, IOException { - if (useCache) { - BinaryMapLoader loader = new BinaryMapLoader(diskCachePath, processor); - loader.load(); - return loader; - } else { - OSMParser parser = new OSMParser(processor, mixed); - processOsmFiles(parser); - return parser; - } + private MapReader processMap(MapProcessor processor) throws XmlPullParserException, IOException { + OSMParser parser = new OSMParser(processor, mixed); + processOsmFiles(parser); + return parser; } private void processOsmFiles(OSMParser parser) throws IOException, XmlPullParserException { diff --git a/src/uk/me/parabola/splitter/disk/AbstractStoreReader.java b/src/uk/me/parabola/splitter/disk/AbstractStoreReader.java deleted file mode 100644 index cb0b863..0000000 --- a/src/uk/me/parabola/splitter/disk/AbstractStoreReader.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter.disk; - -import java.io.BufferedInputStream; -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; - -/** - * Abstract base class for reading in binary cache files - */ -public abstract class AbstractStoreReader { - protected final LengthPrefixInputStream in; - protected final KeyLookupReader keys; - protected int id; - protected Map<String, String> tags = new HashMap<String, String>(20); - - public AbstractStoreReader(InputStream in, KeyLookupReader keys) throws IOException { - this.in = new LengthPrefixInputStream(new BufferedInputStream(in, 16384)); - this.keys = keys; - } - - public int getId() { - return id; - } - - public Map<String, String> getTags() { - return tags; - } - - public boolean next() throws IOException { - try { - if (!in.next()) { - return false; - } - id = in.readInt(); - readHeader(); - readTags(); - return true; - } catch (EOFException e) { - return false; - } - } - - protected LengthPrefixInputStream getIn() { - return in; - } - - protected abstract void readHeader() throws IOException; - - private void readTags() throws IOException { - short keyId; - tags.clear(); - while ((keyId = in.readShort()) != 0) { - tags.put(keys.get(keyId), in.readUTF()); - } - } - - public void close() throws IOException { - in.close(); - } -} diff --git a/src/uk/me/parabola/splitter/disk/AbstractStoreWriter.java b/src/uk/me/parabola/splitter/disk/AbstractStoreWriter.java deleted file mode 100644 index d7bad04..0000000 --- a/src/uk/me/parabola/splitter/disk/AbstractStoreWriter.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter.disk; - -import java.io.IOException; -import java.io.OutputStream; - -/** - * Provides base functionality for writing elements out to disk in binary format - */ -public abstract class AbstractStoreWriter { - - private final LengthPrefixOutputStream out; - private final KeyLookupWriter keys; - - public AbstractStoreWriter(OutputStream out, KeyLookupWriter keys) { - this.out = new LengthPrefixOutputStream(out); - this.keys = keys; - } - - protected LengthPrefixOutputStream getOut() { - return out; - } - - public void writeTag(CharSequence key, CharSequence value) throws IOException { - out.writeShort(keys.set(key)); - out.writeUTF(value); - } - - public void closeTags() throws IOException { - out.writeShort(0); - } - - /** - * Inserts the length header and writes the element through to - * the underlying output stream. - * - * @throws IOException - */ - public void next() throws IOException { - out.next(); - } - - public void close() throws IOException { - out.close(); - keys.close(); - } -} diff --git a/src/uk/me/parabola/splitter/disk/CacheVerifier.java b/src/uk/me/parabola/splitter/disk/CacheVerifier.java deleted file mode 100644 index b99d401..0000000 --- a/src/uk/me/parabola/splitter/disk/CacheVerifier.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ - -package uk.me.parabola.splitter.disk; - -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Maintains a persistent list of the .osm file(s) used to build the cache. Also verifies this - * list against a list of files that are to be processed, so a decision can be made about - * whether to flush the cache or not. - * - * @author Chris Miller - */ -public class CacheVerifier { - private final String cacheDirectory; // the cache location - private final File entriesFile; // the file containing a list of the cache entries - private final List<String> filenames; // the .osm files we expect to have been cached - - public CacheVerifier(String cacheDirectory, List<String> filenames) { - this.cacheDirectory = cacheDirectory; - entriesFile = new File(cacheDirectory, "cache.entries"); - this.filenames = filenames; - } - - /** - * Checks to see if the cache already contains suitable data. It does this by comparing - * filenames and timestamps from the cache.entries file against the files that are - * being processed by this run of the splitter. - * - * @param filenames the files to process. - * @return {@code true} if there is an existing cache and it is valid for this run. - */ - public boolean validateCache() throws IOException { - List<CacheEntry> existingEntries = loadEntries(); - - // If no cache.entries file was found, we have to regenerate the cache - if (existingEntries == null) - return false; - - // See if there are any cache files - boolean nodesExist = new File(cacheDirectory, "nodes.bin").exists(); - - // If no filenames were provided we use the cache (if it exists) - if (filenames.isEmpty()) - return nodesExist; - - // Compare the existing cache entries with the .osm filenames provided. If they match we can use the cache - List<CacheEntry> entries = new ArrayList<CacheEntry>(filenames.size()); - for (String filename : filenames) { - entries.add(new CacheEntry(filename)); - } - Collections.sort(existingEntries); - Collections.sort(entries); - - return existingEntries.equals(entries) && nodesExist; - } - - public void saveEntries() throws IOException { - Writer out = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(entriesFile), 4096), "UTF-8"); - try { - for (String filename : filenames) { - File file = new File(filename); - if (file.exists()) { - long size = file.length(); - long timestamp = file.lastModified(); - out.write(String.valueOf(size)); - out.write(','); - out.write(String.valueOf(timestamp)); - out.write(','); - out.write(file.getCanonicalPath()); - out.write('\n'); - } - } - } finally { - out.close(); - } - } - - protected List<CacheEntry> loadEntries() throws IOException { - if (!entriesFile.exists()) { - return null; - } - BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(entriesFile), "UTF-8")); - List<CacheEntry> result = new ArrayList<CacheEntry>(); - int count = 0; - String line; - try { - while ((line = in.readLine()) != null) { - count++; - String[] parts = line.split(",", 3); - if (parts.length < 3) { - System.err.println("Invalid cache.entries file on line " + count + ". The cache will be rebuilt"); - return null; - } - long size = Long.valueOf(parts[0]); - long timestamp = Long.valueOf(parts[1]); - result.add(new CacheEntry(parts[2], size, timestamp)); - } - } finally { - in.close(); - } - return result; - } - - public boolean clearEntries() { - return entriesFile.delete(); - } - - protected static class CacheEntry implements Comparable<CacheEntry> { - private final String filename; - private final long size; - private final long timestamp; - - public CacheEntry(String filename, long size, long timestamp) { - this.filename = filename; - this.size = size; - this.timestamp = timestamp; - } - - protected CacheEntry(String filename) throws IOException { - File file = new File(filename); - this.filename = file.getCanonicalPath(); - size = file.length(); - timestamp = file.lastModified(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - CacheEntry that = (CacheEntry) o; - - if (size != that.size) return false; - if (timestamp != that.timestamp) return false; - if (!filename.equals(that.filename)) return false; - - return true; - } - - @Override - public int hashCode() { - int result = filename.hashCode(); - result = 31 * result + (int) (size ^ (size >>> 32)); - result = 31 * result + (int) (timestamp ^ (timestamp >>> 32)); - return result; - } - - @Override - public int compareTo(CacheEntry o) { - int result = filename.compareTo(o.filename); - if (result == 0) { - result = size < o.size ? -1 : size > o.size ? 1 : 0; - } - if (result == 0) { - result = timestamp < o.timestamp ? -1 : timestamp > o.timestamp ? 1 : 0; - } - return result; - } - } -} diff --git a/src/uk/me/parabola/splitter/disk/KeyLookupReader.java b/src/uk/me/parabola/splitter/disk/KeyLookupReader.java deleted file mode 100644 index d983679..0000000 --- a/src/uk/me/parabola/splitter/disk/KeyLookupReader.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter.disk; - -import java.io.EOFException; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; - -import uk.me.parabola.splitter.IntObjMap; - -/** - * Reads in a set of strings to binary format so they can be indexed by ID - */ -public class KeyLookupReader { - private final IntObjMap<String> lookup = new IntObjMap<String>(); - - public KeyLookupReader(String keyFilename) throws IOException { - File keyFile = new File(keyFilename); - LengthPrefixInputStream is = new LengthPrefixInputStream(new FileInputStream(keyFile), 16384); - load(is); - } - - public KeyLookupReader(InputStream in) throws IOException { - LengthPrefixInputStream is = new LengthPrefixInputStream(in, 16384); - load(is); - } - - private void load(LengthPrefixInputStream is) throws IOException { - try { - int s = 0; - while (is.next()) { - try { - lookup.put(++s, is.readUTF()); - } catch (EOFException e) { - break; - } - } - } finally { - is.close(); - } - } - - public String get(int i) { - return lookup.get(i); - } -} diff --git a/src/uk/me/parabola/splitter/disk/KeyLookupWriter.java b/src/uk/me/parabola/splitter/disk/KeyLookupWriter.java deleted file mode 100644 index 536a219..0000000 --- a/src/uk/me/parabola/splitter/disk/KeyLookupWriter.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter.disk; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.HashMap; -import java.util.Map; - -/** - * Writes out a set of strings to binary format so they can be indexed by ID - */ -public class KeyLookupWriter { - private final LengthPrefixOutputStream out; - private final Map<CharSequence, Short> lookup = new HashMap<CharSequence, Short>(20); - - public KeyLookupWriter(String keyFilename) throws IOException { - out = new LengthPrefixOutputStream(new FileOutputStream(keyFilename)); - } - - public KeyLookupWriter(OutputStream out) throws IOException { - this.out = new LengthPrefixOutputStream(out); - } - - public short set(CharSequence key) throws IOException { - Short result = lookup.get(key); - if (result == null) { - result = Short.valueOf((short) (lookup.size() + 1)); - lookup.put(key, result); - out.writeUTF(key); - out.next(); - } - return result; - } - - public void close() throws IOException { - out.close(); - } -} \ No newline at end of file diff --git a/src/uk/me/parabola/splitter/disk/LengthPrefixInputStream.java b/src/uk/me/parabola/splitter/disk/LengthPrefixInputStream.java deleted file mode 100644 index 7f41c5f..0000000 --- a/src/uk/me/parabola/splitter/disk/LengthPrefixInputStream.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ - -package uk.me.parabola.splitter.disk; - -import java.io.IOException; -import java.io.InputStream; -import java.io.UTFDataFormatException; - -import uk.me.parabola.splitter.Utils; - -/** - * Reads in a 'unit' of data at a time into a buffer every time next() is called. - * Each unit has a two-byte prefix indicating how many bytes long the unit is. - * Every time next() is called, the next unit will be read in. - * <p/> - * This is similar to a {@code DataInputStream} but offers better - * performance (more specialised buffering, no synchronisation). - * - * @author Chris Miller - */ -public class LengthPrefixInputStream extends InputStream { - - private InputStream in; - private byte[] buf; - private int index; - private int maxLen; - private long bytesRead; - - public LengthPrefixInputStream(InputStream in) { - this(in, 4096); - } - - public LengthPrefixInputStream(InputStream in, int bufferSize) { - this.in = in; - buf = new byte[bufferSize]; - } - - public int remaining() { - return maxLen - index; - } - - public byte readByte() throws IOException { - ensureData(1); - return buf[index++]; - } - - public int readUnsignedByte() throws IOException { - ensureData(1); - return buf[index++] & 0xff; - } - - public short readShort() throws IOException { - ensureData(2); - return (short) ((buf[index++] << 8) | (buf[index++] & 0xff)); - } - - public int readUnsignedShort() throws IOException { - ensureData(2); - return (buf[index++] & 0xff) << 8 | (buf[index++] & 0xff); - } - - public int readInt() throws IOException { - ensureData(4); - return buf[index++] << 24 | (buf[index++] & 0xff) << 16 | (buf[index++] & 0xff) << 8 | (buf[index++] & 0xff); - } - - public long readLong() throws IOException { - ensureData(8); - return (long) buf[index++] << 56 | - ((long) buf[index++] & 0xff) << 48 | - ((long) buf[index++] & 0xff) << 40 | - ((long) buf[index++] & 0xff) << 32 | - ((long) buf[index++] & 0xff) << 24 | - ((long) buf[index++] & 0xff) << 16 | - ((long) buf[index++] & 0xff) << 8 | - buf[index++] & 0xff; - } - - public double readDouble() throws IOException { - return Double.longBitsToDouble(readLong()); - } - - /** - * This has been lifted from DataInputStream. We inline it here rather than - * delegate to DIS because that way we can avoid creating a temporary buffer - * for the UTF-8 chars. - */ - public String readUTF() throws IOException { - int utflen = readUnsignedShort(); - ensureData(utflen); - char[] chararr = new char[utflen]; - - int c, char2, char3; - int count = 0; - int chararr_count = 0; - - while (count < utflen) { - c = (int) buf[index + count] & 0xff; - if (c > 127) break; - count++; - chararr[chararr_count++] = (char) c; - } - - while (count < utflen) { - c = (int) buf[index + count] & 0xff; - switch (c >> 4) { - case 0: - case 1: - case 2: - case 3: - case 4: - case 5: - case 6: - case 7: - /* 0xxxxxxx*/ - count++; - chararr[chararr_count++] = (char) c; - break; - case 12: - case 13: - /* 110x xxxx 10xx xxxx*/ - count += 2; - if (count > utflen) - throw new UTFDataFormatException("malformed input: partial character at end"); - char2 = (int) buf[index + count - 1]; - if ((char2 & 0xC0) != 0x80) - throw new UTFDataFormatException("malformed input around byte " + count); - chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F)); - break; - case 14: - /* 1110 xxxx 10xx xxxx 10xx xxxx */ - count += 3; - if (count > utflen) - throw new UTFDataFormatException("malformed input: partial character at end"); - char2 = (int) buf[index + count - 2]; - char3 = (int) buf[index + count - 1]; - if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) - throw new UTFDataFormatException("malformed input around byte " + (count - 1)); - chararr[chararr_count++] = (char) ((c & 0x0F) << 12 | (char2 & 0x3F) << 6 | (char3 & 0x3F)); - break; - default: - /* 10xx xxxx, 1111 xxxx */ - throw new UTFDataFormatException("malformed input around byte " + count); - } - } - index += count; - // The number of chars produced may be less than utflen - return new String(chararr, 0, chararr_count); - } - - private byte[] lengthBuf = new byte[2]; - - public boolean next() throws IOException { - // Read the length prefix - int len = 0; - while (len < 2) { - int read = in.read(lengthBuf, len, 2 - len); - if (read < 0) { - if (len > 0) { - displayByteBuffer(); - System.out.println("Unexpected EOF reached while reading segment length. Only " + len + " of 2 bytes read. Total bytes read " + Utils.format(bytesRead)); - } - return false; - } - len += read; - bytesRead += read; - } - maxLen = (lengthBuf[0] & 0xFF) << 8 | (lengthBuf[1] & 0xFF); - if (maxLen == 0xFFFF) { - // we've hit a marker that indicates the length is held in an int rather than a short - byte[] temp = new byte[4]; - len = 0; - while (len < 4) { - int read = in.read(temp, len, 4 - len); - if (read < 0) { - displayByteBuffer(); - System.out.println("Unexpected EOF reached while reading segment length. Only " + len + " of 4 bytes read. Total bytes read " + Utils.format(bytesRead)); - return false; - } - len += read; - bytesRead += read; - } - maxLen = temp[0] << 24 | (temp[1] & 0xff) << 16 | (temp[2] & 0xff) << 8 | (temp[3] & 0xff); - } - index = 0; - - // Make sure the buffer has enough room for this segment of data - if (buf.length < index + maxLen) { - buf = new byte[index + maxLen]; - } - // Read in the whole buffer - len = 0; - while (len < maxLen) { - int read = in.read(buf, index + len, maxLen - len); - if (read < 0) { - // We've hit the EOF - this shouldn't happen! - System.out.println("Unexpected EOF reached while loading segment. Expected " + maxLen + " bytes, only read " + len + ". Total bytes read " + Utils.format(bytesRead)); - System.out.println("Here's the buffer that was read:"); - maxLen = len; - displayByteBuffer(); - return false; - } - len += read; - bytesRead += read; - } - return true; - } - - private void ensureData(int len) throws IOException { - if (index + len > maxLen) { - displayByteBuffer(); // useful for debugging - throw new IOException("Attempt was made to read " + len + " bytes when there are only " + (maxLen - index) + " remaining (from " + maxLen + " total in this segment)"); - } - } - - @Override - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - int result = Math.min(len, maxLen - index); - System.arraycopy(buf, index, b, off, result); - index += result; - return result; - } - - @Override - public int read() throws IOException { - return in.read(); - } - - @Override - public void close() throws IOException { - in.close(); - } - - private static final char[] HEX_CHARS = new char[] {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'}; - private static final int BYTES_PER_ROW = 32; - - private void displayByteBuffer() { - char[] ascii = new char[BYTES_PER_ROW]; - for (int i = 0; i < maxLen; i++) { - System.out.print(HEX_CHARS[(buf[i] & 0xFF) >> 4]); - System.out.print(HEX_CHARS[buf[i] & 0x0F]); - if (i % 8 == 7) - System.out.print(" "); - else - System.out.print(' '); - - char c = (char) buf[i]; - if (c < 32 || c == 127) - c = '.'; - ascii[i % BYTES_PER_ROW] = c; - - if (i % BYTES_PER_ROW == BYTES_PER_ROW - 1) - System.out.println(ascii); - } - int missing = (BYTES_PER_ROW - (maxLen % BYTES_PER_ROW) % BYTES_PER_ROW); - for (int i = 0; i < missing; i++) - System.out.print(" "); - for (int i = 0; i < missing / 8; i++) - System.out.print(' '); - System.out.println(ascii); - } -} \ No newline at end of file diff --git a/src/uk/me/parabola/splitter/disk/LengthPrefixOutputStream.java b/src/uk/me/parabola/splitter/disk/LengthPrefixOutputStream.java deleted file mode 100644 index ff97315..0000000 --- a/src/uk/me/parabola/splitter/disk/LengthPrefixOutputStream.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ - -package uk.me.parabola.splitter.disk; - -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.UTFDataFormatException; - -/** - * Buffers the output then writes it all at once with a 2-byte length prefix - * every time next() is called. - * <p/> - * This is similar to a {@code DataOutputStream} but offers better - * performance (more specialised buffering, no synchronisation). - * - * @author Chris Miller - */ -public class LengthPrefixOutputStream extends OutputStream { - - private OutputStream out; - private byte[] buf; - private int index = 2; - - public LengthPrefixOutputStream(OutputStream out) { - this(out, 4096); - } - - public LengthPrefixOutputStream(OutputStream out, int bufferSize) { - this.out = new BufferedOutputStream(out, 8192); - buf = new byte[bufferSize]; - } - - public void writeByte(int b) throws IOException { - ensureCapacity(1); - buf[index++] = (byte) b; - } - - public final void writeShort(int v) throws IOException { - ensureCapacity(2); - buf[index++] = (byte) (v >>> 8); - buf[index++] = (byte) v; - } - - public final void writeInt(int v) throws IOException { - ensureCapacity(4); - buf[index++] = (byte) (v >>> 24); - buf[index++] = (byte) (v >>> 16); - buf[index++] = (byte) (v >>> 8); - buf[index++] = (byte) v; - } - - public final void writeLong(long v) throws IOException { - ensureCapacity(8); - buf[index++] = (byte) (v >>> 56); - buf[index++] = (byte) (v >>> 48); - buf[index++] = (byte) (v >>> 40); - buf[index++] = (byte) (v >>> 32); - buf[index++] = (byte) (v >>> 24); - buf[index++] = (byte) (v >>> 16); - buf[index++] = (byte) (v >>> 8); - buf[index++] = (byte) v; - } - - public final void writeDouble(double v) throws IOException { - writeLong(Double.doubleToLongBits(v)); - } - - /** - * This has been lifted from DataOutputStream. We inline it here rather than - * delegate to DOS because that way we can avoid creating a temporary buffer - * for the UTF-8 chars. - */ - public void writeUTF(CharSequence str) throws IOException { - int strlen = str.length(); - int utflen = 0; - int c; - - for (int i = 0; i < strlen; i++) { - c = str.charAt(i); - if ((c >= 0x0001) && (c <= 0x007F)) { - utflen++; - } else if (c > 0x07FF) { - utflen += 3; - } else { - utflen += 2; - } - } - - if (utflen > 65535) - throw new UTFDataFormatException("encoded string too long: " + utflen + " bytes"); - - ensureCapacity(utflen + 2); - buf[index++] = (byte) (utflen >>> 8); - buf[index++] = (byte) utflen; - - int i; - for (i = 0; i < strlen; i++) { - c = str.charAt(i); - if (!((c >= 0x0001) && (c <= 0x007F))) - break; - buf[index++] = (byte) c; - } - - for (; i < strlen; i++) { - c = str.charAt(i); - if ((c >= 0x0001) && (c <= 0x007F)) { - buf[index++] = (byte) c; - - } else if (c > 0x07FF) { - buf[index++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); - buf[index++] = (byte) (0x80 | ((c >> 6) & 0x3F)); - buf[index++] = (byte) (0x80 | c & 0x3F); - } else { - buf[index++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); - buf[index++] = (byte) (0x80 | c & 0x3F); - } - } - } - - @Override - public void write(int b) throws IOException { - ensureCapacity(1); - buf[index++] = (byte) b; - } - - @Override - public void write(byte[] b) throws IOException { - ensureCapacity(b.length); - System.arraycopy(b, 0, buf, index, b.length); - index += b.length; - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - ensureCapacity(len); - System.arraycopy(b, off, buf, this.index, len); - this.index += len; - } - - public void next() throws IOException { - // Write the length prefix - index -= 2; - - if (index > 0xFFFF) { - // Write a marker so we know the length is held in an int rather than a short - byte[] temp = new byte[4]; - temp[0] = (byte) 0xFF; - temp[1] = (byte) 0xFF; - temp[2] = (byte) (index >>> 24); - temp[3] = (byte) (index >>> 16); - out.write(temp); - buf[0] = (byte) (index >>> 8); - buf[1] = (byte) index; - } else { - buf[0] = (byte) (index >>> 8); - buf[1] = (byte) index; - } - // Dump out the whole buffer - out.write(buf, 0, index + 2); - // Reset the length - index = 2; - } - - @Override - public void flush() throws IOException { - out.flush(); - } - - private void ensureCapacity(int len) { - if (buf.length < this.index + len) { - // We need to grow our buffer - byte[] temp = new byte[(this.index + len) * 3 / 2]; - System.arraycopy(buf, 0, temp, 0, buf.length); - buf = temp; - } - } - - @Override - public void close() throws IOException { - out.close(); - } -} diff --git a/src/uk/me/parabola/splitter/disk/Member.java b/src/uk/me/parabola/splitter/disk/Member.java deleted file mode 100644 index 34cd086..0000000 --- a/src/uk/me/parabola/splitter/disk/Member.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter.disk; - -/** - * Represents a member of a relation - */ -public class Member { - private MemberType type; - private int id; - private String role; - - public Member(MemberType type, int id, String role) { - this.type = type; - this.id = id; - this.role = role; - } - - public MemberType getType() { - return type; - } - - public int getId() { - return id; - } - - public String getRole() { - return role; - } -} diff --git a/src/uk/me/parabola/splitter/disk/MemberType.java b/src/uk/me/parabola/splitter/disk/MemberType.java deleted file mode 100644 index 0f37659..0000000 --- a/src/uk/me/parabola/splitter/disk/MemberType.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter.disk; - -/** - * Represents the type of a relation's member - */ -public enum MemberType { - Node, Way -} - diff --git a/src/uk/me/parabola/splitter/disk/NodeStoreReader.java b/src/uk/me/parabola/splitter/disk/NodeStoreReader.java deleted file mode 100644 index 3a73c85..0000000 --- a/src/uk/me/parabola/splitter/disk/NodeStoreReader.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter.disk; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; - -/** - * Reads in nodes from a binary format - */ -public class NodeStoreReader extends AbstractStoreReader { - private double lat; - private double lon; - - public NodeStoreReader(String filename) throws IOException { - this(new FileInputStream(new File(filename)), new KeyLookupReader(filename + ".keys")); - } - - public NodeStoreReader(InputStream in, KeyLookupReader keys) throws IOException { - super(in, keys); - } - - public double getLat() { - return lat; - } - - public double getLon() { - return lon; - } - - @Override - protected void readHeader() throws IOException { - lat = getIn().readDouble(); - lon = getIn().readDouble(); - } -} \ No newline at end of file diff --git a/src/uk/me/parabola/splitter/disk/NodeStoreWriter.java b/src/uk/me/parabola/splitter/disk/NodeStoreWriter.java deleted file mode 100644 index 0be262c..0000000 --- a/src/uk/me/parabola/splitter/disk/NodeStoreWriter.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter.disk; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -/** - * Persists a collection of nodes to binary format on disk for later retrieval - */ -public class NodeStoreWriter extends AbstractStoreWriter { - - public NodeStoreWriter(String filename) throws IOException { - this(new FileOutputStream(filename), new KeyLookupWriter(filename + ".keys")); - } - - public NodeStoreWriter(OutputStream out, KeyLookupWriter keys) { - super(out, keys); - } - - public void write(int nodeId, double lat, double lon) throws IOException { - getOut().writeInt(nodeId); - getOut().writeDouble(lat); - getOut().writeDouble(lon); - } -} \ No newline at end of file diff --git a/src/uk/me/parabola/splitter/disk/RelationStoreReader.java b/src/uk/me/parabola/splitter/disk/RelationStoreReader.java deleted file mode 100644 index 748ef1d..0000000 --- a/src/uk/me/parabola/splitter/disk/RelationStoreReader.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter.disk; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; - -/** - * Reads in relations from a binary format - */ -public class RelationStoreReader extends AbstractStoreReader { - private List<Member> members = new ArrayList<Member>(10); - private final KeyLookupReader roles; - - public RelationStoreReader(String filename) throws IOException { - this(new FileInputStream(new File(filename)), new KeyLookupReader(filename + ".keys"), new KeyLookupReader(filename + ".roles")); - } - - public RelationStoreReader(InputStream in, KeyLookupReader keys, KeyLookupReader roles) throws IOException { - super(in, keys); - this.roles = roles; - } - - public List<Member> getMembers() { - return members; - } - - @Override - protected void readHeader() throws IOException { - members.clear(); - byte typeByte; - while ((typeByte = getIn().readByte()) != 0) { - MemberType type = MemberType.values()[typeByte - 1]; - int id = getIn().readInt(); - short roleId = getIn().readShort(); - String role = roles.get(roleId); - members.add(new Member(type, id, role)); - } - } -} \ No newline at end of file diff --git a/src/uk/me/parabola/splitter/disk/RelationStoreWriter.java b/src/uk/me/parabola/splitter/disk/RelationStoreWriter.java deleted file mode 100644 index 5e513b9..0000000 --- a/src/uk/me/parabola/splitter/disk/RelationStoreWriter.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter.disk; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -/** - * Persists a collection of relations to binary format on disk for later retrieval - */ -public class RelationStoreWriter extends AbstractStoreWriter { - private final KeyLookupWriter roles; - - public RelationStoreWriter(String filename) throws IOException { - this(new FileOutputStream(filename), new KeyLookupWriter(filename + ".keys"), new KeyLookupWriter(filename + ".roles")); - } - - public RelationStoreWriter(OutputStream out, KeyLookupWriter keys, KeyLookupWriter roles) { - super(out, keys); - this.roles = roles; - } - - public void write(int id) throws IOException { - getOut().writeInt(id); - } - - public void writeMember(MemberType type, int id, CharSequence role) throws IOException { - getOut().writeByte(type.ordinal() + 1); - getOut().writeInt(id); - getOut().writeShort(roles.set(role)); - } - - public void closeMembers() throws IOException { - getOut().writeByte(0); - } - - @Override - public void close() throws IOException { - super.close(); - roles.close(); - } -} \ No newline at end of file diff --git a/src/uk/me/parabola/splitter/disk/WayStoreReader.java b/src/uk/me/parabola/splitter/disk/WayStoreReader.java deleted file mode 100644 index 05a659e..0000000 --- a/src/uk/me/parabola/splitter/disk/WayStoreReader.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter.disk; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; - -/** - * Reads in ways from a binary file format - */ -public class WayStoreReader extends AbstractStoreReader { - private int[] buf = new int[256]; - private int[] nodeIds; - - public WayStoreReader(String filename) throws IOException { - this(new FileInputStream(new File(filename)), new KeyLookupReader(filename + ".keys")); - } - - public WayStoreReader(InputStream in, KeyLookupReader keys) throws IOException { - super(in, keys); - } - - public int[] getNodeIds() { - return nodeIds; - } - - @Override - protected void readHeader() throws IOException { - int i = 0; - int id; - while ((id = getIn().readInt()) != 0) { - if (buf.length <= i) { - int[] temp = new int[(buf.length * 3) / 2 + 1]; - System.arraycopy(buf, 0, temp, 0, buf.length); - buf = temp; - } - buf[i++] = id; - } - nodeIds = new int[i]; - System.arraycopy(buf, 0, nodeIds, 0, i); - } -} \ No newline at end of file diff --git a/src/uk/me/parabola/splitter/disk/WayStoreWriter.java b/src/uk/me/parabola/splitter/disk/WayStoreWriter.java deleted file mode 100644 index c17d7d4..0000000 --- a/src/uk/me/parabola/splitter/disk/WayStoreWriter.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -package uk.me.parabola.splitter.disk; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -/** - * Writes out a sequence of ways to binary format for later retrieval - */ -public class WayStoreWriter extends AbstractStoreWriter { - - public WayStoreWriter(String filename) throws IOException { - this(new FileOutputStream(filename), new KeyLookupWriter(filename + ".keys")); - } - - public WayStoreWriter(OutputStream out, KeyLookupWriter keys) { - super(out, keys); - } - - public void write(int id) throws IOException { - getOut().writeInt(id); - } - - public void writeNodeRef(int id) throws IOException { - getOut().writeInt(id); - } - - public void closeNodeRefs() throws IOException { - getOut().writeInt(0); - } -} \ No newline at end of file diff --git a/test/uk/me/parabola/splitter/disk/TestKeyLookups.java b/test/uk/me/parabola/splitter/disk/TestKeyLookups.java deleted file mode 100644 index 782471c..0000000 --- a/test/uk/me/parabola/splitter/disk/TestKeyLookups.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ - -package uk.me.parabola.splitter.disk; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; - -import org.testng.Assert; -import org.testng.annotations.Test; - -/** - * - */ -public class TestKeyLookups { - @Test - public void test() throws IOException { - ByteArrayOutputStream result = new ByteArrayOutputStream(); - KeyLookupWriter writer = new KeyLookupWriter(result); - writer.set("Testing1"); - writer.set("Testing2"); - writer.set("Testing3"); - writer.set("Testing4"); - writer.set("Euro: \u20AC Pound: \u00A3"); - writer.close(); - - byte[] data = result.toByteArray(); - InputStream in = new ByteArrayInputStream(data); - KeyLookupReader reader = new KeyLookupReader(in); - - Assert.assertEquals(reader.get(0), null); - Assert.assertEquals(reader.get(1), "Testing1"); - Assert.assertEquals(reader.get(2), "Testing2"); - Assert.assertEquals(reader.get(3), "Testing3"); - Assert.assertEquals(reader.get(4), "Testing4"); - Assert.assertEquals(reader.get(5), "Euro: \u20AC Pound: \u00A3"); - Assert.assertEquals(reader.get(6), null); - } -} \ No newline at end of file diff --git a/test/uk/me/parabola/splitter/disk/TestLengthPrefixStreams.java b/test/uk/me/parabola/splitter/disk/TestLengthPrefixStreams.java deleted file mode 100644 index b50e6bc..0000000 --- a/test/uk/me/parabola/splitter/disk/TestLengthPrefixStreams.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ - -package uk.me.parabola.splitter.disk; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; - -import org.testng.Assert; -import org.testng.annotations.Test; - -/** - * - */ -public class TestLengthPrefixStreams { - @Test - public void testOutput() throws IOException { - ByteArrayOutputStream result = new ByteArrayOutputStream(); - LengthPrefixOutputStream os = new LengthPrefixOutputStream(result, 2); // 2 so we test the buffer expansion - os.writeShort(0xABCD); - os.writeInt(0x12345678); - os.writeInt(0xAABBCCDD); - os.writeUTF("Testing"); - os.next(); - os.writeShort(0x9988); - os.writeByte(240); - os.next(); - os.close(); - - byte[] bytes = result.toByteArray(); - - // Test to see if the byte array contains what we'd expect - Assert.assertEquals(bytes, new byte[] { - 0x00, 0x13, // length prefix - (byte) 0xAB, (byte) 0xCD, // 0xABCD - 0x12, 0x34, 0x56, 0x78, // 0x12345678 - (byte) 0xAA, (byte) 0xBB, (byte) 0xCC, (byte) 0xDD, // 0xAABBCCDD - 0x00, 0x07, 'T', 'e', 's', 't', 'i', 'n', 'g', // "Testing" - 0x00, 0x03, // length prefix - (byte) 0x99, (byte) 0x88, // 0x9988 - (byte) 240, // 240 - }, - "Resultant byte array does not match expected result"); - } - - @Test - public void testinput() throws IOException { - byte[] buf = new byte[] { - 0x00, 0x13, // length prefix - (byte) 0xAB, (byte) 0xCD, // 0xABCD - 0x12, 0x34, 0x56, 0x78, // 0x12345678 - (byte) 0xAA, (byte) 0xBB, (byte) 0xCC, (byte) 0xDD, // 0xAABBCCDD - 0x00, 0x07, 'T', 'e', 's', 't', 'i', 'n', 'g', // "Testing" - 0x00, 0x04, // length prefix - (byte) 0x99, (byte) 0x88, // 0x9988 - (byte) 240, // 240 - (byte) 240, // 240 - }; - ByteArrayInputStream result = new ByteArrayInputStream(buf); - LengthPrefixInputStream is = new LengthPrefixInputStream(result, 2); // 2 so we test the buffer expansion - - Assert.assertEquals(is.next(), true, "Premature end of stream reached"); - Assert.assertEquals(is.remaining(), 0x13, "Unexpected number of bytes remaining in this segment"); - Assert.assertEquals(is.readUnsignedShort(), 0xABCD); - Assert.assertEquals(is.readInt(), 0x12345678); - Assert.assertEquals(is.readInt(), 0xAABBCCDD); - Assert.assertEquals(is.readUTF(), "Testing"); - Assert.assertEquals(is.remaining(), 0x00, "Unexpected number of bytes remaining in this segment"); - Assert.assertEquals(is.next(), true, "Premature end of stream reached"); - Assert.assertEquals(is.readShort(), (short) 0x9988); - Assert.assertEquals(is.readUnsignedByte(), 240); - Assert.assertEquals(is.readByte(), (byte) 240); - Assert.assertEquals(is.remaining(), 0x00, "Unexpected number of bytes remaining in this segment"); - Assert.assertEquals(is.next(), false, "Expected to reach the end of stream but didn't"); - } -} \ No newline at end of file diff --git a/test/uk/me/parabola/splitter/disk/TestStores.java b/test/uk/me/parabola/splitter/disk/TestStores.java deleted file mode 100644 index 9f1bbe5..0000000 --- a/test/uk/me/parabola/splitter/disk/TestStores.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright (c) 2009. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ - -package uk.me.parabola.splitter.disk; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import org.testng.Assert; -import org.testng.annotations.Test; - -/** - * - */ -public class TestStores { - @Test - public void testNodeStore() throws IOException { - ByteArrayOutputStream nodeOut = new ByteArrayOutputStream(); - ByteArrayOutputStream keyOut = new ByteArrayOutputStream(); - KeyLookupWriter keyWriter = new KeyLookupWriter(keyOut); - - NodeStoreWriter nodeWriter = new NodeStoreWriter(nodeOut, keyWriter); - nodeWriter.write(1, 0.1234d, 99.125d); - nodeWriter.writeTag("key1", "value1"); - nodeWriter.writeTag("key2", "value2"); - nodeWriter.writeTag("key3", "Euro: \u20AC Pound: \u00A3"); - nodeWriter.closeTags(); - nodeWriter.next(); - nodeWriter.write(2, 5432.12d, 48484.48484d); - nodeWriter.writeTag("key2", "Euro: \u20AC"); - nodeWriter.writeTag("key4", "Pound: \u00A3"); - nodeWriter.closeTags(); - nodeWriter.next(); - - nodeWriter.close(); - - byte[] nodeBuf = nodeOut.toByteArray(); - byte[] keyBuf = keyOut.toByteArray(); - KeyLookupReader keyReader = new KeyLookupReader(new ByteArrayInputStream(keyBuf)); - ByteArrayInputStream nodeIn = new ByteArrayInputStream(nodeBuf); - NodeStoreReader nodeReader = new NodeStoreReader(nodeIn, keyReader); - - Assert.assertTrue(nodeReader.next()); - Assert.assertEquals(nodeReader.getId(), 1); - Assert.assertEquals(nodeReader.getLat(), 0.1234d); - Assert.assertEquals(nodeReader.getLon(), 99.125d); - Map<String, String> tags = nodeReader.getTags(); - Assert.assertEquals(tags.size(), 3); - Assert.assertEquals(tags.get("key1"), "value1"); - Assert.assertEquals(tags.get("key2"), "value2"); - Assert.assertEquals(tags.get("key3"), "Euro: \u20AC Pound: \u00A3"); - Assert.assertNull(tags.get("key4")); - - Assert.assertTrue(nodeReader.next()); - Assert.assertEquals(nodeReader.getId(), 2); - Assert.assertEquals(nodeReader.getLat(), 5432.12d); - Assert.assertEquals(nodeReader.getLon(), 48484.48484d); - tags = nodeReader.getTags(); - Assert.assertEquals(tags.size(), 2); - Assert.assertNull(tags.get("key1")); - Assert.assertEquals(tags.get("key2"), "Euro: \u20AC"); - Assert.assertNull(tags.get("key3")); - Assert.assertEquals(tags.get("key4"), "Pound: \u00A3"); - - Assert.assertFalse(nodeReader.next()); - } - - @Test - public void testWayStore() throws IOException { - ByteArrayOutputStream nodeOut = new ByteArrayOutputStream(); - ByteArrayOutputStream keyOut = new ByteArrayOutputStream(); - KeyLookupWriter keyWriter = new KeyLookupWriter(keyOut); - - WayStoreWriter wayWriter = new WayStoreWriter(nodeOut, keyWriter); - wayWriter.write(1); - wayWriter.writeNodeRef(10); - wayWriter.writeNodeRef(11); - wayWriter.writeNodeRef(12); - wayWriter.closeNodeRefs(); - wayWriter.writeTag("key1", "value1"); - wayWriter.writeTag("key2", "value2"); - wayWriter.writeTag("key3", "Euro: \u20AC Pound: \u00A3"); - wayWriter.closeTags(); - wayWriter.next(); - wayWriter.write(2); - wayWriter.closeNodeRefs(); - wayWriter.writeTag("key2", "Euro: \u20AC"); - wayWriter.writeTag("key4", "Pound: \u00A3"); - wayWriter.closeTags(); - wayWriter.next(); - - wayWriter.close(); - - byte[] wayBuf = nodeOut.toByteArray(); - byte[] keyBuf = keyOut.toByteArray(); - KeyLookupReader keyReader = new KeyLookupReader(new ByteArrayInputStream(keyBuf)); - ByteArrayInputStream wayIn = new ByteArrayInputStream(wayBuf); - WayStoreReader wayReader = new WayStoreReader(wayIn, keyReader); - - Assert.assertTrue(wayReader.next()); - Assert.assertEquals(wayReader.getId(), 1); - int[] nodeIds = wayReader.getNodeIds(); - Assert.assertEquals(nodeIds.length, 3); - Assert.assertEquals(nodeIds[0], 10); - Assert.assertEquals(nodeIds[1], 11); - Assert.assertEquals(nodeIds[2], 12); - Map<String, String> tags = wayReader.getTags(); - Assert.assertEquals(tags.size(), 3); - Assert.assertEquals(tags.get("key1"), "value1"); - Assert.assertEquals(tags.get("key2"), "value2"); - Assert.assertEquals(tags.get("key3"), "Euro: \u20AC Pound: \u00A3"); - Assert.assertNull(tags.get("key4")); - - Assert.assertTrue(wayReader.next()); - Assert.assertEquals(wayReader.getId(), 2); - nodeIds = wayReader.getNodeIds(); - Assert.assertEquals(nodeIds.length, 0); - tags = wayReader.getTags(); - Assert.assertEquals(tags.size(), 2); - Assert.assertNull(tags.get("key1")); - Assert.assertEquals(tags.get("key2"), "Euro: \u20AC"); - Assert.assertNull(tags.get("key3")); - Assert.assertEquals(tags.get("key4"), "Pound: \u00A3"); - - Assert.assertFalse(wayReader.next()); - } - - @Test - public void testRelStore() throws IOException { - ByteArrayOutputStream relOut = new ByteArrayOutputStream(); - ByteArrayOutputStream keyOut = new ByteArrayOutputStream(); - KeyLookupWriter keyWriter = new KeyLookupWriter(keyOut); - ByteArrayOutputStream roleOut = new ByteArrayOutputStream(); - KeyLookupWriter roleWriter = new KeyLookupWriter(roleOut); - - RelationStoreWriter relWriter = new RelationStoreWriter(relOut, keyWriter, roleWriter); - relWriter.write(1); - relWriter.writeMember(MemberType.Node, 5, "testRole1"); - relWriter.writeMember(MemberType.Node, 4, "testRole2"); - relWriter.writeMember(MemberType.Node, 3, "Euro: \u20AC"); - relWriter.writeMember(MemberType.Way, 3, "Pound: \u00A3"); - relWriter.closeMembers(); - relWriter.writeTag("key1", "value1"); - relWriter.writeTag("key2", "value2"); - relWriter.writeTag("key3", "Euro: \u20AC Pound: \u00A3"); - relWriter.closeTags(); - relWriter.next(); - relWriter.write(2); - relWriter.closeMembers(); - relWriter.writeTag("key2", "Euro: \u20AC"); - relWriter.writeTag("key4", "Pound: \u00A3"); - relWriter.closeTags(); - relWriter.next(); - - relWriter.close(); - - - byte[] relBuf = relOut.toByteArray(); - byte[] keyBuf = keyOut.toByteArray(); - KeyLookupReader keyReader = new KeyLookupReader(new ByteArrayInputStream(keyBuf)); - byte[] roleBuf = roleOut.toByteArray(); - KeyLookupReader roleReader = new KeyLookupReader(new ByteArrayInputStream(roleBuf)); - - ByteArrayInputStream relIn = new ByteArrayInputStream(relBuf); - RelationStoreReader relReader = new RelationStoreReader(relIn, keyReader, roleReader); - - Assert.assertTrue(relReader.next()); - Assert.assertEquals(relReader.getId(), 1); - List<Member> members = relReader.getMembers(); - Assert.assertEquals(members.size(), 4); - Assert.assertEquals(members.get(0).getType(), MemberType.Node); - Assert.assertEquals(members.get(0).getId(), 5); - Assert.assertEquals(members.get(0).getRole(), "testRole1"); - Assert.assertEquals(members.get(1).getType(), MemberType.Node); - Assert.assertEquals(members.get(1).getId(), 4); - Assert.assertEquals(members.get(1).getRole(), "testRole2"); - Assert.assertEquals(members.get(2).getType(), MemberType.Node); - Assert.assertEquals(members.get(2).getId(), 3); - Assert.assertEquals(members.get(2).getRole(), "Euro: \u20AC"); - Assert.assertEquals(members.get(3).getType(), MemberType.Way); - Assert.assertEquals(members.get(3).getId(), 3); - Assert.assertEquals(members.get(3).getRole(), "Pound: \u00A3"); - Map<String, String> tags = relReader.getTags(); - Assert.assertEquals(tags.size(), 3); - Assert.assertEquals(tags.get("key1"), "value1"); - Assert.assertEquals(tags.get("key2"), "value2"); - Assert.assertEquals(tags.get("key3"), "Euro: \u20AC Pound: \u00A3"); - Assert.assertNull(tags.get("key4")); - - Assert.assertTrue(relReader.next()); - Assert.assertEquals(relReader.getId(), 2); - members = relReader.getMembers(); - Assert.assertEquals(members.size(), 0); - tags = relReader.getTags(); - Assert.assertEquals(tags.size(), 2); - Assert.assertNull(tags.get("key1")); - Assert.assertEquals(tags.get("key2"), "Euro: \u20AC"); - Assert.assertNull(tags.get("key3")); - Assert.assertEquals(tags.get("key4"), "Pound: \u00A3"); - - Assert.assertFalse(relReader.next()); - } -} \ No newline at end of file -- 1.7.2.3
- Previous message: [mkgmap-dev] Binary OSM file support?
- Next message: [mkgmap-dev] [PATCH 02/14] Add support to intlist for getting as an array.
- Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]
More information about the mkgmap-dev mailing list