[mkgmap-dev] [PATCH 06/14] Binary Map parser class port to splitter.
From Jeffrey C. Ollie jeff at ocjtech.us on Thu Sep 9 21:12:06 BST 2010
From: Scott Crosby <scrosby at cs.rice.edu> --- src/uk/me/parabola/splitter/BinaryMapParser.java | 167 ++++++++++++++++++++++ src/uk/me/parabola/splitter/Main.java | 56 ++++++-- src/uk/me/parabola/splitter/MapReader.java | 10 +- src/uk/me/parabola/splitter/OSMParser.java | 6 +- 4 files changed, 216 insertions(+), 23 deletions(-) create mode 100644 src/uk/me/parabola/splitter/BinaryMapParser.java diff --git a/src/uk/me/parabola/splitter/BinaryMapParser.java b/src/uk/me/parabola/splitter/BinaryMapParser.java new file mode 100644 index 0000000..4d76b01 --- /dev/null +++ b/src/uk/me/parabola/splitter/BinaryMapParser.java @@ -0,0 +1,167 @@ +package uk.me.parabola.splitter; + +import java.util.List; + + +import crosby.binary.BinaryParser; +import crosby.binary.Osmformat; + +public class BinaryMapParser extends BinaryParser { + // How many elements to process before displaying a status update + private static final int NODE_STATUS_UPDATE_THRESHOLD = 2500000; + private static final int WAY_STATUS_UPDATE_THRESHOLD = 500000; + private static final int RELATION_STATUS_UPDATE_THRESHOLD = 50000; + + + private long nodeCount; + private long wayCount; + private long relationCount; + + BinaryMapParser(MapProcessor processor) { + this.processor = processor; + } + MapProcessor processor; + + public void complete() { + // End of map is sent when all input files are processed. + // So do nothing. + } + + // Per-block state for parsing, set when processing the header of a block; + protected void parseDense(Osmformat.DenseNodes nodes) { + long last_id = 0, last_lat = 0, last_lon = 0; + int j = 0; + for (int i=0 ; i < nodes.getIdCount(); i++) { + Node tmp = new Node(); + long lat = nodes.getLat(i)+last_lat; last_lat = lat; + long lon = nodes.getLon(i)+last_lon; last_lon = lon; + long id = nodes.getId(i)+last_id; last_id = id; + double latf = parseLat(lat), lonf = parseLon(lon); + tmp = new Node(); + tmp.set((int)id, latf, lonf); + if (nodes.getKeysValsCount() > 0) { + while (nodes.getKeysVals(j) != 0) { + int keyid = nodes.getKeysVals(j++); + int valid = nodes.getKeysVals(j++); + tmp.addTag(getStringById(keyid),getStringById(valid)); + } + j++; // Skip over the '0' delimiter. + } + processor.processNode(tmp); + processNodes(); + } + } + + protected void parseNodes(List<Osmformat.Node> nodes) { + for (Osmformat.Node i : nodes) { + Node tmp = new Node(); + for (int j=0 ; j < i.getKeysCount(); j++) + tmp.addTag(getStringById(i.getKeys(j)),getStringById(i.getVals(j))); + long id = i.getId(); + double latf = parseLat(i.getLat()), lonf = parseLon(i.getLon()); + + tmp.set((int)id, latf, lonf); + + processor.processNode(tmp); + processNodes(); + } + } + + + protected void parseWays(List<Osmformat.Way> ways) { + for (Osmformat.Way i : ways) { + Way tmp = new Way(); + for (int j=0 ; j < i.getKeysCount(); j++) + tmp.addTag(getStringById(i.getKeys(j)),getStringById(i.getVals(j))); + + long last_id=0; + for (long j : i.getRefsList()) { + tmp.addRef((int)(j+last_id)); + last_id = j+last_id; + } + + long id = i.getId(); + tmp.set((int)id); + + processor.processWay(tmp); + processWays(); + } + } + protected void parseRelations(List<Osmformat.Relation> rels) { + for (Osmformat.Relation i : rels) { + Relation tmp = new Relation(); + for (int j=0 ; j < i.getKeysCount(); j++) + tmp.addTag(getStringById(i.getKeys(j)),getStringById(i.getVals(j))); + + long id = i.getId(); + tmp.set((int)id); + + long last_mid=0; + for (int j =0; j < i.getMemidsCount() ; j++) { + long mid = last_mid + i.getMemids(j); + last_mid = mid; + String role = getStringById(i.getRolesSid(j)); + String etype=null; + + if (i.getTypes(j) == Osmformat.Relation.MemberType.NODE) + etype = "node"; + else if (i.getTypes(j) == Osmformat.Relation.MemberType.WAY) + etype = "way"; + else if (i.getTypes(j) == Osmformat.Relation.MemberType.RELATION) + continue; + else + assert false; // TODO; Illegal file? + + tmp.addMember(etype,(int)mid,role); + } + processor.processRelation(tmp); + processRelations(); + } + } + + public void parse(Osmformat.HeaderBlock block) { + double multiplier = .000000001; + double rightf = block.getBbox().getRight() * multiplier; + double leftf = block.getBbox().getLeft() * multiplier; + double topf = block.getBbox().getTop() * multiplier; + double bottomf = block.getBbox().getBottom() * multiplier; + + for (String s : block.getRequiredFeaturesList()) { + if (s.equals("OsmSchema-V0.6")) continue; // OK. + if (s.equals("DenseNodes")) continue; // OK. + throw new Error("File requires unknown feature: " + s); + } + + System.out.println("Bounding box "+leftf+" "+bottomf+" "+rightf+" "+topf); + + Area area = new Area( + Utils.toMapUnit(bottomf), + Utils.toMapUnit(leftf), + Utils.toMapUnit(topf), + Utils.toMapUnit(rightf)); + processor.boundTag(area); + } + + + private void processNodes() { + nodeCount++; + if (nodeCount % NODE_STATUS_UPDATE_THRESHOLD == 0) { + System.out.println(Utils.format(nodeCount) + " nodes processed..."); + } + +} + +private void processWays() { + wayCount++; + if (wayCount % WAY_STATUS_UPDATE_THRESHOLD == 0) { + System.out.println(Utils.format(wayCount) + " ways processed..."); + } +} +private void processRelations() { + relationCount++; + if (relationCount % RELATION_STATUS_UPDATE_THRESHOLD == 0) { + System.out.println(Utils.format(relationCount) + " ways processed..."); + } +} + +} diff --git a/src/uk/me/parabola/splitter/Main.java b/src/uk/me/parabola/splitter/Main.java index 84e5dd7..b5469c6 100644 --- a/src/uk/me/parabola/splitter/Main.java +++ b/src/uk/me/parabola/splitter/Main.java @@ -13,6 +13,9 @@ package uk.me.parabola.splitter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; @@ -34,6 +37,8 @@ import uk.me.parabola.splitter.geo.DummyCityFinder; import org.xmlpull.v1.XmlPullParserException; +import crosby.binary.file.BlockInputStream; + /** * Splitter for OSM files with the purpose of providing input files for mkgmap. * <p/> @@ -264,14 +269,15 @@ public class Main { MapCollector nodes = densityMap ? new DensityMapCollector(trim, resolution) : new NodeCollector(); MapProcessor processor = nodes; - MapReader mapReader = processMap(processor); - System.out.print("A total of " + Utils.format(mapReader.getNodeCount()) + " nodes, " + - Utils.format(mapReader.getWayCount()) + " ways and " + - Utils.format(mapReader.getRelationCount()) + " relations were processed "); + //MapReader mapReader = + processMap(processor); + //System.out.print("A total of " + Utils.format(mapReader.getNodeCount()) + " nodes, " + + // Utils.format(mapReader.getWayCount()) + " ways and " + + // Utils.format(mapReader.getRelationCount()) + " relations were processed "); - System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files")); - System.out.println("Min node ID = " + mapReader.getMinNodeId()); - System.out.println("Max node ID = " + mapReader.getMaxNodeId()); + //System.out.println("in " + filenames.size() + (filenames.size() == 1 ? " file" : " files")); + //System.out.println("Min node ID = " + mapReader.getMinNodeId()); + //System.out.println("Max node ID = " + mapReader.getMaxNodeId()); System.out.println("Time: " + new Date()); @@ -342,10 +348,10 @@ public class Main { areas.get(i * areasPerPass + currentWriters.length - 1).getMapId() + ')'); MapProcessor processor = new SplitProcessor(currentWriters, maxThreads); - MapReader mapReader = processMap(processor); - System.out.println("Wrote " + Utils.format(mapReader.getNodeCount()) + " nodes, " + - Utils.format(mapReader.getWayCount()) + " ways, " + - Utils.format(mapReader.getRelationCount()) + " relations"); + processMap(processor); + //System.out.println("Wrote " + Utils.format(mapReader.getNodeCount()) + " nodes, " + + // Utils.format(mapReader.getWayCount()) + " ways, " + + // Utils.format(mapReader.getRelationCount()) + " relations"); } } @@ -366,12 +372,32 @@ public class Main { parse(parser, reader); } else { for (String filename : filenames) { - System.out.println("Processing " + filename + "..."); - Reader reader = Utils.openFile(filename, maxThreads > 1); - parse(parser, reader); + try { + if (filename.endsWith(".bin")) { + // Is it a binary file? + File file = new File(filename); + BlockInputStream blockinput = (new BlockInputStream( + new FileInputStream(file), new BinaryMapParser(processor))); + try { + blockinput.process(); + } finally { + blockinput.close(); + } + } else { + // No, try XML. + Reader reader = Utils.openFile(filename, maxThreads > 1); + parse(parser, reader); + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (XmlPullParserException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } } } - parser.endMap(); + processor.endMap(); } private void parse(OSMParser parser, Reader reader) throws IOException, XmlPullParserException { diff --git a/src/uk/me/parabola/splitter/MapReader.java b/src/uk/me/parabola/splitter/MapReader.java index 8afa12b..d8a8a21 100644 --- a/src/uk/me/parabola/splitter/MapReader.java +++ b/src/uk/me/parabola/splitter/MapReader.java @@ -14,13 +14,13 @@ package uk.me.parabola.splitter; public interface MapReader { - long getNodeCount(); + //long getNodeCount(); - long getWayCount(); + //long getWayCount(); - long getRelationCount(); + //long getRelationCount(); - int getMinNodeId(); + //int getMinNodeId(); - int getMaxNodeId(); + //int getMaxNodeId(); } diff --git a/src/uk/me/parabola/splitter/OSMParser.java b/src/uk/me/parabola/splitter/OSMParser.java index 50569be..1388261 100644 --- a/src/uk/me/parabola/splitter/OSMParser.java +++ b/src/uk/me/parabola/splitter/OSMParser.java @@ -51,7 +51,7 @@ class OSMParser extends AbstractXppParser implements MapReader { this.startNodeOnly = processor.isStartNodeOnly(); this.mixed = mixed; } - + /* @Override public long getNodeCount() { return nodeCount; @@ -76,11 +76,11 @@ class OSMParser extends AbstractXppParser implements MapReader { public int getMaxNodeId() { return maxNodeId; } - + */ public void endMap() { processor.endMap(); } - + /** * Receive notification of the start of an element. */ -- 1.7.2.3
- Previous message: [mkgmap-dev] [PATCH 05/14] Refactor to pass Relations as objects.
- Next message: [mkgmap-dev] [PATCH 07/14] Implement code to only send nodes to the set of writers that actually use them.
- Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]
More information about the mkgmap-dev mailing list