diff --git a/brouter-map-creator/build.gradle b/brouter-map-creator/build.gradle index 609bb50..7c48259 100644 --- a/brouter-map-creator/build.gradle +++ b/brouter-map-creator/build.gradle @@ -2,11 +2,12 @@ plugins { id 'java-library' } - dependencies { implementation project(':brouter-codec') implementation project(':brouter-util') implementation project(':brouter-expressions') - + + implementation group: 'org.openstreetmap.osmosis', name: 'osmosis-osm-binary', version: '0.48.3' + testImplementation('junit:junit:4.13.1') } diff --git a/misc/pbfparser/BPbfBlobDecoder.java b/brouter-map-creator/src/main/java/btools/mapcreator/BPbfBlobDecoder.java similarity index 96% rename from misc/pbfparser/BPbfBlobDecoder.java rename to brouter-map-creator/src/main/java/btools/mapcreator/BPbfBlobDecoder.java index 9ada4f5..bf26671 100644 --- a/misc/pbfparser/BPbfBlobDecoder.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/BPbfBlobDecoder.java @@ -5,14 +5,18 @@ import com.google.protobuf.InvalidProtocolBufferException; import org.openstreetmap.osmosis.osmbinary.Fileformat; import org.openstreetmap.osmosis.osmbinary.Osmformat; -import btools.util.LongList; - import java.io.IOException; -import java.util.*; -import java.util.logging.Level; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import java.util.zip.DataFormatException; import java.util.zip.Inflater; +import btools.util.LongList; + /** * Converts PBF block data into decoded entities ready to be passed into an Osmosis pipeline. This * class is designed to be passed into a pool of worker threads to allow multi-threaded decoding. @@ -82,8 +86,8 @@ public class BPbfBlobDecoder { // Build the list of active and unsupported features in the file. List supportedFeatures = Arrays.asList("OsmSchema-V0.6", "DenseNodes"); - List activeFeatures = new ArrayList(); - List unsupportedFeatures = new ArrayList(); + List activeFeatures = new ArrayList<>(); + List unsupportedFeatures = new ArrayList<>(); for (String feature : header.getRequiredFeaturesList()) { if (supportedFeatures.contains(feature)) { activeFeatures.add(feature); @@ -106,7 +110,7 @@ public class BPbfBlobDecoder { Iterator keyIterator = keys.iterator(); Iterator valueIterator = values.iterator(); if (keyIterator.hasNext()) { - Map tags = new HashMap(); + Map tags = new HashMap<>(); while (keyIterator.hasNext()) { String key = fieldDecoder.decodeString(keyIterator.next()); String value = fieldDecoder.decodeString(valueIterator.next()); @@ -155,7 +159,7 @@ public class BPbfBlobDecoder { int valueIndex = keysValuesIterator.next(); if (tags == null) { - tags = new HashMap(); + tags = new HashMap<>(); } tags.put(fieldDecoder.decodeString(keyIndex), fieldDecoder.decodeString(valueIndex)); diff --git a/misc/pbfparser/BPbfFieldDecoder.java b/brouter-map-creator/src/main/java/btools/mapcreator/BPbfFieldDecoder.java similarity index 100% rename from misc/pbfparser/BPbfFieldDecoder.java rename to brouter-map-creator/src/main/java/btools/mapcreator/BPbfFieldDecoder.java diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/OsmParser.java b/brouter-map-creator/src/main/java/btools/mapcreator/OsmParser.java index 9539a51..f188031 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/OsmParser.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/OsmParser.java @@ -1,10 +1,17 @@ package btools.mapcreator; +import org.openstreetmap.osmosis.osmbinary.Fileformat; + +import java.io.BufferedInputStream; import java.io.BufferedReader; +import java.io.DataInputStream; +import java.io.EOFException; import java.io.File; import java.io.FileInputStream; -import java.io.InputStreamReader; -import java.util.zip.GZIPInputStream; +import java.util.HashMap; +import java.util.Map; + +import btools.util.LongList; /** * Parser for OSM data @@ -22,179 +29,118 @@ public class OsmParser extends MapCreatorBase { NodeListener nListener, WayListener wListener, RelationListener rListener) throws Exception { - this.nListener = nListener; this.wListener = wListener; this.rListener = rListener; - if (mapFile == null) { - _br = new BufferedReader(new InputStreamReader(System.in)); - } else { - if (mapFile.getName().endsWith(".gz")) { - _br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(mapFile)))); - } else { - _br = new BufferedReader(new InputStreamReader(new FileInputStream(mapFile))); + System.out.println("*** PBF Parsing: " + mapFile); + + // once more for testing + int rawBlobCount = 0; + + long bytesRead = 0L; + Boolean avoidMapPolling = Boolean.getBoolean("avoidMapPolling"); + + if (!avoidMapPolling) { + // wait for file to become available + while (!mapFile.exists()) { + System.out.println("--- waiting for " + mapFile + " to become available"); + Thread.sleep(10000); } } + long currentSize = mapFile.length(); + long currentSizeTime = System.currentTimeMillis(); + + DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(mapFile))); + + for (; ; ) { - String line = _br.readLine(); - if (line == null) break; - - if (checkNode(line)) continue; - if (checkWay(line)) continue; - if (checkRelation(line)) continue; - if (checkChangeset(line)) continue; - } - - if (mapFile != null) { - _br.close(); - } - } - - - private boolean checkNode(String line) throws Exception { - int idx0 = line.indexOf("")) { - // read additional tags - for (; ; ) { - String l2 = _br.readLine(); - if (l2 == null) return false; - - int i2; - if ((i2 = l2.indexOf("= 0) { // property-tag - i2 += 8; - int ri2 = l2.indexOf('"', i2); - String key = l2.substring(i2, ri2); - i2 = l2.indexOf(" v=\"", ri2); - if (i2 >= 0) { - i2 += 4; - int ri3 = l2.indexOf('"', i2); - String value = l2.substring(i2, ri3); - - n.putTag(key, value); + if (!avoidMapPolling) { + // continue reading if either more then a 100 MB unread, or the current-size is known for more than 2 Minutes + while (currentSize - bytesRead < 100000000L) { + long newSize = mapFile.length(); + if (newSize != currentSize) { + currentSize = newSize; + currentSizeTime = System.currentTimeMillis(); + } else if (System.currentTimeMillis() - currentSizeTime > 120000) { + break; + } + if (currentSize - bytesRead < 100000000L) { + System.out.println("--- waiting for more data, currentSize=" + currentSize + " bytesRead=" + bytesRead); + Thread.sleep(10000); } - } else if (l2.indexOf("") >= 0) { // end-tag - break; } } - } - nListener.nextNode(n); - return true; - } - - private boolean checkWay(String line) throws Exception { - int idx0 = line.indexOf("= 0) { // node reference - i2 += 9; - int ri2 = l2.indexOf('"', i2); - long nid = Long.parseLong(l2.substring(i2, ri2)); - w.nodes.add(nid); - } else if ((i2 = l2.indexOf("= 0) { // property-tag - i2 += 8; - int ri2 = l2.indexOf('"', i2); - String key = l2.substring(i2, ri2); - i2 = l2.indexOf(" v=\"", ri2); - if (i2 >= 0) { - i2 += 4; - int ri3 = l2.indexOf('"', i2); - String value = l2.substring(i2, ri3); - w.putTag(key, value); - } - } else if (l2.indexOf("") >= 0) { // end-tag + int headerLength; + try { + headerLength = dis.readInt(); + bytesRead += 4; + } catch (EOFException e) { break; } + + byte[] headerBuffer = new byte[headerLength]; + dis.readFully(headerBuffer); + bytesRead += headerLength; + Fileformat.BlobHeader blobHeader = Fileformat.BlobHeader.parseFrom(headerBuffer); + + byte[] blobData = new byte[blobHeader.getDatasize()]; + dis.readFully(blobData); + bytesRead += blobData.length; + + new BPbfBlobDecoder(blobHeader.getType(), blobData, this).process(); + + rawBlobCount++; } - wListener.nextWay(w); - return true; + dis.close(); + System.out.println("read raw blobs: " + rawBlobCount); } - private boolean checkChangeset(String line) throws Exception { - int idx0 = line.indexOf("")) { - int loopcheck = 0; - for (; ; ) { - String l2 = _br.readLine(); - if (l2.indexOf("") >= 0 || ++loopcheck > 10000) break; - } + public void addNode(long nid, Map tags, double lat, double lon) { + NodeData n = new NodeData(nid, lon, lat); + n.setTags((HashMap) tags); + try { + nListener.nextNode(n); + } catch (Exception e) { + throw new RuntimeException("error writing node: " + e); } - return true; } - private boolean checkRelation(String line) throws Exception { - int idx0 = line.indexOf(" tags, LongList nodes) { + WayData w = new WayData(wid, nodes); + w.setTags((HashMap) tags); - idx0 += 14; - int idx1 = line.indexOf('"', idx0); - long rid = Long.parseLong(line.substring(idx0, idx1)); + try { + wListener.nextWay(w); + } catch (Exception e) { + throw new RuntimeException("error writing way: " + e); + } + } - RelationData r = new RelationData(rid); + public void addRelation(long rid, Map tags, LongList wayIds, LongList fromWid, LongList toWid, LongList viaNid) { + RelationData r = new RelationData(rid, wayIds); + r.setTags((HashMap) tags); - // read the nodes - for (; ; ) { - String l2 = _br.readLine(); - if (l2 == null) return false; - - int i2; - if ((i2 = l2.indexOf("= 0) { // node reference - i2 += 24; - int ri2 = l2.indexOf('"', i2); - long wid = Long.parseLong(l2.substring(i2, ri2)); - r.ways.add(wid); - } else if ((i2 = l2.indexOf("= 0) { // property-tag - i2 += 8; - int ri2 = l2.indexOf('"', i2); - String key = l2.substring(i2, ri2); - i2 = l2.indexOf(" v=\"", ri2); - if (i2 >= 0) { - i2 += 4; - int ri3 = l2.indexOf('"', i2); - String value = l2.substring(i2, ri3); - r.putTag(key, value); + try { + rListener.nextRelation(r); + if (fromWid == null || toWid == null || viaNid == null || viaNid.size() != 1) { + // dummy-TR for each viaNid + for (int vi = 0; vi < (viaNid == null ? 0 : viaNid.size()); vi++) { + rListener.nextRestriction(r, 0L, 0L, viaNid.get(vi)); } - } else if (l2.indexOf("") >= 0) { // end-tag - break; + return; } + for (int fi = 0; fi < fromWid.size(); fi++) { + for (int ti = 0; ti < toWid.size(); ti++) { + rListener.nextRestriction(r, fromWid.get(fi), toWid.get(ti), viaNid.get(0)); + } + } + } catch (Exception e) { + throw new RuntimeException("error writing relation", e); } - rListener.nextRelation(r); - return true; } } diff --git a/brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java b/brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java index 5daae74..562d4d2 100644 --- a/brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java +++ b/brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java @@ -9,8 +9,10 @@ import java.net.URL; public class MapcreatorTest { @Test public void mapcreatorTest() throws Exception { - URL mapurl = this.getClass().getResource("/dreieich.osm.gz"); - Assert.assertNotNull("test-osm-map dreieich.osm not found", mapurl); + System.setProperty("avoidMapPolling", "true"); + + URL mapurl = this.getClass().getResource("/dreieich.pbf"); + Assert.assertNotNull("test-osm-map dreieich.pbf not found", mapurl); File mapFile = new File(mapurl.getFile()); File workingDir = mapFile.getParentFile(); File profileDir = new File(workingDir, "/../../../../misc/profiles2"); diff --git a/brouter-map-creator/src/test/resources/dreieich.pbf b/brouter-map-creator/src/test/resources/dreieich.pbf new file mode 100644 index 0000000..0fe3927 Binary files /dev/null and b/brouter-map-creator/src/test/resources/dreieich.pbf differ diff --git a/docs/developers/build_segments.md b/docs/developers/build_segments.md index 604d8eb..513fab5 100644 --- a/docs/developers/build_segments.md +++ b/docs/developers/build_segments.md @@ -13,43 +13,6 @@ also build them yourself from an OSM dump (e.g. planet or [GeoFabrik extract](https://download.geofabrik.de/)) -## Build the pbfparser - -First, there are two file formats available to download OSM data: `bzip`-ed -XML files (very large) and `.pbf` -([Protobuf](https://github.com/protocolbuffers/protobuf) format) which is much -more efficient. If you want to use the latter one, you will have to build the -`pbfparser` (located in `misc/pbfparser` first): - -* Download [the latest - version](https://github.com/openstreetmap/osmosis/releases) - of [Osmosis](https://wiki.openstreetmap.org/wiki/Osmosis) and unzip it - somewhere. -* Copy the `lib/default/protobuf-java-*.jar` and - `lib/default/osmosis-osm-binary-*.jar` files from the unzipped Osmosis - archive to `misc/pbfparser/protobuf.jar` and `misc/pbfparser/osmosis.jar`. -* Build BRouter and copy - `brouter-server/build/libs/brouter-*-all.jar` to - `misc/pbfparser/brouter.jar`. -* You can build the `pbfparser` using, in the `misc/pbfparser/` - folder, - -``` -javac -d . -cp "brouter.jar:protobuf.jar:osmosis.jar" *.java -``` - -* Finally, you can build a `jar` file from these files using - -``` -jar cf pbfparser.jar btools/**/*.class -``` - -_Note:_ If the `jar` file is not properly created, everything else will seem -to work normally but there will not be any data extracted from the OSM data -dump. You can check what is actually inside the built `jar` file using -`jar tf pbfparser.jar`. - - ## Run the map creation script If you want to have elevation information in the generated segments files, you diff --git a/misc/pbfparser/.gitignore b/misc/pbfparser/.gitignore deleted file mode 100644 index 4dd82da..0000000 --- a/misc/pbfparser/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.jar -*.BAK -btools/ diff --git a/misc/pbfparser/OsmParser.java b/misc/pbfparser/OsmParser.java deleted file mode 100644 index ce5f8a4..0000000 --- a/misc/pbfparser/OsmParser.java +++ /dev/null @@ -1,136 +0,0 @@ -package btools.mapcreator; - -import java.io.*; -import java.util.*; -import java.util.zip.*; - -import btools.util.*; - -import org.openstreetmap.osmosis.osmbinary.Fileformat; - -/** - * Parser for OSM data - * - * @author ab - */ -public class OsmParser extends MapCreatorBase { - private BufferedReader _br; - - private NodeListener nListener; - private WayListener wListener; - private RelationListener rListener; - - public void readMap(File mapFile, - NodeListener nListener, - WayListener wListener, - RelationListener rListener) throws Exception { - this.nListener = nListener; - this.wListener = wListener; - this.rListener = rListener; - - System.out.println("*** PBF Parsing: " + mapFile); - - // once more for testing - int rawBlobCount = 0; - - long bytesRead = 0L; - - // wait for file to become available - while (!mapFile.exists()) { - System.out.println("--- waiting for " + mapFile + " to become available"); - Thread.sleep(10000); - } - - long currentSize = mapFile.length(); - long currentSizeTime = System.currentTimeMillis(); - - DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(mapFile))); - - - for (; ; ) { - // continue reading if either more then a 100 MB unread, or the current-size is known for more then 2 Minutes - while (currentSize - bytesRead < 100000000L) { - long newSize = mapFile.length(); - if (newSize != currentSize) { - currentSize = newSize; - currentSizeTime = System.currentTimeMillis(); - } else if (System.currentTimeMillis() - currentSizeTime > 120000) { - break; - } - if (currentSize - bytesRead < 100000000L) { - System.out.println("--- waiting for more data, currentSize=" + currentSize + " bytesRead=" + bytesRead); - Thread.sleep(10000); - } - } - - int headerLength; - try { - headerLength = dis.readInt(); - bytesRead += 4; - } catch (EOFException e) { - break; - } - - byte[] headerBuffer = new byte[headerLength]; - dis.readFully(headerBuffer); - bytesRead += headerLength; - Fileformat.BlobHeader blobHeader = Fileformat.BlobHeader.parseFrom(headerBuffer); - - byte[] blobData = new byte[blobHeader.getDatasize()]; - dis.readFully(blobData); - bytesRead += blobData.length; - - new BPbfBlobDecoder(blobHeader.getType(), blobData, this).process(); - - rawBlobCount++; - } - dis.close(); - System.out.println("read raw blobs: " + rawBlobCount); - } - - - public void addNode(long nid, Map tags, double lat, double lon) { - NodeData n = new NodeData(nid, lon, lat); - n.setTags(tags); - try { - nListener.nextNode(n); - } catch (Exception e) { - throw new RuntimeException("error writing node: " + e); - } - } - - public void addWay(long wid, Map tags, LongList nodes) { - WayData w = new WayData(wid, nodes); - w.setTags((HashMap) tags); - - try { - wListener.nextWay(w); - } catch (Exception e) { - throw new RuntimeException("error writing way: " + e); - } - } - - public void addRelation(long rid, Map tags, LongList wayIds, LongList fromWid, LongList toWid, LongList viaNid) { - RelationData r = new RelationData(rid, wayIds); - r.setTags((HashMap) tags); - - try { - rListener.nextRelation(r); - if (fromWid == null || toWid == null || viaNid == null || viaNid.size() != 1) { - // dummy-TR for each viaNid - for (int vi = 0; vi < (viaNid == null ? 0 : viaNid.size()); vi++) { - rListener.nextRestriction(r, 0L, 0L, viaNid.get(vi)); - } - return; - } - for (int fi = 0; fi < fromWid.size(); fi++) { - for (int ti = 0; ti < toWid.size(); ti++) { - rListener.nextRestriction(r, fromWid.get(fi), toWid.get(ti), viaNid.get(0)); - } - } - } catch (Exception e) { - throw new RuntimeException("error writing relation", e); - } - } - -} diff --git a/misc/pbfparser/README.txt b/misc/pbfparser/README.txt deleted file mode 100644 index b43e019..0000000 --- a/misc/pbfparser/README.txt +++ /dev/null @@ -1,23 +0,0 @@ -The pbf-parse is not included in the regular source tree -to avoid the library dependencies to "osmosis" and "protobuf" - -In order to run the mapcreator from a pbf-file (as it is -done in the process_pbf_planet.sh script included in -the git-repo), you have to build yourself the "pbfparser.jar" -by doing the following: - --> get osmosis from https://bretth.dev.openstreetmap.org/osmosis-build/osmosis-latest.zip --> copy lib/default/osmosis-osm-binary-*.jar in the archive to osmosis.jar in -this folder --> copy lib/default/protobuf-java-*.jar in the archive to protobuf.jar in this -folder --> copy the brouter-server/build/libs/brouter-...-all.jar to -brouter.jar in this folder --> compile the PBF-Parser using: - javac -d . -cp protobuf.jar:osmosis.jar:brouter.jar *.java --> pack all the compiled class files together in a jar -"pbfparser.jar" with "jar cf pbfparser.jar btools/**/*.class" - -Alternatively, just for testing you can run the Mapcreator against a *xml.bz2 Database-Extract, -then you don't need the pbf-parser. However, the XML-Parser does not (yet) parse -Turn-Restrictions, so really just for testing... diff --git a/misc/pbfparser/compile_parser.bat b/misc/pbfparser/compile_parser.bat deleted file mode 100644 index 7a5452b..0000000 --- a/misc/pbfparser/compile_parser.bat +++ /dev/null @@ -1 +0,0 @@ -javac -d . -cp pbfparser.jar;brouter.jar BPbfFieldDecoder.java BPbfBlobDecoder.java OsmParser.java diff --git a/misc/scripts/mapcreation/process_pbf_planet.sh b/misc/scripts/mapcreation/process_pbf_planet.sh index 2ad173d..d3b190e 100755 --- a/misc/scripts/mapcreation/process_pbf_planet.sh +++ b/misc/scripts/mapcreation/process_pbf_planet.sh @@ -24,14 +24,11 @@ touch lastmaprun.date rm -rf /var/www/brouter/segments4_lastrun -JAVA='/java/bin/java -Xmx2600m -Xms2600m -Xmn32m' +JAVA='java -Xmx2600m -Xms2600m -Xmn32m' BROUTER_PROFILES=$(realpath "../../profiles2") BROUTER_JAR=$(realpath $(ls ../../../brouter-server/build/libs/brouter-*-all.jar)) -OSMOSIS_JAR=$(realpath "../../pbfparser/osmosis.jar") -PROTOBUF_JAR=$(realpath "../../pbfparser/protobuf.jar") -PBFPARSER_JAR=$(realpath "../../pbfparser/pbfparser.jar") PLANET_FILE=${PLANET_FILE:-$(realpath "./planet-latest.osm.pbf")} # Download SRTM zip files from @@ -43,7 +40,7 @@ SRTM_PATH="/private-backup/srtm" mkdir tmp cd tmp mkdir nodetiles -${JAVA} -cp "${OSMOSIS_JAR}:${PROTOBUF_JAR}:${PBFPARSER_JAR}:${BROUTER_JAR}" btools.mapcreator.OsmCutter ${BROUTER_PROFILES}/lookups.dat nodetiles ways.dat relations.dat restrictions.dat ${BROUTER_PROFILES}/all.brf ${PLANET_FILE} +${JAVA} -cp ${BROUTER_JAR} -DavoidMapPolling=true btools.mapcreator.OsmCutter ${BROUTER_PROFILES}/lookups.dat nodetiles ways.dat relations.dat restrictions.dat ${BROUTER_PROFILES}/all.brf ${PLANET_FILE} mkdir ftiles ${JAVA} -cp ${BROUTER_JAR} -Ddeletetmpfiles=true -DuseDenseMaps=true btools.mapcreator.NodeFilter nodetiles ways.dat ftiles diff --git a/misc/scripts/mapcreation/process_pbf_planet_production.sh b/misc/scripts/mapcreation/process_pbf_planet_production.sh index 4da46f6..f9a6deb 100644 --- a/misc/scripts/mapcreation/process_pbf_planet_production.sh +++ b/misc/scripts/mapcreation/process_pbf_planet_production.sh @@ -15,7 +15,7 @@ mkdir waytiles mkdir waytiles55 mkdir nodes55 -../../jdk8/bin/java -Xmx6144M -Xms6144M -Xmn256M -cp ../pbfparser.jar:../brouter_fc.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.util.StackSampler btools.mapcreator.OsmFastCutter ../lookups.dat nodetiles waytiles nodes55 waytiles55 bordernids.dat relations.dat restrictions.dat ../all.brf ../trekking.brf ../softaccess.brf ../planet-new.osm.pbf +../../jdk8/bin/java -Xmx6144M -Xms6144M -Xmn256M -cp ../brouter_fc.jar -Ddeletetmpfiles=true -DuseDenseMaps=true btools.util.StackSampler btools.mapcreator.OsmFastCutter ../lookups.dat nodetiles waytiles nodes55 waytiles55 bordernids.dat relations.dat restrictions.dat ../all.brf ../trekking.brf ../softaccess.brf ../planet-new.osm.pbf mv ../planet-latest.osm.pbf ../planet-old.osm.pbf mv ../planet-new.osm.pbf ../planet-latest.osm.pbf