diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/NodeCutter.java b/brouter-map-creator/src/main/java/btools/mapcreator/NodeCutter.java index 0eab31a..8dc20ed 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/NodeCutter.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/NodeCutter.java @@ -13,7 +13,7 @@ public class NodeCutter extends MapCreatorBase { private int lonoffset; private int latoffset; - + public static void main(String[] args) throws Exception { System.out.println("*** NodeCutter: Cut big node-tiles into 5x5 tiles"); @@ -25,9 +25,14 @@ public class NodeCutter extends MapCreatorBase new NodeCutter().process( new File( args[0] ), new File( args[1] ) ); } - public void process( File nodeTilesIn, File nodeTilesOut ) throws Exception + public void init( File nodeTilesOut ) { this.outTileDir = nodeTilesOut; + } + + public void process( File nodeTilesIn, File nodeTilesOut ) throws Exception + { + init( nodeTilesOut ); new NodeIterator( this, true ).processDir( nodeTilesIn, ".tlf" ); } diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/NodeFilter.java b/brouter-map-creator/src/main/java/btools/mapcreator/NodeFilter.java index 3ab42a5..f40b2fe 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/NodeFilter.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/NodeFilter.java @@ -33,12 +33,18 @@ public class NodeFilter extends MapCreatorBase new NodeFilter().process( new File( args[0] ), new File( args[1] ), new File( args[2] ) ); } + public void init() throws Exception + { + nodebitmap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap( 512 ) : new TinyDenseLongMap(); + } + public void process( File nodeTilesIn, File wayFileIn, File nodeTilesOut ) throws Exception { + init(); + this.nodeTilesOut = nodeTilesOut; // read the wayfile into a bitmap of used nodes - nodebitmap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap( 512 ) : new TinyDenseLongMap(); new WayIterator( this, false ).processFile( wayFileIn ); // finally filter all node files @@ -59,7 +65,6 @@ public class NodeFilter extends MapCreatorBase public void nodeFileStart( File nodefile ) throws Exception { String filename = nodefile.getName(); - filename = filename.substring( 0, filename.length() - 3 ) + "tlf"; File outfile = new File( nodeTilesOut, filename ); nodesOutStream = new DiffCoderDataOutputStream( new BufferedOutputStream ( new FileOutputStream( outfile ) ) ); } @@ -67,13 +72,18 @@ public class NodeFilter extends MapCreatorBase @Override public void nextNode( NodeData n ) throws Exception { - // check if node passes bitmap - if ( nodebitmap.getInt( n.nid ) == 0 ) // 0 -> bit set, -1 -> unset + if ( isRelevant( n ) ) { n.writeTo( nodesOutStream ); } } + public boolean isRelevant( NodeData n ) + { + // check if node passes bitmap + return nodebitmap.getInt( n.nid ) == 0; // 0 -> bit set, -1 -> unset + } + @Override public void nodeFileEnd( File nodeFile ) throws Exception { diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java b/brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java index 63ee8b1..7d38faa 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java @@ -12,6 +12,7 @@ import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.util.HashMap; +import java.util.Map; import btools.expressions.BExpressionContextNode; import btools.expressions.BExpressionContextWay; @@ -29,6 +30,9 @@ public class OsmCutter extends MapCreatorBase private DataOutputStream cyclewayDos; private DataOutputStream restrictionsDos; + public WayCutter wayCutter; + public NodeFilter nodeFilter; + public static void main(String[] args) throws Exception { System.out.println("*** OsmCutter: cut an osm map in node-tiles + a way file"); @@ -53,8 +57,8 @@ public class OsmCutter extends MapCreatorBase private BExpressionContextWay _expctxWay; private BExpressionContextNode _expctxNode; - private BExpressionContextWay _expctxWayStat; - private BExpressionContextNode _expctxNodeStat; + // private BExpressionContextWay _expctxWayStat; + // private BExpressionContextNode _expctxNodeStat; public void process (File lookupFile, File outTileDir, File wayFile, File relFile, File resFile, File profileFile, File mapFile ) throws Exception { @@ -71,13 +75,13 @@ public class OsmCutter extends MapCreatorBase _expctxWay.parseFile( profileFile, "global" ); - _expctxWayStat = new BExpressionContextWay( null ); - _expctxNodeStat = new BExpressionContextNode( null ); + // _expctxWayStat = new BExpressionContextWay( null ); + // _expctxNodeStat = new BExpressionContextNode( null ); this.outTileDir = outTileDir; if ( !outTileDir.isDirectory() ) throw new RuntimeException( "out tile directory " + outTileDir + " does not exist" ); - wayDos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( wayFile ) ) ); + wayDos = wayFile == null ? null : new DataOutputStream( new BufferedOutputStream( new FileOutputStream( wayFile ) ) ); cyclewayDos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( relFile ) ) ); restrictionsDos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( resFile ) ) ); @@ -90,7 +94,10 @@ public class OsmCutter extends MapCreatorBase // close all files closeTileOutStreams(); - wayDos.close(); + if ( wayDos != null ) + { + wayDos.close(); + } cyclewayDos.close(); restrictionsDos.close(); @@ -122,11 +129,10 @@ public class OsmCutter extends MapCreatorBase if ( n.getTagsOrNull() != null ) { int[] lookupData = _expctxNode.createNewLookupData(); - for( String key : n.getTagsOrNull().keySet() ) + for( Map.Entry e : n.getTagsOrNull().entrySet() ) { - String value = n.getTag( key ); - _expctxNode.addLookupValue( key, value, lookupData ); - _expctxNodeStat.addLookupValue( key, value, null ); + _expctxNode.addLookupValue( e.getKey(), e.getValue(), lookupData ); + // _expctxNodeStat.addLookupValue( key, value, null ); } n.description = _expctxNode.encode(lookupData); } @@ -136,6 +142,10 @@ public class OsmCutter extends MapCreatorBase { n.writeTo( getOutStreamForTile( tileIndex ) ); } + if ( wayCutter != null ) + { + wayCutter.nextNode( n ); + } } @@ -144,15 +154,17 @@ public class OsmCutter extends MapCreatorBase // add pseudo.tags for concrete:lanes and concrete:plates String concrete = null; - for( String key : map.keySet() ) + for( Map.Entry e : map.entrySet() ) { + String key = e.getKey(); + if ( "concrete".equals( key ) ) { return; } if ( "surface".equals( key ) ) { - String value = map.get( key ); + String value = e.getValue(); if ( value.startsWith( "concrete:" ) ) { concrete = value.substring( "concrete:".length() ); @@ -182,7 +194,7 @@ public class OsmCutter extends MapCreatorBase { String value = w.getTag( key ); _expctxWay.addLookupValue( key, value.replace( ' ', '_' ), lookupData ); - _expctxWayStat.addLookupValue( key, value, null ); + // _expctxWayStat.addLookupValue( key, value, null ); } w.description = _expctxWay.encode(lookupData); @@ -195,7 +207,18 @@ public class OsmCutter extends MapCreatorBase ok |= _expctxWay.getCostfactor() < 10000.; if ( !ok ) return; - w.writeTo( wayDos ); + if ( wayDos != null ) + { + w.writeTo( wayDos ); + } + if ( wayCutter != null ) + { + wayCutter.nextWay( w ); + } + if ( nodeFilter != null ) + { + nodeFilter.nextWay( w ); + } } @Override @@ -300,6 +323,6 @@ public class OsmCutter extends MapCreatorBase int lat = (tileIndex % 6 ) * 30 - 90; String slon = lon < 0 ? "W" + (-lon) : "E" + lon; String slat = lat < 0 ? "S" + (-lat) : "N" + lat; - return slon + "_" + slat + ".tls"; + return slon + "_" + slat + ".ntl"; } } diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/OsmFastCutter.java b/brouter-map-creator/src/main/java/btools/mapcreator/OsmFastCutter.java new file mode 100644 index 0000000..69ca8fa --- /dev/null +++ b/brouter-map-creator/src/main/java/btools/mapcreator/OsmFastCutter.java @@ -0,0 +1,76 @@ +/** + * This program + * - reads an *.osm from stdin + * - writes 45*30 degree node tiles + a way file + a rel file + * + * @author ab + */ +package btools.mapcreator; + +import java.io.File; + +public class OsmFastCutter extends MapCreatorBase +{ + public static void main(String[] args) throws Exception + { + System.out.println("*** OsmFastCutter: cut an osm map in node-tiles + way-tiles"); + if (args.length != 11 && args.length != 12) + { + String common = "java OsmFastCutter "; + + System.out.println("usage: bzip2 -dc | " + common ); + System.out.println("or : " + common + " " ); + return; + } + + doCut( + new File( args[0] ) + , new File( args[1] ) + , new File( args[2] ) + , new File( args[3] ) + , new File( args[4] ) + , new File( args[5] ) + , new File( args[6] ) + , new File( args[7] ) + , new File( args[8] ) + , new File( args[9] ) + , new File( args[10] ) + , args.length > 11 ? new File( args[11] ) : null + ); + } + + public static void doCut (File lookupFile, File nodeDir, File wayDir, File node55Dir, File way55Dir, File borderFile, File relFile, File resFile, File profileAll, File profileReport, File profileCheck, File mapFile ) throws Exception + { + // **** run OsmCutter **** + OsmCutter cutter = new OsmCutter(); + + // ... inject WayCutter + cutter.wayCutter = new WayCutter(); + cutter.wayCutter.init( wayDir ); + + // ... inject NodeFilter + NodeFilter nodeFilter = new NodeFilter(); + nodeFilter.init(); + cutter.nodeFilter = nodeFilter; + + cutter.process( lookupFile, nodeDir, null, relFile, resFile, profileAll, mapFile ); + cutter.wayCutter.finish(); + cutter = null; + + // ***** run WayCutter5 **** + WayCutter5 wayCut5 = new WayCutter5(); + + //... inject RelationMerger + wayCut5.relMerger = new RelationMerger(); + wayCut5.relMerger.init( relFile, lookupFile, profileReport, profileCheck ); + + //... inject NodeFilter + wayCut5.nodeFilter = nodeFilter; + + // ... inject NodeCutter + wayCut5.nodeCutter = new NodeCutter(); + wayCut5.nodeCutter.init( node55Dir ); + + wayCut5.process( nodeDir, wayDir, way55Dir, borderFile ); + } +} diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/PosUnifier.java b/brouter-map-creator/src/main/java/btools/mapcreator/PosUnifier.java index a3c4900..790f080 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/PosUnifier.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/PosUnifier.java @@ -107,6 +107,21 @@ public class PosUnifier extends MapCreatorBase } private void findUniquePos( NodeData n ) + { + int lon = n.ilon; + int lat = n.ilat; + long pid = ( (long) lon ) << 32 | lat; // id from position + if ( !positionSet.contains( pid ) ) + { + positionSet.fastAdd( pid ); + } + else + { + _findUniquePos( n ); + } + } + + private void _findUniquePos( NodeData n ) { // fix the position for uniqueness int lonmod = n.ilon % 1000000; diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java b/brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java index 70a5072..b21f757 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java @@ -40,14 +40,14 @@ public class RelationMerger extends MapCreatorBase new RelationMerger().process( new File( args[0] ), new File( args[1] ), new File( args[2] ), new File( args[3] ), new File( args[4] ), new File( args[5] ) ); } - public void process( File wayFileIn, File wayFileOut, File relationFileIn, File lookupFile, File reportProfile, File checkProfile ) throws Exception + public void init( File relationFileIn, File lookupFile, File reportProfile, File checkProfile ) throws Exception { // read lookup + profile for relation access-check - BExpressionMetaData metaReport = new BExpressionMetaData(); + BExpressionMetaData metaReport = new BExpressionMetaData(); expctxReport = new BExpressionContextWay( metaReport ); metaReport.readMetaData( lookupFile ); - BExpressionMetaData metaCheck = new BExpressionMetaData(); + BExpressionMetaData metaCheck = new BExpressionMetaData(); expctxCheck = new BExpressionContextWay( metaCheck ); metaCheck.readMetaData( lookupFile ); @@ -106,6 +106,11 @@ public class RelationMerger extends MapCreatorBase routesets.put( key, routeset ); System.out.println( "marked " + routeset.size() + " routes for key: " + key ); } + } + + public void process( File wayFileIn, File wayFileOut, File relationFileIn, File lookupFile, File reportProfile, File checkProfile ) throws Exception + { + init( relationFileIn, lookupFile, reportProfile, checkProfile ); // *** finally process the way-file wayOutStream = createOutStream( wayFileOut ); @@ -151,8 +156,10 @@ public class RelationMerger extends MapCreatorBase data.description = expctxReport.encode(); } } - - data.writeTo( wayOutStream ); + if ( wayOutStream != null ) + { + data.writeTo( wayOutStream ); + } } } diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java b/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java index 03c7ec2..8e3acc8 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java @@ -31,14 +31,25 @@ public class WayCutter extends MapCreatorBase public void process( File nodeTilesIn, File wayFileIn, File wayTilesOut ) throws Exception { - this.outTileDir = wayTilesOut; + init( wayTilesOut ); - // *** read all nodes into tileIndexMap - tileIndexMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap() : new TinyDenseLongMap(); new NodeIterator( this, false ).processDir( nodeTilesIn, ".tlf" ); // *** finally process the way-file, cutting into pieces new WayIterator( this, true ).processFile( wayFileIn ); + finish(); + } + + public void init( File wayTilesOut ) throws Exception + { + this.outTileDir = wayTilesOut; + + // *** read all nodes into tileIndexMap + tileIndexMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap() : new TinyDenseLongMap(); + } + + public void finish() throws Exception + { closeTileOutStreams(); } diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter5.java b/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter5.java index ba234d6..21a587c 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter5.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/WayCutter5.java @@ -22,6 +22,10 @@ public class WayCutter5 extends MapCreatorBase private int lonoffset; private int latoffset; + public RelationMerger relMerger; + public NodeFilter nodeFilter; + public NodeCutter nodeCutter; + public static void main(String[] args) throws Exception { System.out.println("*** WayCutter5: Soft-Cut way-data into tiles"); @@ -50,12 +54,17 @@ public class WayCutter5 extends MapCreatorBase { // read corresponding node-file into tileIndexMap String name = wayfile.getName(); - String nodefilename = name.substring( 0, name.length()-3 ) + "tlf"; + String nodefilename = name.substring( 0, name.length()-3 ) + "ntl"; File nodefile = new File( nodeTilesIn, nodefilename ); tileIndexMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap() : new TinyDenseLongMap(); lonoffset = -1; latoffset = -1; + + if ( nodeCutter != null ) + { + nodeCutter.nodeFileStart( null ); + } new NodeIterator( this, false ).processFile( nodefile ); return true; } @@ -63,6 +72,18 @@ public class WayCutter5 extends MapCreatorBase @Override public void nextNode( NodeData n ) throws Exception { + if ( nodeFilter != null ) + { + if ( !nodeFilter.isRelevant( n ) ) + { + return; + } + } + if ( nodeCutter != null ) + { + nodeCutter.nextNode( n ); + } + tileIndexMap.put( n.nid, getTileIndex( n.ilon, n.ilat ) ); } @@ -83,6 +104,11 @@ public class WayCutter5 extends MapCreatorBase } tiForNode[i] = tileIndex; } + + if ( relMerger != null ) + { + relMerger.nextWay( data ); + } // now write way to all tiles hit for( int tileIndex=0; tileIndex<54; tileIndex++ ) @@ -112,6 +138,10 @@ public class WayCutter5 extends MapCreatorBase public void wayFileEnd( File wayFile ) throws Exception { closeTileOutStreams(); + if ( nodeCutter != null ) + { + nodeCutter.nodeFileEnd( null ); + } } private int getTileIndex( int ilon, int ilat ) diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java b/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java index ef40dc3..8e42e0e 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java @@ -64,6 +64,8 @@ public class WayLinker extends MapCreatorBase private int divisor = 32; private int cellsize = 1000000 / divisor; + private boolean skipEncodingCheck; + private void reset() { minLon = -1; @@ -110,6 +112,8 @@ public class WayLinker extends MapCreatorBase abUnifier = new ByteArrayUnifier( 16384, false ); + skipEncodingCheck = Boolean.getBoolean( "skipEncodingCheck" ); + // then process all segments new WayIterator( this, true ).processDir( wayTilesIn, ".wt5" ); } @@ -433,7 +437,11 @@ public class WayLinker extends MapCreatorBase int len = mc.encodeMicroCache( abBuf1 ); subBytes = new byte[len]; System.arraycopy( abBuf1, 0, subBytes, 0, len ); - + + if ( skipEncodingCheck ) + { + break; + } // cross-check the encoding: re-instantiate the cache MicroCache mc2 = new MicroCache2( new StatCoderContext( subBytes ), new DataBuffers( null ), lonIdxDiv, latIdxDiv, divisor, null, null ); // ..and check if still the same diff --git a/brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java b/brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java index a90b659..2ea044f 100644 --- a/brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java +++ b/brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java @@ -15,58 +15,40 @@ public class MapcreatorTest { URL mapurl = this.getClass().getResource( "/dreieich.osm.gz" ); Assert.assertTrue( "test-osm-map dreieich.osm not found", mapurl != null ); - File mapfile = new File(mapurl.getFile()); - File workingDir = mapfile.getParentFile(); + File mapFile = new File(mapurl.getFile()); + File workingDir = mapFile.getParentFile(); File profileDir = new File( workingDir, "/../../../misc/profiles2" ); File tmpdir = new File( workingDir, "tmp" ); tmpdir.mkdir(); - // run OsmCutter - File nodetiles = new File( tmpdir, "nodetiles" ); - nodetiles.mkdir(); - File lookupFile = new File( profileDir, "lookups.dat" ); - File wayFile = new File( tmpdir, "ways.dat" ); - File relFile = new File( tmpdir, "cycleways.dat" ); - File resFile = new File( tmpdir, "restrictions.dat" ); - File profileAllFile = new File( profileDir, "all.brf" ); - new OsmCutter().process( lookupFile, nodetiles, wayFile, relFile, resFile, profileAllFile, mapfile ); - - // run NodeFilter - File ftiles = new File( tmpdir, "ftiles" ); - ftiles.mkdir(); - new NodeFilter().process( nodetiles, wayFile, ftiles ); - - // run RelationMerger - File wayFile2 = new File( tmpdir, "ways2.dat" ); - File profileReport = new File( profileDir, "trekking.brf" ); - File profileCheck = new File( profileDir, "softaccess.brf" ); - new RelationMerger().process( wayFile, wayFile2, relFile, lookupFile, profileReport, profileCheck ); - - // run WayCutter - File waytiles = new File( tmpdir, "waytiles" ); - waytiles.mkdir(); - new WayCutter().process( ftiles, wayFile2, waytiles ); - - // run WayCutter5 - File waytiles55 = new File( tmpdir, "waytiles55" ); - File bordernids = new File( tmpdir, "bordernids.dat" ); - waytiles55.mkdir(); - new WayCutter5().process( ftiles, waytiles, waytiles55, bordernids ); - - // run NodeCutter + File nodes = new File( tmpdir, "nodetiles" ); + nodes.mkdir(); + File ways = new File( tmpdir, "waytiles" ); + ways.mkdir(); File nodes55 = new File( tmpdir, "nodes55" ); nodes55.mkdir(); - new NodeCutter().process( ftiles, nodes55 ); + File ways55 = new File( tmpdir, "waytiles55" ); + ways55.mkdir(); + File lookupFile = new File( profileDir, "lookups.dat" ); + File relFile = new File( tmpdir, "cycleways.dat" ); + File resFile = new File( tmpdir, "restrictions.dat" ); + File profileAll = new File( profileDir, "all.brf" ); + File profileReport = new File( profileDir, "trekking.brf" ); + File profileCheck = new File( profileDir, "softaccess.brf" ); + File borderFile = new File( tmpdir, "bordernids.dat" ); + + new OsmFastCutter().doCut( lookupFile, nodes, ways, nodes55, ways55, borderFile, relFile, resFile, profileAll, profileReport, profileCheck, mapFile ); + // run PosUnifier File unodes55 = new File( tmpdir, "unodes55" ); File bordernodes = new File( tmpdir, "bordernodes.dat" ); unodes55.mkdir(); - new PosUnifier().process( nodes55, unodes55, bordernids, bordernodes, "/private-backup/srtm" ); + new PosUnifier().process( nodes55, unodes55, borderFile, bordernodes, "/private-backup/srtm" ); // run WayLinker File segments = new File( tmpdir, "segments" ); segments.mkdir(); - new WayLinker().process( unodes55, waytiles55, bordernodes, resFile, lookupFile, profileAllFile, segments, "rd5" ); + new WayLinker().process( unodes55, ways55, bordernodes, resFile, lookupFile, profileAll, segments, "rd5" ); } }