preprocessor speedup using more memory

brouter-map-creator/src/main/java/btools/mapcreator/NodeCutter.java

@@ -25,9 +25,14 @@ public class NodeCutter extends MapCreatorBase
     new NodeCutter().process( new File( args[0] ), new File( args[1] ) );
   }
 
-  public void process( File nodeTilesIn, File nodeTilesOut ) throws Exception
+  public void init( File nodeTilesOut )
   {
     this.outTileDir = nodeTilesOut;
+  }
+
+  public void process( File nodeTilesIn, File nodeTilesOut ) throws Exception
+  {
+    init( nodeTilesOut );
 
     new NodeIterator( this, true ).processDir( nodeTilesIn, ".tlf" );
   }

brouter-map-creator/src/main/java/btools/mapcreator/NodeFilter.java

@@ -33,12 +33,18 @@ public class NodeFilter extends MapCreatorBase
     new NodeFilter().process( new File( args[0] ), new File( args[1] ), new File( args[2] ) );
   }
 
+  public void init() throws Exception
+  {
+    nodebitmap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap( 512 ) : new TinyDenseLongMap();
+  }
+
   public void process( File nodeTilesIn, File wayFileIn, File nodeTilesOut ) throws Exception
   {
+    init();
+  
     this.nodeTilesOut = nodeTilesOut;
 
     // read the wayfile into a bitmap of used nodes
-    nodebitmap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap( 512 ) : new TinyDenseLongMap();
     new WayIterator( this, false ).processFile( wayFileIn );
 
     // finally filter all node files
@@ -59,7 +65,6 @@ public class NodeFilter extends MapCreatorBase
   public void nodeFileStart( File nodefile ) throws Exception
   {
     String filename = nodefile.getName();
-    filename = filename.substring( 0, filename.length() - 3 ) + "tlf"; 
     File outfile = new File( nodeTilesOut, filename );
     nodesOutStream = new DiffCoderDataOutputStream( new BufferedOutputStream ( new FileOutputStream( outfile ) ) );
   }
@@ -67,13 +72,18 @@ public class NodeFilter extends MapCreatorBase
   @Override
   public void nextNode( NodeData n ) throws Exception
   {
-    // check if node passes bitmap
+    if ( isRelevant( n ) )
-    if ( nodebitmap.getInt( n.nid ) == 0 ) // 0 -> bit set, -1 -> unset
     {
       n.writeTo( nodesOutStream );
     }
   }
 
+  public boolean isRelevant( NodeData n )
+  {
+    // check if node passes bitmap
+    return nodebitmap.getInt( n.nid ) == 0; // 0 -> bit set, -1 -> unset
+  }
+
   @Override
   public void nodeFileEnd( File nodeFile ) throws Exception
   {

brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java

@@ -12,6 +12,7 @@ import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.util.HashMap;
+import java.util.Map;
 
 import btools.expressions.BExpressionContextNode;
 import btools.expressions.BExpressionContextWay;
@@ -29,6 +30,9 @@ public class OsmCutter extends MapCreatorBase
   private DataOutputStream cyclewayDos;
   private DataOutputStream restrictionsDos;
 
+  public WayCutter wayCutter;
+  public NodeFilter nodeFilter;
+  
   public static void main(String[] args) throws Exception
   {
     System.out.println("*** OsmCutter: cut an osm map in node-tiles + a way file");
@@ -53,8 +57,8 @@ public class OsmCutter extends MapCreatorBase
   private BExpressionContextWay _expctxWay;
   private BExpressionContextNode _expctxNode;
 
-  private BExpressionContextWay _expctxWayStat;
+  // private BExpressionContextWay _expctxWayStat;
-  private BExpressionContextNode _expctxNodeStat;
+  // private BExpressionContextNode _expctxNodeStat;
 
   public void process (File lookupFile, File outTileDir, File wayFile, File relFile, File resFile, File profileFile, File mapFile ) throws Exception
   {
@@ -71,13 +75,13 @@ public class OsmCutter extends MapCreatorBase
     _expctxWay.parseFile( profileFile, "global" );
 
     
-   _expctxWayStat = new BExpressionContextWay( null );
+   // _expctxWayStat = new BExpressionContextWay( null );
-   _expctxNodeStat = new BExpressionContextNode( null );
+   // _expctxNodeStat = new BExpressionContextNode( null );
 
     this.outTileDir = outTileDir;
     if ( !outTileDir.isDirectory() ) throw new RuntimeException( "out tile directory " + outTileDir + " does not exist" );
 
-    wayDos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( wayFile ) ) );
+    wayDos = wayFile == null ? null : new DataOutputStream( new BufferedOutputStream( new FileOutputStream( wayFile ) ) );
     cyclewayDos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( relFile ) ) );
     restrictionsDos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( resFile ) ) );
 
@@ -90,7 +94,10 @@ public class OsmCutter extends MapCreatorBase
 
     // close all files
     closeTileOutStreams();
-    wayDos.close();
+    if ( wayDos != null )
+    {
+      wayDos.close();
+    }
     cyclewayDos.close();
     restrictionsDos.close();
 
@@ -122,11 +129,10 @@ public class OsmCutter extends MapCreatorBase
     if ( n.getTagsOrNull() != null )
     {
       int[] lookupData = _expctxNode.createNewLookupData();
-      for( String key : n.getTagsOrNull().keySet() )
+      for( Map.Entry<String,String> e : n.getTagsOrNull().entrySet() )
       {
-        String value = n.getTag( key );
+        _expctxNode.addLookupValue( e.getKey(), e.getValue(), lookupData );
-        _expctxNode.addLookupValue( key, value, lookupData );
+        // _expctxNodeStat.addLookupValue( key, value, null );
-        _expctxNodeStat.addLookupValue( key, value, null );
       }
       n.description = _expctxNode.encode(lookupData);
     }
@@ -136,6 +142,10 @@ public class OsmCutter extends MapCreatorBase
     {
       n.writeTo( getOutStreamForTile( tileIndex ) );
     }
+    if ( wayCutter != null )
+    {
+      wayCutter.nextNode( n );
+    }
   }
 
 
@@ -144,15 +154,17 @@ public class OsmCutter extends MapCreatorBase
     // add pseudo.tags for concrete:lanes and concrete:plates
   
     String concrete = null;
-    for( String key : map.keySet() )
+    for( Map.Entry<String,String> e : map.entrySet() )
     {
+      String key = e.getKey();
+    
       if ( "concrete".equals( key ) )
       {
         return;
       }
       if ( "surface".equals( key ) )
       {
-        String value = map.get( key );
+        String value = e.getValue();
         if ( value.startsWith( "concrete:" ) )
         {
           concrete = value.substring( "concrete:".length() );
@@ -182,7 +194,7 @@ public class OsmCutter extends MapCreatorBase
     {
       String value = w.getTag( key );
       _expctxWay.addLookupValue( key, value.replace( ' ', '_' ), lookupData );
-      _expctxWayStat.addLookupValue( key, value, null );
+      // _expctxWayStat.addLookupValue( key, value, null );
     }
     w.description = _expctxWay.encode(lookupData);
     
@@ -195,7 +207,18 @@ public class OsmCutter extends MapCreatorBase
     ok |= _expctxWay.getCostfactor() < 10000.;
     if ( !ok ) return;
     
-    w.writeTo( wayDos );
+    if ( wayDos != null )
+    {
+      w.writeTo( wayDos );
+    }
+    if ( wayCutter != null )
+    {
+      wayCutter.nextWay( w );
+    }
+    if ( nodeFilter != null )
+    {
+      nodeFilter.nextWay( w );
+    }
   }
 
   @Override
@@ -300,6 +323,6 @@ public class OsmCutter extends MapCreatorBase
     int lat = (tileIndex % 6 ) * 30 - 90;
     String slon = lon < 0 ? "W" + (-lon) : "E" + lon;
     String slat = lat < 0 ? "S" + (-lat) : "N" + lat;
-    return slon + "_" + slat + ".tls";
+    return slon + "_" + slat + ".ntl";
   }
 }

brouter-map-creator/src/main/java/btools/mapcreator/OsmFastCutter.java

@@ -0,0 +1,76 @@
+/**
+ * This program
+ * - reads an *.osm from stdin
+ * - writes 45*30 degree node tiles + a way file + a rel file
+ *
+ * @author ab
+ */
+package btools.mapcreator;
+
+import java.io.File;
+
+public class OsmFastCutter extends MapCreatorBase
+{
+  public static void main(String[] args) throws Exception
+  {
+    System.out.println("*** OsmFastCutter: cut an osm map in node-tiles + way-tiles");
+    if (args.length != 11 && args.length != 12)
+    {
+      String common = "java OsmFastCutter <lookup-file> <node-dir> <way-dir> <node55-dir> <way55-dir> <border-file> <out-rel-file> <out-res-file> <filter-profile> <report-profile> <check-profile>";
+    
+      System.out.println("usage: bzip2 -dc <map> | " + common );
+      System.out.println("or   : " + common + " <inputfile> " );
+      return;
+    }
+
+    doCut(
+                   new File( args[0] )
+                 , new File( args[1] )
+                 , new File( args[2] )
+                 , new File( args[3] )
+                 , new File( args[4] )
+                 , new File( args[5] )
+                 , new File( args[6] )
+                 , new File( args[7] )
+                 , new File( args[8] )
+                 , new File( args[9] )
+                 , new File( args[10] )
+                 , args.length > 11 ? new File( args[11] ) : null
+                		 );
+  }
+
+  public static void doCut (File lookupFile, File nodeDir, File wayDir, File node55Dir, File way55Dir, File borderFile, File relFile, File resFile, File profileAll, File profileReport, File profileCheck, File mapFile ) throws Exception
+  {
+    // **** run OsmCutter ****
+    OsmCutter cutter = new OsmCutter();
+
+    // ... inject WayCutter
+    cutter.wayCutter = new WayCutter();
+    cutter.wayCutter.init( wayDir );
+
+    // ... inject NodeFilter
+    NodeFilter nodeFilter = new NodeFilter();
+    nodeFilter.init();
+    cutter.nodeFilter = nodeFilter;
+
+    cutter.process( lookupFile, nodeDir, null, relFile, resFile, profileAll, mapFile );
+    cutter.wayCutter.finish();
+    cutter = null;
+
+    // ***** run WayCutter5 ****
+    WayCutter5 wayCut5 = new WayCutter5();
+
+    //... inject RelationMerger
+    wayCut5.relMerger = new RelationMerger();
+    wayCut5.relMerger.init( relFile, lookupFile, profileReport, profileCheck );
+
+    //... inject NodeFilter
+    wayCut5.nodeFilter = nodeFilter;
+
+    // ... inject NodeCutter
+    wayCut5.nodeCutter = new NodeCutter();
+    wayCut5.nodeCutter.init( node55Dir );
+
+    wayCut5.process( nodeDir, wayDir, way55Dir, borderFile );
+  }
+}

brouter-map-creator/src/main/java/btools/mapcreator/PosUnifier.java

@@ -107,6 +107,21 @@ public class PosUnifier extends MapCreatorBase
   }
 
   private void findUniquePos( NodeData n )
+  {
+    int lon = n.ilon;
+    int lat = n.ilat;
+    long pid = ( (long) lon ) << 32 | lat; // id from position
+    if ( !positionSet.contains( pid ) )
+    {
+      positionSet.fastAdd( pid );
+    }
+    else
+    {
+      _findUniquePos( n );
+    }
+  }
+
+  private void _findUniquePos( NodeData n )
   {
     // fix the position for uniqueness
     int lonmod = n.ilon % 1000000;

brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java

@@ -40,14 +40,14 @@ public class RelationMerger extends MapCreatorBase
     new RelationMerger().process( new File( args[0] ), new File( args[1] ), new File( args[2] ), new File( args[3] ), new File( args[4] ), new File( args[5] ) );
   }
 
-  public void process( File wayFileIn, File wayFileOut, File relationFileIn, File lookupFile, File reportProfile, File checkProfile ) throws Exception
+  public void init( File relationFileIn, File lookupFile, File reportProfile, File checkProfile ) throws Exception
   {
     // read lookup + profile for relation access-check
-	BExpressionMetaData metaReport = new BExpressionMetaData();
+  BExpressionMetaData metaReport = new BExpressionMetaData();
     expctxReport = new BExpressionContextWay( metaReport );
     metaReport.readMetaData( lookupFile );
 
-	BExpressionMetaData metaCheck = new BExpressionMetaData();
+  BExpressionMetaData metaCheck = new BExpressionMetaData();
     expctxCheck = new BExpressionContextWay( metaCheck );
     metaCheck.readMetaData( lookupFile );
 
@@ -106,6 +106,11 @@ public class RelationMerger extends MapCreatorBase
       routesets.put( key, routeset );
       System.out.println( "marked " + routeset.size() + " routes for key: " + key );
     }
+  }
+
+  public void process( File wayFileIn, File wayFileOut, File relationFileIn, File lookupFile, File reportProfile, File checkProfile ) throws Exception
+  {
+    init( relationFileIn, lookupFile, reportProfile, checkProfile );
 
     // *** finally process the way-file
     wayOutStream = createOutStream( wayFileOut );
@@ -151,8 +156,10 @@ public class RelationMerger extends MapCreatorBase
     	data.description = expctxReport.encode();
       }
     }
-
+    if ( wayOutStream != null )
-    data.writeTo( wayOutStream );
+    {
+      data.writeTo( wayOutStream );
+    }
   }
 
 }

brouter-map-creator/src/main/java/btools/mapcreator/WayCutter.java

@@ -31,14 +31,25 @@ public class WayCutter extends MapCreatorBase
 
   public void process( File nodeTilesIn, File wayFileIn, File wayTilesOut ) throws Exception
   {
-    this.outTileDir = wayTilesOut;
+    init( wayTilesOut );
 
-    // *** read all nodes into tileIndexMap
-    tileIndexMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap() : new TinyDenseLongMap();
     new NodeIterator( this, false ).processDir( nodeTilesIn, ".tlf" );
 
     // *** finally process the way-file, cutting into pieces
     new WayIterator( this, true ).processFile( wayFileIn );
+    finish();
+  }
+
+  public void init( File wayTilesOut ) throws Exception
+  {
+    this.outTileDir = wayTilesOut;
+
+    // *** read all nodes into tileIndexMap
+    tileIndexMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap() : new TinyDenseLongMap();
+  }
+
+  public void finish() throws Exception
+  {
     closeTileOutStreams();
   }
 

brouter-map-creator/src/main/java/btools/mapcreator/WayCutter5.java

@@ -22,6 +22,10 @@ public class WayCutter5 extends MapCreatorBase
   private int lonoffset;
   private int latoffset;
 
+  public RelationMerger relMerger;
+  public NodeFilter nodeFilter;
+  public NodeCutter nodeCutter;
+
   public static void main(String[] args) throws Exception
   {
     System.out.println("*** WayCutter5: Soft-Cut way-data into tiles");
@@ -50,12 +54,17 @@ public class WayCutter5 extends MapCreatorBase
   {
     // read corresponding node-file into tileIndexMap
     String name = wayfile.getName();
-    String nodefilename = name.substring( 0, name.length()-3 ) + "tlf";
+    String nodefilename = name.substring( 0, name.length()-3 ) + "ntl";
     File nodefile = new File( nodeTilesIn, nodefilename );
 
     tileIndexMap = Boolean.getBoolean( "useDenseMaps" ) ? new DenseLongMap() : new TinyDenseLongMap();
     lonoffset = -1;
     latoffset = -1;
+    
+    if ( nodeCutter != null )
+    {
+      nodeCutter.nodeFileStart( null );
+    }
     new NodeIterator( this, false ).processFile( nodefile );
     return true;
   }
@@ -63,6 +72,18 @@ public class WayCutter5 extends MapCreatorBase
   @Override
   public void nextNode( NodeData n ) throws Exception
   {
+    if ( nodeFilter != null )
+    {
+      if ( !nodeFilter.isRelevant( n ) )
+      {
+        return;
+      }
+    }
+    if ( nodeCutter != null )
+    {
+      nodeCutter.nextNode( n );
+    }
+  
     tileIndexMap.put( n.nid, getTileIndex( n.ilon, n.ilat ) );
   }
 
@@ -84,6 +105,11 @@ public class WayCutter5 extends MapCreatorBase
       tiForNode[i] = tileIndex;
     }
     
+    if ( relMerger != null )
+    {
+      relMerger.nextWay( data );
+    }
+
     // now write way to all tiles hit
     for( int tileIndex=0; tileIndex<54; tileIndex++ )
     {
@@ -112,6 +138,10 @@ public class WayCutter5 extends MapCreatorBase
   public void wayFileEnd( File wayFile ) throws Exception
   {
     closeTileOutStreams();
+    if ( nodeCutter != null )
+    {
+      nodeCutter.nodeFileEnd( null );
+    }
   }
 
   private int getTileIndex( int ilon, int ilat )

brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java

@@ -64,6 +64,8 @@ public class WayLinker extends MapCreatorBase
   private int divisor = 32;
   private int cellsize = 1000000 / divisor;
 
+  private boolean skipEncodingCheck;
+
   private void reset()
   {
     minLon = -1;
@@ -110,6 +112,8 @@ public class WayLinker extends MapCreatorBase
 
     abUnifier = new ByteArrayUnifier( 16384, false );
 
+    skipEncodingCheck = Boolean.getBoolean( "skipEncodingCheck" );
+
     // then process all segments
     new WayIterator( this, true ).processDir( wayTilesIn, ".wt5" );
   }
@@ -434,6 +438,10 @@ public class WayLinker extends MapCreatorBase
                     subBytes = new byte[len];
                     System.arraycopy( abBuf1, 0, subBytes, 0, len );
                     
+                    if ( skipEncodingCheck )
+                    {
+                      break;
+                    }
                     // cross-check the encoding: re-instantiate the cache
                     MicroCache mc2 = new MicroCache2( new StatCoderContext( subBytes ), new DataBuffers( null ), lonIdxDiv, latIdxDiv, divisor, null, null );
                     // ..and check if still the same

brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java

@@ -15,58 +15,40 @@ public class MapcreatorTest
   {
     URL mapurl = this.getClass().getResource( "/dreieich.osm.gz" );
     Assert.assertTrue( "test-osm-map dreieich.osm not found", mapurl != null );
-    File mapfile = new File(mapurl.getFile());
+    File mapFile = new File(mapurl.getFile());
-    File workingDir = mapfile.getParentFile();
+    File workingDir = mapFile.getParentFile();
     File profileDir = new File( workingDir, "/../../../misc/profiles2" );
     File tmpdir = new File( workingDir, "tmp" );
     tmpdir.mkdir();
 
-    // run OsmCutter
+    File nodes = new File( tmpdir, "nodetiles" );
-    File nodetiles = new File( tmpdir, "nodetiles" );
+    nodes.mkdir();
-    nodetiles.mkdir();
+    File ways = new File( tmpdir, "waytiles" );
-    File lookupFile = new File( profileDir, "lookups.dat" );
+    ways.mkdir();
-    File wayFile = new File( tmpdir, "ways.dat" );
-    File relFile = new File( tmpdir, "cycleways.dat" );
-    File resFile = new File( tmpdir, "restrictions.dat" );
-    File profileAllFile = new File( profileDir, "all.brf" );
-    new OsmCutter().process( lookupFile, nodetiles, wayFile, relFile, resFile, profileAllFile, mapfile );
-
-    // run NodeFilter
-    File ftiles = new File( tmpdir, "ftiles" );
-    ftiles.mkdir();
-    new NodeFilter().process( nodetiles, wayFile, ftiles );
-
-    // run RelationMerger
-    File wayFile2 = new File( tmpdir, "ways2.dat" );
-    File profileReport = new File( profileDir, "trekking.brf" );
-    File profileCheck = new File( profileDir, "softaccess.brf" );
-    new RelationMerger().process( wayFile, wayFile2, relFile, lookupFile, profileReport, profileCheck );
-
-    // run WayCutter
-    File waytiles = new File( tmpdir, "waytiles" );
-    waytiles.mkdir();
-    new WayCutter().process( ftiles, wayFile2, waytiles );
-
-    // run WayCutter5
-    File waytiles55 = new File( tmpdir, "waytiles55" );
-    File bordernids = new File( tmpdir, "bordernids.dat" );
-    waytiles55.mkdir();
-    new WayCutter5().process( ftiles, waytiles, waytiles55, bordernids );
-
-    // run NodeCutter
     File nodes55 = new File( tmpdir, "nodes55" );
     nodes55.mkdir();
-    new NodeCutter().process( ftiles, nodes55 );
+    File ways55 = new File( tmpdir, "waytiles55" );
+    ways55.mkdir();
+    File lookupFile = new File( profileDir, "lookups.dat" );
+    File relFile = new File( tmpdir, "cycleways.dat" );
+    File resFile = new File( tmpdir, "restrictions.dat" );
+    File profileAll = new File( profileDir, "all.brf" );
+    File profileReport = new File( profileDir, "trekking.brf" );
+    File profileCheck = new File( profileDir, "softaccess.brf" );
+    File borderFile = new File( tmpdir, "bordernids.dat" );
+
+    new OsmFastCutter().doCut( lookupFile, nodes, ways, nodes55, ways55, borderFile, relFile, resFile, profileAll, profileReport, profileCheck, mapFile );
+
 
     // run PosUnifier
     File unodes55 = new File( tmpdir, "unodes55" );
     File bordernodes = new File( tmpdir, "bordernodes.dat" );
     unodes55.mkdir();
-    new PosUnifier().process( nodes55, unodes55, bordernids, bordernodes, "/private-backup/srtm" );
+    new PosUnifier().process( nodes55, unodes55, borderFile, bordernodes, "/private-backup/srtm" );
 
     // run WayLinker
     File segments = new File( tmpdir, "segments" );
     segments.mkdir();
-    new WayLinker().process( unodes55, waytiles55, bordernodes, resFile, lookupFile, profileAllFile, segments, "rd5" );
+    new WayLinker().process( unodes55, ways55, bordernodes, resFile, lookupFile, profileAll, segments, "rd5" );
   }
 }