variable length tag descriptions (minor lookup versions, bugfixes)

brouter-core/src/main/java/btools/router/RoutingEngine.java

@@ -432,7 +432,7 @@ public class RoutingEngine extends Thread
   private void resetCache()
   {
     nodesMap = new OsmNodesMap();
-    nodesCache = new NodesCache(segmentDir, nodesMap, routingContext.expctxWay.lookupVersion, routingContext.carMode, nodesCache );
+    nodesCache = new NodesCache(segmentDir, nodesMap, routingContext.expctxWay.lookupVersion,routingContext.expctxWay.lookupMinorVersion, routingContext.carMode, nodesCache );
   }
 
   private OsmNode getStartNode( long startId )

brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java

@@ -24,6 +24,7 @@ public final class BExpressionContext
 {
   private static final String CONTEXT_TAG = "---context:";
   private static final String VERSION_TAG = "---lookupversion:";
+  private static final String MINOR_VERSION_TAG = "---minorversion:";
   private static final String VARLENGTH_TAG = "---readvarlength";
 
   private String context;
@@ -78,6 +79,7 @@ public final class BExpressionContext
   private int linenr;
 
   public short lookupVersion = -1;
+  public short lookupMinorVersion = -1;
   public boolean readVarLength = false;
 
   public BExpressionContext( String context )
@@ -104,7 +106,7 @@ public final class BExpressionContext
 
 
   /**
-   * encode lookup data to a byte array
+   * encode internal lookup data to a byte array
    */
   public byte[] encode()
   {
@@ -130,12 +132,15 @@ public final class BExpressionContext
         skippedTags++;
         continue;
       }
-      ctx.encodeDistance( skippedTags );
+      ctx.encodeDistance( skippedTags+1 );
       skippedTags = 0;
-      int n = lookupValues.get(inum).length - 2;
+      
-      if ( n > 1 ) ctx.encode( n, d-1 ); // booleans are encoded just by presence...
+      // 0 excluded already, 1 (=unknown) we rotate up to 8
+      // to have the good code space for the popular values
+      int dd = d < 2 ? 7 : ( d < 9 ? d - 2 : d - 1);
+      ctx.encodeDistance(  dd );
     }
-    ctx.encodeDistance( skippedTags );
+    ctx.encodeDistance( 0 );
     int len = ctx.getEncodedLength();
     byte[] ab = new byte[len];
     System.arraycopy( abBuf, 0, ab, 0, len );
@@ -148,12 +153,20 @@ public final class BExpressionContext
   }
   
 
+  /**
+   * decode byte array to internal lookup data
+   */
+  public void decode( byte[] ab )
+  {
+    decode( lookupData, ab );
+  }
+
   /**
    * decode a byte-array into a lookup data array
    */
   public void decode( int[] ld, byte[] ab )
   {
-	if ( ab.length == 8 ) { decodeFix( ld, ab ); return; }
+	if ( !readVarLength ) { decodeFix( ld, ab ); return; }
 
     BitCoderContext ctx = new BitCoderContext(ab);
 	  
@@ -161,21 +174,22 @@ public final class BExpressionContext
   	ld[0] = ctx.decodeBit() ? 2 : 0;
   	
     // all others are generic
-    for( int inum = 1; inum < lookupValues.size(); inum++ ) // loop over lookup names
+  	int inum = 1;
+    for(;;)
     {
-      int skip = ctx.decodeDistance();
+      int delta = ctx.decodeDistance();
-      while ( skip-- > 0 ) ld[inum++] = 0;
+      if ( delta == 0) break;
-      if ( inum >= lookupValues.size() ) break;
+      if ( inum + delta > ld.length ) break; // higher minor version is o.k.
-      int n = lookupValues.get(inum).length - 2;
+      
-      if ( n > 1 )
+      while ( delta-- > 1 ) ld[inum++] = 0;
-      {
+
-        ld[inum] = ctx.decode( n ) + 1;
+      // see encoder for value rotation
-      }
+      int dd = ctx.decodeDistance();
-      else
+      int d = dd == 7 ? 1 : ( dd < 7 ? dd + 2 : dd + 1);
-      {
+      if ( d >= lookupValues.get(inum).length ) d = 1; // map out-of-range to unkown
-          ld[inum] = 2; // boolean
+      ld[inum++] = d;
-      }
     }
+    while( inum < ld.length ) ld[inum++] = 0;
   }
 
   /**
@@ -243,7 +257,10 @@ public final class BExpressionContext
      for( int inum = 0; inum < lookupValues.size(); inum++ ) // loop over lookup names
      {
        BExpressionLookupValue[] va = lookupValues.get(inum);
-       String value = va[lookupData[inum]].toString();
+       int dataIdx = lookupData[inum];
+       if ( dataIdx >= va.length )
+          throw new RuntimeException( "ups, inum=" + inum + " dataIdx=" + dataIdx + " va.length=" + va.length + " sb=" + sb );
+       String value = va[dataIdx].toString();
        if ( value != null && value.length() > 0 )
        {
          sb.append( " " + lookupNames.get( inum ) + "=" + value );
@@ -277,6 +294,11 @@ public final class BExpressionContext
         lookupVersion = Short.parseShort( line.substring( VERSION_TAG.length() ) );
         continue;
       }
+      if ( line.startsWith( MINOR_VERSION_TAG ) )
+      {
+        lookupMinorVersion = Short.parseShort( line.substring( MINOR_VERSION_TAG.length() ) );
+        continue;
+      }
       if ( line.startsWith( VARLENGTH_TAG ) )
       {
     	readVarLength = true;

brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java

@@ -58,11 +58,11 @@ public class OsmCutter extends MapCreatorBase
 
     _expctxWay = new BExpressionContext("way");
     _expctxWay.readMetaData( lookupFile );
-    _expctxWayStat = new BExpressionContext("way");
+//  _expctxWayStat = new BExpressionContext("way");
 
     _expctxNode = new BExpressionContext("node");
     _expctxNode.readMetaData( lookupFile );
-    _expctxNodeStat = new BExpressionContext("node");
+//  _expctxNodeStat = new BExpressionContext("node");
 
     this.outTileDir = outTileDir;
     if ( !outTileDir.isDirectory() ) throw new RuntimeException( "out tile directory " + outTileDir + " does not exist" );
@@ -82,10 +82,10 @@ public class OsmCutter extends MapCreatorBase
     wayDos.close();
     cyclewayDos.close();
 
-    System.out.println( "-------- way-statistics -------- " );
+//    System.out.println( "-------- way-statistics -------- " );
-    _expctxWayStat.dumpStatistics();
+//    _expctxWayStat.dumpStatistics();
-    System.out.println( "-------- node-statistics -------- " );
+//    System.out.println( "-------- node-statistics -------- " );
-    _expctxNodeStat.dumpStatistics();
+//    _expctxNodeStat.dumpStatistics();
 
     System.out.println( statsLine() );
   }
@@ -114,7 +114,7 @@ public class OsmCutter extends MapCreatorBase
       {
         String value = n.getTag( key );
         _expctxNode.addLookupValue( key, value, lookupData );
-        _expctxNodeStat.addLookupValue( key, value, null );
+//        _expctxNodeStat.addLookupValue( key, value, null );
       }
       n.description = _expctxNode.encode(lookupData);
     }
@@ -161,7 +161,7 @@ public class OsmCutter extends MapCreatorBase
       {
         String value = w.getTag( key );
         _expctxWay.addLookupValue( key, value, lookupData );
-        _expctxWayStat.addLookupValue( key, value, null );
+//        _expctxWayStat.addLookupValue( key, value, null );
       }
       w.description = _expctxWay.encode(lookupData);
     }

brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java

@@ -23,7 +23,7 @@ public class RelationMerger extends MapCreatorBase
   private CompactLongSet routesetall;
   private BExpressionContext expctxReport;
   private BExpressionContext expctxCheck;
-  private BExpressionContext expctxStat;
+ // private BExpressionContext expctxStat;
 
   private DataOutputStream wayOutStream;
   
@@ -48,7 +48,7 @@ public class RelationMerger extends MapCreatorBase
     expctxCheck = new BExpressionContext("way");
     expctxCheck.readMetaData( lookupFile );
     expctxCheck.parseFile( checkProfile, "global" );
-    expctxStat = new BExpressionContext("way");
+    // expctxStat = new BExpressionContext("way");
     
     // *** read the relation file into sets for each processed tag
     routesets = new HashMap<String,CompactLongSet>();
@@ -78,7 +78,7 @@ public class RelationMerger extends MapCreatorBase
         {
           long wid = readId( dis );
           if ( wid == -1 ) break;
-    	  expctxStat.addLookupValue( tagname, "yes", null );
+    	  // expctxStat.addLookupValue( tagname, "yes", null );
           if ( routeset != null && !routeset.contains( wid ) )
           {
         	  routeset.add( wid );
@@ -103,8 +103,8 @@ public class RelationMerger extends MapCreatorBase
     new WayIterator( this, true ).processFile( wayFileIn );
     wayOutStream.close();
 
-    System.out.println( "-------- route-statistics -------- " );
+//    System.out.println( "-------- route-statistics -------- " );
-    expctxStat.dumpStatistics();
+//    expctxStat.dumpStatistics();
 }
 
   @Override

brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java

@@ -39,6 +39,7 @@ public class WayLinker extends MapCreatorBase
   private List<OsmNodeP> nodesList;
   private CompactLongSet borderSet;
   private short lookupVersion;
+  private short lookupMinorVersion;
 
   private long creationTimeStamp;
   
@@ -59,7 +60,7 @@ public class WayLinker extends MapCreatorBase
   
   public static void main(String[] args) throws Exception
   {
-    System.out.println("*** WayLinker: Format a regionof an OSM map for routing");
+    System.out.println("*** WayLinker: Format a region of an OSM map for routing");
     if (args.length != 7)
     {
       System.out.println("usage: java WayLinker <node-tiles-in> <way-tiles-in> <bordernodes> <lookup-file> <profile-file> <data-tiles-out> <data-tiles-suffix> ");
@@ -79,6 +80,7 @@ public class WayLinker extends MapCreatorBase
     expctxWay = new BExpressionContext("way");
     expctxWay.readMetaData( lookupFile );
     lookupVersion = expctxWay.lookupVersion;
+    lookupMinorVersion = expctxWay.lookupMinorVersion;
     expctxWay.parseFile( profileFile, "global" );
     
     creationTimeStamp = System.currentTimeMillis();
@@ -152,13 +154,13 @@ public class WayLinker extends MapCreatorBase
     boolean ok = expctxWay.getCostfactor() < 10000.; 
     expctxWay.evaluate( true, description, null );
     ok |= expctxWay.getCostfactor() < 10000.;
+    if ( !ok ) return;
 
     byte bridgeTunnel = 0;
+    expctxWay.decode( description );
     if ( expctxWay.getBooleanLookupValue( "bridge" ) ) bridgeTunnel |= OsmNodeP.BRIDGE_AND_BIT;
     if ( expctxWay.getBooleanLookupValue( "tunnel" ) ) bridgeTunnel |= OsmNodeP.TUNNEL_AND_BIT;
     
-    if ( !ok ) return;
-    
     OsmNodeP n1 = null;
     OsmNodeP n2 = null;
     for (int i=0; i<way.nodes.size(); i++)
@@ -182,7 +184,6 @@ public class WayLinker extends MapCreatorBase
       if ( n2 != null )
       {
         n2.wayAndBits &= bridgeTunnel;
-//        if ( n2 instanceof OsmNodePT ) ((OsmNodePT)n2).wayOrBits |= lowbyte;
       }
     }
   }
@@ -298,7 +299,7 @@ public class WayLinker extends MapCreatorBase
         }
       }
       
-      byte[] abFileIndex = compileFileIndex( fileIndex, lookupVersion );
+      byte[] abFileIndex = compileFileIndex( fileIndex, lookupVersion, lookupMinorVersion );
       
       // write extra data: timestamp + index-checksums
       os.writeLong( creationTimeStamp );
@@ -317,14 +318,14 @@ public class WayLinker extends MapCreatorBase
     }
   }
   
-  private byte[] compileFileIndex( long[] fileIndex, short lookupVersion ) throws Exception
+  private byte[] compileFileIndex( long[] fileIndex, short lookupVersion, short lookupMinorVersion ) throws Exception
   {
     ByteArrayOutputStream bos = new ByteArrayOutputStream( );
     DataOutputStream dos = new DataOutputStream( bos );
-    long versionPrefix = lookupVersion;
-    versionPrefix <<= 48;
     for( int i55=0; i55<25; i55++)
     {
+      long versionPrefix = i55 == 1 ? lookupMinorVersion : lookupVersion;
+      versionPrefix <<= 48;
       dos.writeLong( fileIndex[i55] | versionPrefix );
     }
     dos.close();

brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java

@@ -16,6 +16,7 @@ public final class NodesCache
   private String segmentDir;
   private OsmNodesMap nodesMap;
   private int lookupVersion;
+  private int lookupMinorVersion;
   private boolean carMode;
   private String currentFileName;
 
@@ -32,11 +33,12 @@ public final class NodesCache
   private long cacheSum = 0;
   private boolean garbageCollectionEnabled = false;
 
-  public NodesCache( String segmentDir, OsmNodesMap nodesMap, int lookupVersion, boolean carMode, NodesCache oldCache )
+  public NodesCache( String segmentDir, OsmNodesMap nodesMap, int lookupVersion, int lookupMinorVersion, boolean carMode, NodesCache oldCache )
   {
     this.segmentDir = segmentDir;
     this.nodesMap = nodesMap;
     this.lookupVersion = lookupVersion;
+    this.lookupMinorVersion = lookupMinorVersion;
     this.carMode = carMode;
 
     if ( oldCache != null )
@@ -215,7 +217,7 @@ public final class NodesCache
       if ( f != null )
       {
         currentFileName = f.getName();
-        ra = new PhysicalFile( f, iobuffer, lookupVersion );
+        ra = new PhysicalFile( f, iobuffer, lookupVersion, lookupMinorVersion );
       }
       fileCache.put( filenameBase, ra );
     }

brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java

@@ -17,6 +17,8 @@ public class OsmNode implements OsmPos
   public static final int WRITEDESC_BITMASK       = 0x10;
   public static final int SKIPDETAILS_BITMASK     = 0x08;
   public static final int NODEDESC_BITMASK        = 0x04;
+  public static final int RESERVED1_BITMASK       = 0x02;
+  public static final int RESERVED2_BITMASK       = 0x01;
 
   public OsmNode()
   {
@@ -159,6 +161,16 @@ public class OsmNode implements OsmPos
         	 is.readFully( ab );
         	 nodeDescription = abUnifier.unify( ab );
          }
+         if ( (bitField & RESERVED1_BITMASK ) != 0 )
+         {
+        	 byte[] ab = new byte[is.readByte()];
+        	 is.readFully( ab );
+         }
+         if ( (bitField & RESERVED2_BITMASK ) != 0 )
+         {
+        	 byte[] ab = new byte[is.readByte()];
+        	 is.readFully( ab );
+         }
          if ( (bitField & SKIPDETAILS_BITMASK ) != 0 )
          {
            link.counterLinkWritten = true;

brouter-mapaccess/src/main/java/btools/mapaccess/PhysicalFile.java

@@ -30,7 +30,7 @@ final public class PhysicalFile
 	  try
 	  {
         byte[] iobuffer = new byte[65636];
-        pf = new PhysicalFile( f, new byte[65636], -1 );
+        pf = new PhysicalFile( f, new byte[65636], -1, -1 );
       	for( int tileIndex=0; tileIndex<25; tileIndex++ )
       	{
 		  OsmFile osmf = new OsmFile( pf, tileIndex, iobuffer );
@@ -55,7 +55,7 @@ final public class PhysicalFile
 	  return null;
   }
 
-  public PhysicalFile( File f, byte[] iobuffer, int lookupVersion ) throws Exception
+  public PhysicalFile( File f, byte[] iobuffer, int lookupVersion, int lookupMinorVersion ) throws Exception
   {
     fileName = f.getName();
 
@@ -67,11 +67,16 @@ final public class PhysicalFile
     {
       long lv = dis.readLong();
       short readVersion = (short)(lv >> 48);
-      if ( readVersion != lookupVersion && lookupVersion != -1 )
+      if ( i == 0 && lookupVersion != -1 && readVersion != lookupVersion )
       {
         throw new IllegalArgumentException( "lookup version mismatch (old rd5?) lookups.dat="
                  + lookupVersion + " " + f. getAbsolutePath() + "=" + readVersion );
       }
+      if ( i == 1 && lookupMinorVersion != -1 && readVersion < lookupMinorVersion )
+      {
+        throw new IllegalArgumentException( "lookup minor version mismatch (old rd5?) lookups.dat="
+                 + lookupMinorVersion + " " + f. getAbsolutePath() + "=" + readVersion );
+      }
       fileIndex[i] = lv & 0xffffffffffffL;
     }
 

brouter-server/src/main/java/btools/server/BRouter.java

@@ -84,7 +84,7 @@ public class BRouter
       }
       System.exit(0);
     }
-    System.out.println("BRouter 0.9.9 / 18042014 / abrensch");
+    System.out.println("BRouter 1.0 / 01062014 / abrensch");
     if ( args.length < 6 )
     {
       System.out.println("Find routes in an OSM map");