some microtuning

brouter-codec/src/main/java/btools/codec/MicroCache2.java

@@ -502,7 +502,7 @@ public final class MicroCache2 extends MicroCache
       }
       if ( pass == 3 )
       {
-        return bc.getEncodedLength();
+        return bc.closeAndGetEncodedLength();
       }
     }
   }

brouter-codec/src/main/java/btools/codec/TagValueCoder.java

@@ -137,7 +137,7 @@ public final class TagValueCoder
     }
 
     byte[] res;
-    int len = ctx.getEncodedLength();
+    int len = ctx.closeAndGetEncodedLength();
     if ( validator == null )
     {
       res = new byte[len];

brouter-codec/src/test/java/btools/codec/StatCoderContextTest.java

@@ -20,6 +20,7 @@ public class StatCoderContextTest
         ctx.encodeNoisyNumber( i, noisybits );
       }
     }
+    ctx.closeAndGetEncodedLength();
     ctx = new StatCoderContext( ab );
 
     for ( int noisybits = 1; noisybits < 12; noisybits++ )
@@ -47,6 +48,7 @@ public class StatCoderContextTest
         ctx.encodeNoisyDiff( i, noisybits );
       }
     }
+    ctx.closeAndGetEncodedLength();
     ctx = new StatCoderContext( ab );
 
     for ( int noisybits = 0; noisybits < 12; noisybits++ )
@@ -74,6 +76,7 @@ public class StatCoderContextTest
         ctx.encodePredictedValue( value, predictor );
       }
     }
+    ctx.closeAndGetEncodedLength();
     ctx = new StatCoderContext( ab );
 
     for ( int value = -100; value < 100; value += 5 )
@@ -111,6 +114,7 @@ public class StatCoderContextTest
     StatCoderContext ctx = new StatCoderContext( ab );
     ctx.encodeSortedArray( values, 0, size, 0x08000000, 0 );
 
+    ctx.closeAndGetEncodedLength();
     ctx = new StatCoderContext( ab );
 
     int[] decodedValues = new int[size];

brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java

@@ -161,7 +161,7 @@ public abstract class BExpressionContext implements IByteArrayUnifier
 
     if ( nonNullTags == 0) return null;
 
-    int len = ctx.getEncodedLength();
+    int len = ctx.closeAndGetEncodedLength();
     byte[] ab = new byte[len];
     System.arraycopy( abBuf, 0, ab, 0, len );
 

brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java

@@ -144,16 +144,8 @@ final class OsmFile
       asize = getDataInputForSubIdx( subIdx, ab );
       dataBuffers = new DataBuffers( ab );
     }
-    // hack: the checksum contains the information
-    // which type of microcache we have
 
-    int crcData = Crc32.crc( ab, 0, asize - 4 );
+    try
-    int crcFooter = new ByteDataReader( ab, asize - 4 ).readInt();
-    if ( crcData == crcFooter )
-    {
-      throw new IOException( "old, unsupported data-format" );
-    }
-    else if ( ( crcData ^ 2 ) == crcFooter )
     {
       if ( !reallyDecode )
       {
@@ -166,7 +158,21 @@ final class OsmFile
       new DirectWeaver( dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher, hollowNodes );
       return MicroCache.emptyNonVirgin;
     }
-    throw new IOException( "checkum error" );
+    catch( Throwable t )
+    {
+      // checksum test now only in case of trouble
+      int crcData = Crc32.crc( ab, 0, asize - 4 );
+      int crcFooter = new ByteDataReader( ab, asize - 4 ).readInt();
+      if ( crcData == crcFooter )
+      {
+        throw new IOException( "old, unsupported data-format" );
+      }
+      else if ( ( crcData ^ 2 ) == crcFooter )
+      {
+        throw new IOException( "checkum error" );
+      }
+      throw t instanceof Exception ? (Exception)t : new Exception( t.toString(), t );
+    }
   }
 
   // set this OsmFile to ghost-state:

brouter-mapaccess/src/main/java/btools/mapaccess/OsmLink.java

@@ -45,13 +45,8 @@ public class OsmLink
    */
   public final OsmNode getTarget( OsmNode source )
   {
-/*    if ( isBidirectional() )
+    return n2 != source && n2 != null ? n2 : n1;
-    {
+    /* if ( n2 != null && n2 != source )
-      return n2 != source ? n2 : n1;
-    }
-    return n2 != null ? n2 : n1; */
-    // return n2 != source && n2 != null ? n2 : n1;
-    if ( n2 != null && n2 != source )
     {
       return n2;
     }
@@ -63,7 +58,7 @@ public class OsmLink
     {    
       new Throwable( "ups" ).printStackTrace();
       throw new IllegalArgumentException( "internal error: getTarget: unknown source; " + source + " n1=" + n1 + " n2=" + n2 );
-    }
+    } */
   }
   
   /**
@@ -71,8 +66,8 @@ public class OsmLink
    */
   public final OsmLink getNext( OsmNode source )
   {
-    // return n2 != source && n2 != null ? next : previous;
+    return n2 != source && n2 != null ? next : previous;
-    if ( n2 != null && n2 != source )
+    /* if ( n2 != null && n2 != source )
     {
       return next;
     }
@@ -83,7 +78,7 @@ public class OsmLink
     else
     {
       throw new IllegalArgumentException( "internal error: gextNext: unknown source" );
-    }
+    } */
   }
 
   /**
@@ -152,8 +147,8 @@ public class OsmLink
 
   public final boolean isReverse( OsmNode source )
   {
-  //  return n1 != source && n1 != null;
+    return n1 != source && n1 != null;
-    if ( n2 != null && n2 != source )
+    /* if ( n2 != null && n2 != source )
     {
       return false;
     }
@@ -164,7 +159,7 @@ public class OsmLink
     else
     {
       throw new IllegalArgumentException( "internal error: isReverse: unknown source" );
-    }
+    } */
   }
 
   public final boolean isBidirectional()

brouter-util/src/main/java/btools/util/BitCoderContext.java

@@ -6,15 +6,19 @@ public class BitCoderContext
   private byte[] ab;
   private int idxMax;
   private int idx = -1;
-  private int bm = 0x100; // byte mask (write mode)
+  private int bits; // bits left in buffer
-  private int bits; // bits left in buffer (read mode)
+  private int b; // buffer word
-  private int b;
 
   private static final int[] vl_values = new int[4096];
   private static final int[] vl_length = new int[4096];
 
+  private static final int[] vc_values = new int[4096];
+  private static final int[] vc_length = new int[4096];
+
   private static final int[] reverse_byte = new int[256];
 
+  private static final int[] bm2bits = new int[256];
+
   static
   {
     // fill varbits lookup table
@@ -30,6 +34,24 @@ public class BitCoderContext
       vl_values[i] = bc.decodeVarBits2();
       vl_length[i] = bc.getReadingBitPosition() - b0;
     }
+    for( int i=0; i<4096; i++ )
+    {
+      bc.reset();
+      int b0 = bc.getWritingBitPosition();
+      bc.encodeVarBits2( i );
+      vc_values[i] = bc.b;
+      vc_length[i] = bc.getWritingBitPosition() - b0;
+    }
+    for( int i=0; i<1024; i++ )
+    {
+      bc.reset();
+      bc.bits = 14;
+      bc.b = 0x1000 + i;
+
+      int b0 = bc.getReadingBitPosition();
+      vl_values[i] = bc.decodeVarBits2();
+      vl_length[i] = bc.getReadingBitPosition() - b0;
+    }
     for( int b=0; b<256; b++ )
     {
       int r = 0;
@@ -39,6 +61,10 @@ public class BitCoderContext
       }
       reverse_byte[b] = r;
     }
+    for( int b=0; b<8; b++ )
+    {
+      bm2bits[1<<b] = b;
+    }
   }
 
 
@@ -58,7 +84,6 @@ public class BitCoderContext
   public final void reset()
   {
     idx = -1;
-    bm = 0x100;
     bits = 0;
     b = 0;
   }
@@ -73,7 +98,7 @@ public class BitCoderContext
    *
    * @see #decodeVarBits
    */
-  public final void encodeVarBits( int value )
+  public final void encodeVarBits2( int value )
   {
     int range = 0;
     while (value > range)
@@ -86,6 +111,20 @@ public class BitCoderContext
     encodeBounded( range, value );
   }
 
+  public final void encodeVarBits( int value )
+  {
+    if ( (value & 0xfff) == value )
+    {
+      flushBuffer();
+      b |= vc_values[value] << bits;
+      bits += vc_length[value];      
+    }
+    else
+    {
+      encodeVarBits2( value ); // slow fallback for large values
+    }
+  }
+
   /**
    * @see #encodeVarBits
    */
@@ -120,20 +159,39 @@ public class BitCoderContext
       bits -= len;
       return mask;
     }
-    return decodeVarBits2();
+    if ( (b & 0xffffff) != 0 )
+    {
+      // here we just know len in [25..47]
+      // ( fillBuffer guarantees only 24 bits! )
+      b >>>= 12;
+      int len3 = 1 + (vl_length[b & 0xfff]>>1);
+      b >>>= len3;
+      int len2 = 11 + len3;
+      bits -= len2+1;
+      fillBuffer();
+      int mask = 0xffffffff >>> ( 32 - len2 );
+      mask += b & mask;
+      b >>>= len2;
+      bits -= len2;
+      return mask;
+    }
+    return decodeVarBits2(); // no chance, use the slow one
   }
 
 
   public final void encodeBit( boolean value )
   {
-    if ( bm == 0x100 )
+    if (bits > 31)
     {
-      bm = 1;
+      ab[++idx] = (byte)(b & 0xff);
-      ab[++idx] = 0;
+      b >>>= 8;
+      bits -=8;
     }
     if ( value )
-      ab[idx] |= bm;
+    {
-    bm <<= 1;
+      b |= 1 << bits;
+    }
+    bits++;
   }
 
   public final boolean decodeBit()
@@ -160,17 +218,15 @@ public class BitCoderContext
     int im = 1; // integer mask
     while (im <= max)
     {
-      if ( bm == 0x100 )
-      {
-        bm = 1;
-        ab[++idx] = 0;
-      }
       if ( ( value & im ) != 0 )
       {
-        ab[idx] |= bm;
+        encodeBit( true );
         max -= im;
       }
-      bm <<= 1;
+      else
+      {
+        encodeBit( false );
+      }
       im <<= 1;
     }
   }
@@ -239,27 +295,37 @@ public class BitCoderContext
     }
   }
 
+  private void flushBuffer()
+  {
+    while (bits > 7)
+    {
+      ab[++idx] = (byte)(b & 0xff);
+      b >>>= 8;
+      bits -=8;
+    }
+  }
+
   /**
+   * flushes and closes the (write-mode) context 
+   * 
    * @return the encoded length in bytes
    */
-  public final int getEncodedLength()
+  public final int closeAndGetEncodedLength()
   {
+    flushBuffer();
+    if ( bits > 0 )
+    {
+      ab[++idx] = (byte)(b & 0xff);
+    }
     return idx + 1;
   }
 
   /**
    * @return the encoded length in bits
    */
-  public final long getWritingBitPosition()
+  public final int getWritingBitPosition()
   {
-    long bitpos = idx << 3;
+    return (idx << 3) + 8 + bits;
-    int m = bm;
-    while (m > 1)
-    {
-      bitpos++;
-      m >>= 1;
-    }
-    return bitpos;
   }
 
   public final int getReadingBitPosition()
@@ -275,40 +341,33 @@ public class BitCoderContext
     b >>>= (8-bits);
   }
 
-  public final void copyBitsTo( byte[] dst, int bitcount )
+  public static void main( String[] args )
   {
-    int dstIdx = 0;
+    byte[] ab = new byte[581969];
-    for(;;)
+    BitCoderContext ctx = new BitCoderContext( ab );
+    for ( int i = 0; i < 31; i++ )
     {
-      if ( bitcount > 8 )
+      ctx.encodeVarBits( (1<<i)+3 );
-      {
+    }
-        if ( bits < 8 )
+    for ( int i = 0; i < 100000; i+=13 )
-        {
+    {
-          b |= (ab[++idx] & 0xff) << bits;
+      ctx.encodeVarBits( i );
-        }
+    }
-        else
+    ctx.closeAndGetEncodedLength();
-        {
+    ctx = new BitCoderContext( ab );
-          bits -= 8;
-        }
-        dst[dstIdx++] = (byte)b;
-        b >>>= 8;
-        bitcount -= 8;
-      }
-      else
-      {
-        if ( bits < bitcount )
-        {
-          b |= (ab[++idx] & 0xff) << bits;
-          bits += 8;
-        }
 
-        int mask = 0xff >>> ( 8 - bitcount );
+    for ( int i = 0; i < 31; i++ )
-        dst[dstIdx] = (byte)(b & mask);
+    {
-        bits -= bitcount;
+      int value = ctx.decodeVarBits();
-        b >>>= bitcount;
+      int v0 = (1<<i)+3;
-        break;
+      if ( !(v0 == value ) )
-      }
+        throw new RuntimeException( "value mismatch value=" + value + "v0=" + v0 );
+    }
+    for ( int i = 0; i < 100000; i+=13 )
+    {
+      int value = ctx.decodeVarBits();
+      if ( !(value == i ) )
+        throw new RuntimeException( "value mismatch i=" + i + "v=" + value );
     }
   }
-
 }

brouter-util/src/main/java/btools/util/SortedHeap.java

@@ -26,21 +26,12 @@ public final class SortedHeap<V>
   public V popLowestKeyValue()
   {
     SortedBin bin = firstNonEmpty;
-    if ( bin == null )
+    if ( firstNonEmpty == null )
     {
       return null;
     }
     size--;
-    int minId = bin.lv;
+    SortedBin minBin = firstNonEmpty.getMinBin();
-    SortedBin minBin = bin;
-    while( ( bin = bin.nextNonEmpty ) != null )
-    {
-      if ( bin.lv < minId )
-      {
-        minId = bin.lv;
-        minBin = bin;
-      }
-    }
     return (V) minBin.dropLowest();
   }
 
@@ -127,7 +118,112 @@ public final class SortedHeap<V>
       }
     }
 
+    // unrolled version of above for binsize = 4
+    void add4( int key, Object value )
+    {
+      int p = lp--;
+      if ( p == 4 || key < al[p] )
+      {
+        lv = al[p - 1] = key;
+        vla[p - 1] = value;
+        return;
+      }
+      lv = al[p - 1] = al[p];
+      vla[p - 1] = vla[p];
+      p++;
 
+      if ( p == 4 || key < al[p] )
+      {
+        al[p - 1] = key;
+        vla[p - 1] = value;
+        return;
+      }
+      al[p - 1] = al[p];
+      vla[p - 1] = vla[p];
+      p++;
+
+      if ( p == 4 || key < al[p] )
+      {
+        al[p - 1] = key;
+        vla[p - 1] = value;
+        return;
+      }
+      al[p - 1] = al[p];
+      vla[p - 1] = vla[p];
+
+      al[p] = key;
+      vla[p] = value;
+    }
+
+    // unrolled loop for performance sake
+    SortedBin getMinBin()
+    {
+      SortedBin minBin = this;
+      SortedBin bin = this;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      if ( (bin = bin.nextNonEmpty) == null ) return minBin;
+      if ( bin.lv < minBin.lv ) minBin = bin;
+      return minBin;
+    }
   }
 
   /**
@@ -146,7 +242,7 @@ public final class SortedHeap<V>
     }
     if ( first.lp > 0 )
     {
-      first.add( key, value );
+      first.add4( key, value );
       if ( firstNonEmpty != first )
       {
         first.nextNonEmpty = firstNonEmpty;
@@ -155,7 +251,7 @@ public final class SortedHeap<V>
     }
     else // second bin not full
     {
-      second.add( key, value );
+      second.add4( key, value );
       if ( first.nextNonEmpty != second )
       {
         second.nextNonEmpty = first.nextNonEmpty;
@@ -199,21 +295,9 @@ public final class SortedHeap<V>
     // now merge the content of these non-empty bins into the target bin
     while( firstNonEmpty != null )
     {
-      SortedBin ne = firstNonEmpty;
+      // copy current minimum to target array
-      SortedBin minBin = ne;
+      SortedBin minBin = firstNonEmpty.getMinBin();
-      int minId = minBin.lv;
+      al_t[tp] = minBin.lv;
-
-      while ( ( ne = ne.nextNonEmpty ) != null )
-      {
-        if ( ne.lv < minId )
-        {
-          minBin = ne;
-          minId = minBin.lv;
-        }
-      }
-
-      // current minimum found, copy to target array
-      al_t[tp] = minId;
       vla_t[tp++] = minBin.dropLowest();
     }
 
@@ -265,35 +349,4 @@ public final class SortedHeap<V>
     }
     return tp;
   }
-
-  public static void main(String[] args)
-  {
-    SortedHeap<String> sh = new SortedHeap<String>();
-    Random rnd = new Random();
-    for( int i = 0; i< 1000; i++ )
-    {
-      int val = rnd.nextInt( 1000000 );
-      sh.add( val, "" + val );
-      val = rnd.nextInt( 1000000 );
-      sh.add( val, "" + val );
-      sh.popLowestKeyValue();
-    }
-
-    int cnt = 0;
-    int lastval = 0;
-    for(;;)
-    {
-      String s = sh.popLowestKeyValue();
-      if ( s == null ) break;
-      cnt ++;
-      int val = Integer.parseInt( s );
-System.out.println( "popLowestKeyValue: " + val);
-//      Assert.assertTrue( "sorting test", val >= lastval );
-      lastval = val;
-    }
-//    Assert.assertTrue( "total count test", cnt == 100000 );
-
-  }
-
-
 }

brouter-util/src/test/java/btools/util/BitCoderContextTest.java

@@ -8,18 +8,29 @@ public class BitCoderContextTest
   @Test
   public void varBitsEncodeDecodeTest()
   {
-    byte[] ab = new byte[4000];
+    byte[] ab = new byte[581969];
     BitCoderContext ctx = new BitCoderContext( ab );
-    for ( int i = 0; i < 1000; i++ )
+    for ( int i = 0; i < 31; i++ )
+    {
+      ctx.encodeVarBits( (1<<i)+3 );
+    }
+    for ( int i = 0; i < 100000; i+=13 )
     {
       ctx.encodeVarBits( i );
     }
+    ctx.closeAndGetEncodedLength();
     ctx = new BitCoderContext( ab );
 
-    for ( int i = 0; i < 1000; i++ )
+    for ( int i = 0; i < 31; i++ )
     {
       int value = ctx.decodeVarBits();
-      Assert.assertTrue( "distance value mismatch i=" + i + "v=" + value, value == i );
+      int v0 = (1<<i)+3;
+      Assert.assertTrue( "value mismatch value=" + value + "v0=" + v0, v0 == value );
+    }
+    for ( int i = 0; i < 100000; i+=13 )
+    {
+      int value = ctx.decodeVarBits();
+      Assert.assertTrue( "value mismatch i=" + i + "v=" + value, value == i );
     }
   }
 
@@ -35,6 +46,7 @@ public class BitCoderContextTest
         ctx.encodeBounded( max, val );
       }
     }
+    ctx.closeAndGetEncodedLength();
 
     ctx = new BitCoderContext( ab );