From 2213d4f7fcd9a2d638aa47f1ab1aa8653f74555b Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Sat, 20 Jul 2019 10:10:51 +0200 Subject: [PATCH] some microtuning --- .../main/java/btools/codec/MicroCache2.java | 2 +- .../main/java/btools/codec/TagValueCoder.java | 2 +- .../btools/codec/StatCoderContextTest.java | 4 + .../expressions/BExpressionContext.java | 2 +- .../main/java/btools/mapaccess/OsmFile.java | 26 ++- .../main/java/btools/mapaccess/OsmLink.java | 23 +-- .../java/btools/util/BitCoderContext.java | 177 ++++++++++++------ .../src/main/java/btools/util/SortedHeap.java | 171 +++++++++++------ .../java/btools/util/BitCoderContextTest.java | 20 +- 9 files changed, 278 insertions(+), 149 deletions(-) diff --git a/brouter-codec/src/main/java/btools/codec/MicroCache2.java b/brouter-codec/src/main/java/btools/codec/MicroCache2.java index 7829384..c291f01 100644 --- a/brouter-codec/src/main/java/btools/codec/MicroCache2.java +++ b/brouter-codec/src/main/java/btools/codec/MicroCache2.java @@ -502,7 +502,7 @@ public final class MicroCache2 extends MicroCache } if ( pass == 3 ) { - return bc.getEncodedLength(); + return bc.closeAndGetEncodedLength(); } } } diff --git a/brouter-codec/src/main/java/btools/codec/TagValueCoder.java b/brouter-codec/src/main/java/btools/codec/TagValueCoder.java index b15c0a5..a733085 100644 --- a/brouter-codec/src/main/java/btools/codec/TagValueCoder.java +++ b/brouter-codec/src/main/java/btools/codec/TagValueCoder.java @@ -137,7 +137,7 @@ public final class TagValueCoder } byte[] res; - int len = ctx.getEncodedLength(); + int len = ctx.closeAndGetEncodedLength(); if ( validator == null ) { res = new byte[len]; diff --git a/brouter-codec/src/test/java/btools/codec/StatCoderContextTest.java b/brouter-codec/src/test/java/btools/codec/StatCoderContextTest.java index 616e421..b24253b 100644 --- a/brouter-codec/src/test/java/btools/codec/StatCoderContextTest.java +++ b/brouter-codec/src/test/java/btools/codec/StatCoderContextTest.java @@ -20,6 +20,7 @@ public class StatCoderContextTest ctx.encodeNoisyNumber( i, noisybits ); } } + ctx.closeAndGetEncodedLength(); ctx = new StatCoderContext( ab ); for ( int noisybits = 1; noisybits < 12; noisybits++ ) @@ -47,6 +48,7 @@ public class StatCoderContextTest ctx.encodeNoisyDiff( i, noisybits ); } } + ctx.closeAndGetEncodedLength(); ctx = new StatCoderContext( ab ); for ( int noisybits = 0; noisybits < 12; noisybits++ ) @@ -74,6 +76,7 @@ public class StatCoderContextTest ctx.encodePredictedValue( value, predictor ); } } + ctx.closeAndGetEncodedLength(); ctx = new StatCoderContext( ab ); for ( int value = -100; value < 100; value += 5 ) @@ -111,6 +114,7 @@ public class StatCoderContextTest StatCoderContext ctx = new StatCoderContext( ab ); ctx.encodeSortedArray( values, 0, size, 0x08000000, 0 ); + ctx.closeAndGetEncodedLength(); ctx = new StatCoderContext( ab ); int[] decodedValues = new int[size]; diff --git a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java index 2f17781..2b6d9e0 100644 --- a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java +++ b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java @@ -161,7 +161,7 @@ public abstract class BExpressionContext implements IByteArrayUnifier if ( nonNullTags == 0) return null; - int len = ctx.getEncodedLength(); + int len = ctx.closeAndGetEncodedLength(); byte[] ab = new byte[len]; System.arraycopy( abBuf, 0, ab, 0, len ); diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java index e468fe3..4645d1f 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java @@ -144,16 +144,8 @@ final class OsmFile asize = getDataInputForSubIdx( subIdx, ab ); dataBuffers = new DataBuffers( ab ); } - // hack: the checksum contains the information - // which type of microcache we have - int crcData = Crc32.crc( ab, 0, asize - 4 ); - int crcFooter = new ByteDataReader( ab, asize - 4 ).readInt(); - if ( crcData == crcFooter ) - { - throw new IOException( "old, unsupported data-format" ); - } - else if ( ( crcData ^ 2 ) == crcFooter ) + try { if ( !reallyDecode ) { @@ -166,7 +158,21 @@ final class OsmFile new DirectWeaver( dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher, hollowNodes ); return MicroCache.emptyNonVirgin; } - throw new IOException( "checkum error" ); + catch( Throwable t ) + { + // checksum test now only in case of trouble + int crcData = Crc32.crc( ab, 0, asize - 4 ); + int crcFooter = new ByteDataReader( ab, asize - 4 ).readInt(); + if ( crcData == crcFooter ) + { + throw new IOException( "old, unsupported data-format" ); + } + else if ( ( crcData ^ 2 ) == crcFooter ) + { + throw new IOException( "checkum error" ); + } + throw t instanceof Exception ? (Exception)t : new Exception( t.toString(), t ); + } } // set this OsmFile to ghost-state: diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmLink.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmLink.java index be962be..4d548fc 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmLink.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmLink.java @@ -45,13 +45,8 @@ public class OsmLink */ public final OsmNode getTarget( OsmNode source ) { -/* if ( isBidirectional() ) - { - return n2 != source ? n2 : n1; - } - return n2 != null ? n2 : n1; */ - // return n2 != source && n2 != null ? n2 : n1; - if ( n2 != null && n2 != source ) + return n2 != source && n2 != null ? n2 : n1; + /* if ( n2 != null && n2 != source ) { return n2; } @@ -63,7 +58,7 @@ public class OsmLink { new Throwable( "ups" ).printStackTrace(); throw new IllegalArgumentException( "internal error: getTarget: unknown source; " + source + " n1=" + n1 + " n2=" + n2 ); - } + } */ } /** @@ -71,8 +66,8 @@ public class OsmLink */ public final OsmLink getNext( OsmNode source ) { - // return n2 != source && n2 != null ? next : previous; - if ( n2 != null && n2 != source ) + return n2 != source && n2 != null ? next : previous; + /* if ( n2 != null && n2 != source ) { return next; } @@ -83,7 +78,7 @@ public class OsmLink else { throw new IllegalArgumentException( "internal error: gextNext: unknown source" ); - } + } */ } /** @@ -152,8 +147,8 @@ public class OsmLink public final boolean isReverse( OsmNode source ) { - // return n1 != source && n1 != null; - if ( n2 != null && n2 != source ) + return n1 != source && n1 != null; + /* if ( n2 != null && n2 != source ) { return false; } @@ -164,7 +159,7 @@ public class OsmLink else { throw new IllegalArgumentException( "internal error: isReverse: unknown source" ); - } + } */ } public final boolean isBidirectional() diff --git a/brouter-util/src/main/java/btools/util/BitCoderContext.java b/brouter-util/src/main/java/btools/util/BitCoderContext.java index 4d80db2..9154702 100644 --- a/brouter-util/src/main/java/btools/util/BitCoderContext.java +++ b/brouter-util/src/main/java/btools/util/BitCoderContext.java @@ -6,15 +6,19 @@ public class BitCoderContext private byte[] ab; private int idxMax; private int idx = -1; - private int bm = 0x100; // byte mask (write mode) - private int bits; // bits left in buffer (read mode) - private int b; + private int bits; // bits left in buffer + private int b; // buffer word private static final int[] vl_values = new int[4096]; private static final int[] vl_length = new int[4096]; + private static final int[] vc_values = new int[4096]; + private static final int[] vc_length = new int[4096]; + private static final int[] reverse_byte = new int[256]; + private static final int[] bm2bits = new int[256]; + static { // fill varbits lookup table @@ -30,6 +34,24 @@ public class BitCoderContext vl_values[i] = bc.decodeVarBits2(); vl_length[i] = bc.getReadingBitPosition() - b0; } + for( int i=0; i<4096; i++ ) + { + bc.reset(); + int b0 = bc.getWritingBitPosition(); + bc.encodeVarBits2( i ); + vc_values[i] = bc.b; + vc_length[i] = bc.getWritingBitPosition() - b0; + } + for( int i=0; i<1024; i++ ) + { + bc.reset(); + bc.bits = 14; + bc.b = 0x1000 + i; + + int b0 = bc.getReadingBitPosition(); + vl_values[i] = bc.decodeVarBits2(); + vl_length[i] = bc.getReadingBitPosition() - b0; + } for( int b=0; b<256; b++ ) { int r = 0; @@ -39,6 +61,10 @@ public class BitCoderContext } reverse_byte[b] = r; } + for( int b=0; b<8; b++ ) + { + bm2bits[1< range) @@ -86,6 +111,20 @@ public class BitCoderContext encodeBounded( range, value ); } + public final void encodeVarBits( int value ) + { + if ( (value & 0xfff) == value ) + { + flushBuffer(); + b |= vc_values[value] << bits; + bits += vc_length[value]; + } + else + { + encodeVarBits2( value ); // slow fallback for large values + } + } + /** * @see #encodeVarBits */ @@ -120,20 +159,39 @@ public class BitCoderContext bits -= len; return mask; } - return decodeVarBits2(); + if ( (b & 0xffffff) != 0 ) + { + // here we just know len in [25..47] + // ( fillBuffer guarantees only 24 bits! ) + b >>>= 12; + int len3 = 1 + (vl_length[b & 0xfff]>>1); + b >>>= len3; + int len2 = 11 + len3; + bits -= len2+1; + fillBuffer(); + int mask = 0xffffffff >>> ( 32 - len2 ); + mask += b & mask; + b >>>= len2; + bits -= len2; + return mask; + } + return decodeVarBits2(); // no chance, use the slow one } public final void encodeBit( boolean value ) { - if ( bm == 0x100 ) + if (bits > 31) { - bm = 1; - ab[++idx] = 0; + ab[++idx] = (byte)(b & 0xff); + b >>>= 8; + bits -=8; } if ( value ) - ab[idx] |= bm; - bm <<= 1; + { + b |= 1 << bits; + } + bits++; } public final boolean decodeBit() @@ -160,17 +218,15 @@ public class BitCoderContext int im = 1; // integer mask while (im <= max) { - if ( bm == 0x100 ) - { - bm = 1; - ab[++idx] = 0; - } if ( ( value & im ) != 0 ) { - ab[idx] |= bm; + encodeBit( true ); max -= im; } - bm <<= 1; + else + { + encodeBit( false ); + } im <<= 1; } } @@ -239,27 +295,37 @@ public class BitCoderContext } } + private void flushBuffer() + { + while (bits > 7) + { + ab[++idx] = (byte)(b & 0xff); + b >>>= 8; + bits -=8; + } + } + /** + * flushes and closes the (write-mode) context + * * @return the encoded length in bytes */ - public final int getEncodedLength() + public final int closeAndGetEncodedLength() { + flushBuffer(); + if ( bits > 0 ) + { + ab[++idx] = (byte)(b & 0xff); + } return idx + 1; } /** * @return the encoded length in bits */ - public final long getWritingBitPosition() + public final int getWritingBitPosition() { - long bitpos = idx << 3; - int m = bm; - while (m > 1) - { - bitpos++; - m >>= 1; - } - return bitpos; + return (idx << 3) + 8 + bits; } public final int getReadingBitPosition() @@ -275,40 +341,33 @@ public class BitCoderContext b >>>= (8-bits); } - public final void copyBitsTo( byte[] dst, int bitcount ) + public static void main( String[] args ) { - int dstIdx = 0; - for(;;) + byte[] ab = new byte[581969]; + BitCoderContext ctx = new BitCoderContext( ab ); + for ( int i = 0; i < 31; i++ ) { - if ( bitcount > 8 ) - { - if ( bits < 8 ) - { - b |= (ab[++idx] & 0xff) << bits; - } - else - { - bits -= 8; - } - dst[dstIdx++] = (byte)b; - b >>>= 8; - bitcount -= 8; - } - else - { - if ( bits < bitcount ) - { - b |= (ab[++idx] & 0xff) << bits; - bits += 8; - } + ctx.encodeVarBits( (1<>> ( 8 - bitcount ); - dst[dstIdx] = (byte)(b & mask); - bits -= bitcount; - b >>>= bitcount; - break; - } + for ( int i = 0; i < 31; i++ ) + { + int value = ctx.decodeVarBits(); + int v0 = (1<= lastval ); - lastval = val; - } -// Assert.assertTrue( "total count test", cnt == 100000 ); - - } - - } diff --git a/brouter-util/src/test/java/btools/util/BitCoderContextTest.java b/brouter-util/src/test/java/btools/util/BitCoderContextTest.java index 9c11e27..084a29c 100644 --- a/brouter-util/src/test/java/btools/util/BitCoderContextTest.java +++ b/brouter-util/src/test/java/btools/util/BitCoderContextTest.java @@ -8,18 +8,29 @@ public class BitCoderContextTest @Test public void varBitsEncodeDecodeTest() { - byte[] ab = new byte[4000]; + byte[] ab = new byte[581969]; BitCoderContext ctx = new BitCoderContext( ab ); - for ( int i = 0; i < 1000; i++ ) + for ( int i = 0; i < 31; i++ ) + { + ctx.encodeVarBits( (1<