Fabio/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

more performance tuning

Browse source
This commit is contained in:
Arndt 2016-08-28 19:43:40 +02:00
parent acb7c6b16f
commit ea572ad47f
10 changed files with 357 additions and 165 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
brouter-codec/src/main/java/btools/codec/MicroCache2.java
View file

@ -102,8 +102,7 @@ public final class MicroCache2 extends MicroCache
if ( debug ) System.out.println( "*** decoding node with links=" + links );
for( int li=0; li<links; li++ )
{
int startPointer = aboffset;
int sizeoffset = writeSizePlaceHolder();
int sizeoffset = 0;
int nodeIdx = n + nodeIdxDiff.decodeSignedValue();
int dlon_remaining;
@ -112,19 +111,25 @@ public final class MicroCache2 extends MicroCache
boolean isReverse = false;
if ( nodeIdx != n ) // internal (forward-) link
{
writeVarLengthSigned( dlon_remaining = alon[nodeIdx] - ilon );
writeVarLengthSigned( dlat_remaining = alat[nodeIdx] - ilat );
dlon_remaining = alon[nodeIdx] - ilon;
dlat_remaining = alat[nodeIdx] - ilat;
}
else
{
isReverse = bc.decodeBit();
writeVarLengthSigned( dlon_remaining = extLonDiff.decodeSignedValue() );
writeVarLengthSigned( dlat_remaining = extLatDiff.decodeSignedValue() );
dlon_remaining = extLonDiff.decodeSignedValue();
dlat_remaining = extLatDiff.decodeSignedValue();
}
TagValueWrapper wayTags = wayTagCoder.decodeTagValueSet();
if ( wayTags != null )
{
int startPointer = aboffset;
sizeoffset = writeSizePlaceHolder();
writeVarLengthSigned( dlon_remaining );
writeVarLengthSigned( dlat_remaining );
validNodes.set( n, true ); // mark source-node valid
if ( nodeIdx != n ) // valid internal (forward-) link
{
@ -132,9 +137,9 @@ public final class MicroCache2 extends MicroCache
finaldatasize += 1 + aboffset-startPointer; // reserve place for reverse
validNodes.set( nodeIdx, true ); // mark target-node valid
}
writeModeAndDesc( isReverse, wayTags.data );
}
writeModeAndDesc( isReverse, wayTags == null ? null : wayTags.data );
if ( !isReverse ) // write geometry for forward links only
{
WaypointMatcher matcher = wayTags == null || wayTags.accessType < 2 ? null : waypointMatcher;
@ -152,19 +157,19 @@ public final class MicroCache2 extends MicroCache
dlon_remaining -= dlon;
dlat_remaining -= dlat;
count--;
writeVarLengthSigned( dlon );
writeVarLengthSigned( dlat );
writeVarLengthSigned( transEleDiff.decodeSignedValue() );
int elediff = transEleDiff.decodeSignedValue();
if ( wayTags != null )
{
writeVarLengthSigned( dlon );
writeVarLengthSigned( dlat );
writeVarLengthSigned( elediff );
}
if ( matcher != null ) matcher.transferNode( ilontarget - dlon_remaining, ilattarget - dlat_remaining );
}
if ( matcher != null ) matcher.endNode( ilontarget, ilattarget );
}
if ( wayTags == null )
{
aboffset = startPointer; // not a valid link, delete it
}
else
if ( wayTags != null )
{
injectSize( sizeoffset );
}

4
brouter-core/src/main/java/btools/router/ProfileCache.java
View file

@ -63,8 +63,8 @@ public final class ProfileCache
BExpressionMetaData meta = new BExpressionMetaData();
BExpressionContextGlobal expctxGlobal = new BExpressionContextGlobal( meta );
rc.expctxWay = new BExpressionContextWay( rc.serversizing ? 262144 : 32768, meta );
rc.expctxNode = new BExpressionContextNode( rc.serversizing ? 16384 : 4096, meta );
rc.expctxWay = new BExpressionContextWay( rc.serversizing ? 131072 : 32768, meta );
rc.expctxNode = new BExpressionContextNode( rc.serversizing ? 4096 : 1024, meta );
meta.readMetaData( new File( profileDir, "lookups.dat" ) );

191
brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java
View file

@ -21,6 +21,7 @@ import java.util.TreeMap;
import btools.util.BitCoderContext;
import btools.util.Crc32;
import btools.util.IByteArrayUnifier;
import btools.util.LruMap;
public abstract class BExpressionContext implements IByteArrayUnifier
@ -49,10 +50,11 @@ public abstract class BExpressionContext implements IByteArrayUnifier
// hash-cache for function results
private byte[][] _arrayBitmap;
private int[] _arrayCrc;
private CacheNode probeCacheNode = new CacheNode();
private LruMap cache;
private int currentHashBucket = -1;
private VarWrapper probeVarSet = new VarWrapper();
private LruMap resultVarCache;
private List<BExpression> expressionList;
@ -60,20 +62,21 @@ public abstract class BExpressionContext implements IByteArrayUnifier
// build-in variable indexes for fast access
private int[] buildInVariableIdx;
private float[][] arrayBuildInVariablesCache;
private float[] hashBucketVars;
private int nBuildInVars;
private float[] currentVars;
private int currentVarOffset;
protected void setInverseVars()
{
currentVarOffset = nBuildInVars;
}
abstract String[] getBuildInVariableNames();
protected final float getBuildInVariable( int idx )
{
return hashBucketVars[idx];
}
protected final float getInverseBuildInVariable( int idx )
{
return arrayBuildInVariablesCache[currentHashBucket | 1][idx];
return currentVars[idx+currentVarOffset];
}
private int linenr;
@ -102,9 +105,8 @@ public abstract class BExpressionContext implements IByteArrayUnifier
if ( Boolean.getBoolean( "disableExpressionCache" ) ) hashSize = 1;
// create the expression cache
_arrayBitmap = new byte[hashSize][];
_arrayCrc = new int[hashSize];
arrayBuildInVariablesCache = new float[hashSize << 1][];
cache = new LruMap( 4*hashSize, hashSize );
resultVarCache = new LruMap( 4096, 4096 );
}
/**
@ -279,117 +281,113 @@ public abstract class BExpressionContext implements IByteArrayUnifier
return "requests=" + requests + " requests2=" + requests2 + " cachemisses=" + cachemisses;
}
@Override
// @Override
public final byte[] unify( byte[] ab, int offset, int len )
{
int crc = Crc32.crc( ab, offset, len );
int hashSize = _arrayBitmap.length;
int idx = ( crc & 0xfffffff ) % hashSize;
byte[] abc = _arrayBitmap[idx];
if ( abc != null && abc.length == len )
probeCacheNode.ab = null; // crc based cache lookup only
probeCacheNode.crc = Crc32.crc( ab, offset, len );
CacheNode cn = (CacheNode)cache.get( probeCacheNode );
if ( cn != null )
{
int i = 0;
while (i < len)
byte[] cab = cn.ab;
if ( cab.length == len )
{
if ( ab[offset + i] != abc[i] )
break;
i++;
for( int i=0; i<len; i++ )
{
if ( cab[i] != ab[i+offset] )
{
cn = null;
break;
}
}
if ( cn != null )
{
return cn.ab;
}
}
if ( i == len )
return abc;
}
byte[] nab = new byte[len];
System.arraycopy( ab, offset, nab, 0, len );
return nab;
}
/**
* evaluates the data in the given byte array
*
* @return true if the data is equivilant to the last calls data
*/
public final void evaluate( boolean inverseDirection, byte[] ab )
{
requests++;
lookupDataValid = false; // this is an assertion for a nasty pifall
// calc hash bucket from crc
int crc = Crc32.crc( ab, 0, ab.length );
int hashSize = _arrayBitmap.length;
int hashBucket2 = ( crc & 0xfffffff ) % hashSize;
int hashBucket01 = hashBucket2 << 1;
int hashBucket02 = hashBucket01 | 1;
int hashBucket = inverseDirection ? hashBucket02 : hashBucket01;
probeCacheNode.ab = ab;
probeCacheNode.crc = Crc32.crc( ab, 0, ab.length );
if ( hashBucket != currentHashBucket )
CacheNode cn = (CacheNode)cache.get( probeCacheNode );
if ( cn == null )
{
currentHashBucket = hashBucket;
hashBucketVars = arrayBuildInVariablesCache[hashBucket];
if ( hashBucketVars == null )
cachemisses++;
cn = (CacheNode)cache.removeLru();
if ( cn == null )
{
arrayBuildInVariablesCache[hashBucket01] = new float[buildInVariableIdx.length];
arrayBuildInVariablesCache[hashBucket02] = new float[buildInVariableIdx.length];
hashBucketVars = arrayBuildInVariablesCache[hashBucket];
cn = new CacheNode();
}
}
if ( crc == _arrayCrc[hashBucket2] )
{
byte[] abBucket = _arrayBitmap[hashBucket2];
if ( ab == abBucket ) // fast identity check
{
return;
}
requests2++;
cn.crc = probeCacheNode.crc;
cn.ab = ab;
cache.put( cn );
// compare input value to hash bucket content
boolean hashBucketEquals = false;
int abLen = ab.length;
if ( abBucket != null && abBucket.length == abLen )
if ( probeVarSet.vars == null )
{
hashBucketEquals = true;
for ( int i = 0; i < abLen; i++ )
probeVarSet.vars = new float[2*nBuildInVars];
}
// forward direction
decode( lookupData, false, ab );
evaluateInto( probeVarSet.vars, 0 );
// inverse direction
lookupData[0] = 2; // inverse shortcut: reuse decoding
evaluateInto( probeVarSet.vars, nBuildInVars );
probeVarSet.hash = Arrays.hashCode( probeVarSet.vars );
// unify the result variable set
VarWrapper vw = (VarWrapper)resultVarCache.get( probeVarSet );
if ( vw == null )
{
vw = (VarWrapper)resultVarCache.removeLru();
if ( vw == null )
{
if ( abBucket[i] != ab[i] )
{
hashBucketEquals = false;
break;
}
vw = new VarWrapper();
}
vw.hash = probeVarSet.hash;
vw.vars = probeVarSet.vars;
probeVarSet.vars = null;
resultVarCache.put( vw );
}
if ( hashBucketEquals )
{
return;
}
cn.vars = vw.vars;
}
cachemisses++;
_arrayBitmap[hashBucket2] = ab;
_arrayCrc[hashBucket2] = crc;
int nBuildInVars = buildInVariableIdx.length;
// forward direction
decode( lookupData, false, ab );
evaluate();
float[] vars = arrayBuildInVariablesCache[hashBucket01];
for ( int vi = 0; vi < nBuildInVars; vi++ )
else
{
int idx = buildInVariableIdx[vi];
vars[vi] = idx == -1 ? 0.f : variableData[idx];
}
// inverse direction
lookupData[0] = 2; // inverse shortcut: reuse decoding
evaluate();
vars = arrayBuildInVariablesCache[hashBucket02];
for ( int vi = 0; vi < nBuildInVars; vi++ )
{
int idx = buildInVariableIdx[vi];
vars[vi] = idx == -1 ? 0.f : variableData[idx];
if ( ab == cn.ab ) requests2++;
cache.touch( cn );
}
currentVars = cn.vars;
currentVarOffset = inverseDirection ? nBuildInVars : 0;
}
private void evaluateInto( float[] vars, int offset )
{
evaluate();
for ( int vi = 0; vi < nBuildInVars; vi++ )
{
int idx = buildInVariableIdx[vi];
vars[vi+offset] = idx == -1 ? 0.f : variableData[idx];
}
}
public void dumpStatistics()
{
TreeMap<String,String> counts = new TreeMap<String,String>();
@ -650,7 +648,8 @@ public abstract class BExpressionContext implements IByteArrayUnifier
// determine the build-in variable indices
String[] varNames = getBuildInVariableNames();
buildInVariableIdx = new int[varNames.length];
nBuildInVars = varNames.length;
buildInVariableIdx = new int[nBuildInVars];
for( int vi=0; vi<varNames.length; vi++ )
{
buildInVariableIdx[vi] = getVariableIdx( varNames[vi], false );

5
brouter-expressions/src/main/java/btools/expressions/BExpressionContextWay.java
View file

@ -32,8 +32,6 @@ public final class BExpressionContextWay extends BExpressionContext implements T
public float getPriorityClassifier() { return getBuildInVariable(9); }
public float getClassifierMask() { return getBuildInVariable(10); }
public float getInverseCostfactor() { return getInverseBuildInVariable(0); }
public BExpressionContextWay( BExpressionMetaData meta )
{
super( "way", meta );
@ -56,7 +54,8 @@ public final class BExpressionContextWay extends BExpressionContext implements T
float minCostFactor = getCostfactor();
if ( minCostFactor >= 9999.f )
{
float reverseCostFactor = getInverseCostfactor();
setInverseVars();
float reverseCostFactor = getCostfactor();
if ( reverseCostFactor < minCostFactor )
{
minCostFactor = reverseCostFactor;

33
brouter-expressions/src/main/java/btools/expressions/CacheNode.java Normal file
View file

@ -0,0 +1,33 @@
package btools.expressions;
import java.util.Arrays;
import btools.util.LruMapNode;
public final class CacheNode extends LruMapNode
{
int crc;
byte[] ab;
float[] vars;
@Override
public int hashCode()
{
return crc;
}
@Override
public boolean equals( Object o )
{
CacheNode n = (CacheNode) o;
if ( crc != n.crc )
{
return false;
}
if ( ab == null )
{
return true; // hack: null = crc match only
}
return Arrays.equals( ab, n.ab );
}
}

28
brouter-expressions/src/main/java/btools/expressions/VarWrapper.java Normal file
View file

@ -0,0 +1,28 @@
package btools.expressions;
import java.util.Arrays;
import btools.util.LruMapNode;
public final class VarWrapper extends LruMapNode
{
int hash;
float[] vars;
@Override
public int hashCode()
{
return hash;
}
@Override
public boolean equals( Object o )
{
VarWrapper n = (VarWrapper) o;
if ( hash != n.hash )
{
return false;
}
return Arrays.equals( vars, n.vars );
}
}

27
brouter-util/src/main/java/btools/util/ByteDataReader.java
View file

@ -107,23 +107,6 @@ public class ByteDataReader
return ( v & 1 ) == 0 ? v >> 1 : -(v >> 1 );
}
public final int readVarLengthUnsigned_old()
{
int v = 0;
int shift = 0;
for ( ;; )
{
byte b = ab[aboffset++];
v |= ( ( b & 0x7f ) << shift );
if ( ( b & 0x80 ) == 0 )
{
break;
}
shift += 7;
}
return v;
}
public final int readVarLengthUnsigned()
{
byte b;
@ -135,15 +118,7 @@ public class ByteDataReader
if ( b >= 0 ) return v;
v |= ( (b=ab[aboffset++]) & 0x7f ) << 21;
if ( b >= 0 ) return v;
v |= ( (b=ab[aboffset++]) & 0x7f ) << 28;
if ( b >= 0 ) return v;
v |= ( (b=ab[aboffset++]) & 0x7f ) << 35;
if ( b >= 0 ) return v;
v |= ( (b=ab[aboffset++]) & 0x7f ) << 42;
if ( b >= 0 ) return v;
v |= ( (b=ab[aboffset++]) & 0x7f ) << 49;
if ( b >= 0 ) return v;
v |= ( (b=ab[aboffset++]) & 0x7f ) << 56;
v |= ( (b=ab[aboffset++]) & 0xf ) << 28;
return v;
}

70
brouter-util/src/main/java/btools/util/ByteDataWriter.java
View file

@ -13,7 +13,7 @@ public class ByteDataWriter extends ByteDataReader
super ( byteArray );
}
public void writeInt( int v )
public final void writeInt( int v )
{
ab[aboffset++] = (byte)( (v >> 24) & 0xff );
ab[aboffset++] = (byte)( (v >> 16) & 0xff );
@ -21,7 +21,7 @@ public class ByteDataWriter extends ByteDataReader
ab[aboffset++] = (byte)( (v ) & 0xff );
}
public void writeLong( long v )
public final void writeLong( long v )
{
ab[aboffset++] = (byte)( (v >> 56) & 0xff );
ab[aboffset++] = (byte)( (v >> 48) & 0xff );
@ -33,35 +33,35 @@ public class ByteDataWriter extends ByteDataReader
ab[aboffset++] = (byte)( (v ) & 0xff );
}
public void writeBoolean( boolean v)
public final void writeBoolean( boolean v)
{
ab[aboffset++] = (byte)( v ? 1 : 0 );
}
public void writeByte( int v )
public final void writeByte( int v )
{
ab[aboffset++] = (byte)( (v ) & 0xff );
}
public void writeShort( int v )
public final void writeShort( int v )
{
ab[aboffset++] = (byte)( (v >> 8) & 0xff );
ab[aboffset++] = (byte)( (v ) & 0xff );
}
public void write( byte[] sa )
public final void write( byte[] sa )
{
System.arraycopy( sa, 0, ab, aboffset, sa.length );
aboffset += sa.length;
}
public void write( byte[] sa, int offset, int len )
public final void write( byte[] sa, int offset, int len )
{
System.arraycopy( sa, offset, ab, aboffset, len );
aboffset += len;
}
public void writeVarBytes( byte[] sa )
public final void writeVarBytes( byte[] sa )
{
if ( sa == null )
{
@ -75,7 +75,7 @@ public class ByteDataWriter extends ByteDataReader
}
}
public void writeModeAndDesc( boolean isReverse, byte[] sa )
public final void writeModeAndDesc( boolean isReverse, byte[] sa )
{
int len = sa == null ? 0 : sa.length;
int sizecode = len << 1 | ( isReverse ? 1 : 0 );
@ -87,7 +87,7 @@ public class ByteDataWriter extends ByteDataReader
}
public byte[] toByteArray()
public final byte[] toByteArray()
{
byte[] c = new byte[aboffset];
System.arraycopy( ab, 0, c, 0, aboffset );
@ -102,12 +102,12 @@ public class ByteDataWriter extends ByteDataReader
*
* @return the offset of the placeholder
*/
public int writeSizePlaceHolder()
public final int writeSizePlaceHolder()
{
return aboffset++;
}
public void injectSize( int sizeoffset )
public final void injectSize( int sizeoffset )
{
int size = 0;
int datasize = aboffset-sizeoffset-1;
@ -127,23 +127,47 @@ public class ByteDataWriter extends ByteDataReader
aboffset = sizeoffset + size + datasize;
}
public int writeVarLengthSigned( int v )
public final void writeVarLengthSigned( int v )
{
return writeVarLengthUnsigned( v < 0 ? ( (-v) << 1 ) | 1 : v << 1 );
writeVarLengthUnsigned( v < 0 ? ( (-v) << 1 ) | 1 : v << 1 );
}
public int writeVarLengthUnsigned( int v )
public final void writeVarLengthUnsigned( int v )
{
int start = aboffset;
do
int i7 = v & 0x7f;
if ( ( v >>>= 7 ) == 0 )
{
int i7 = v & 0x7f;
v >>= 7;
if ( v != 0 ) i7 |= 0x80;
ab[aboffset++] = (byte)( i7 & 0xff );
ab[aboffset++] = (byte) ( i7 );
return;
}
while( v != 0 );
return aboffset - start;
ab[aboffset++] = (byte) ( i7 | 0x80 );
i7 = v & 0x7f;
if ( ( v >>>= 7 ) == 0 )
{
ab[aboffset++] = (byte) ( i7 );
return;
}
ab[aboffset++] = (byte) ( i7 | 0x80 );
i7 = v & 0x7f;
if ( ( v >>>= 7 ) == 0 )
{
ab[aboffset++] = (byte) ( i7 );
return;
}
ab[aboffset++] = (byte) ( i7 | 0x80 );
i7 = v & 0x7f;
if ( ( v >>>= 7 ) == 0 )
{
ab[aboffset++] = (byte) ( i7 );
return;
}
ab[aboffset++] = (byte) ( i7 | 0x80 );
ab[aboffset++] = (byte) ( v );
return;
}
public int size()

121
brouter-util/src/main/java/btools/util/LruMap.java Normal file
View file

@ -0,0 +1,121 @@
package btools.util;
/**
* Something like LinkedHashMap, but purpose build, less dynamic and memory efficient
*
* @author ab
*/
public final class LruMap
{
private int hashbins;
private int maxsize;
private int size;
private LruMapNode lru;
private LruMapNode mru;
private LruMapNode[] binArray;
public LruMap( int bins, int size)
{
hashbins = bins;
maxsize = size;
binArray = new LruMapNode[hashbins];
}
public LruMapNode get( LruMapNode key )
{
int bin = ( key.hashCode() & 0xfffffff ) % hashbins;
LruMapNode e = binArray[bin];
while ( e != null )
{
if ( key.equals( e ) )
{
return e;
}
e = e.nextInBin;
}
return null;
}
// put e to the mru end of the queue
public void touch( LruMapNode e )
{
LruMapNode n = e.next;
LruMapNode p = e.previous;
if ( n == null )
{
return; // already at mru
}
n.previous = p;
if ( p != null )
{
p.next = n;
}
else
{
lru = n;
}
mru.next = e;
e.previous = mru;
e.next = null;
mru = e;
}
public LruMapNode removeLru()
{
if ( size < maxsize )
{
return null;
}
size--;
// unlink the lru from it's bin-queue
int bin = ( lru.hashCode() & 0xfffffff ) % hashbins;
LruMapNode e = binArray[bin];
if ( e == lru )
{
binArray[bin] = lru.nextInBin;
}
else
{
while( e != null )
{
LruMapNode prev = e;
e = e.nextInBin;
if ( e == lru )
{
prev.nextInBin = lru.nextInBin;
break;
}
}
}
LruMapNode res = lru;
lru = lru.next;
lru.previous = null;
return res;
}
public void put( LruMapNode val )
{
int bin = ( val.hashCode() & 0xfffffff ) % hashbins;
val.nextInBin = binArray[bin];
binArray[bin] = val;
val.previous = mru;
val.next = null;
if ( mru == null )
{
lru = val;
}
else
{
mru.next = val;
}
mru = val;
size++;
}
}

8
brouter-util/src/main/java/btools/util/LruMapNode.java Normal file
View file

@ -0,0 +1,8 @@
package btools.util;
public abstract class LruMapNode
{
LruMapNode nextInBin; // next entry for hash-bin
LruMapNode next; // next in lru sequence (towards mru)
LruMapNode previous; // previous in lru sequence (towards lru)
}