Fabio/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

performance

Browse source
This commit is contained in:
Arndt 2016-09-10 16:03:15 +02:00
parent 9d00b0181e
commit 448bb11ad4
13 changed files with 204 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
brouter-codec/src/main/java/btools/codec/DataBuffers.java
View file

@ -1,5 +1,7 @@
package btools.codec;
import btools.util.BitCoderContext;
/**
* Container for some re-usable databuffers for the decoder
*/
@ -7,6 +9,7 @@ public final class DataBuffers
{
public byte[] iobuffer;
public byte[] tagbuf1 = new byte[256];
public BitCoderContext bctx1 = new BitCoderContext( tagbuf1 );
public byte[] bbuf1 = new byte[65636];
public int[] ibuf1 = new int[4096];
public int[] ibuf2 = new int[2048];

80
brouter-codec/src/main/java/btools/codec/MicroCache.java
View file

@ -135,47 +135,51 @@ public class MicroCache extends ByteDataWriter
return n > 0 ? fapos[n - 1] & 0x7fffffff : 0;
}
public final void collect( int threshold )
public final int collect( int threshold )
{
if ( delcount > threshold )
if ( delcount <= threshold )
{
virgin = false;
int nsize = size - delcount;
if ( nsize == 0 )
{
faid = null;
fapos = null;
}
else
{
int[] nfaid = new int[nsize];
int[] nfapos = new int[nsize];
int idx = 0;
byte[] nab = new byte[ab.length - delbytes];
int nab_off = 0;
for ( int i = 0; i < size; i++ )
{
int pos = fapos[i];
if ( ( pos & 0x80000000 ) == 0 )
{
int start = startPos( i );
int end = fapos[i];
int len = end - start;
System.arraycopy( ab, start, nab, nab_off, len );
nfaid[idx] = faid[i];
nab_off += len;
nfapos[idx] = nab_off;
idx++;
}
}
faid = nfaid;
fapos = nfapos;
ab = nab;
}
init( nsize );
return 0;
}
virgin = false;
int nsize = size - delcount;
if ( nsize == 0 )
{
faid = null;
fapos = null;
}
else
{
int[] nfaid = new int[nsize];
int[] nfapos = new int[nsize];
int idx = 0;
byte[] nab = new byte[ab.length - delbytes];
int nab_off = 0;
for ( int i = 0; i < size; i++ )
{
int pos = fapos[i];
if ( ( pos & 0x80000000 ) == 0 )
{
int start = startPos( i );
int end = fapos[i];
int len = end - start;
System.arraycopy( ab, start, nab, nab_off, len );
nfaid[idx] = faid[i];
nab_off += len;
nfapos[idx] = nab_off;
idx++;
}
}
faid = nfaid;
fapos = nfapos;
ab = nab;
}
int deleted = delbytes;
init( nsize );
return deleted;
}
public final void unGhost()

4
brouter-codec/src/main/java/btools/codec/MicroCache2.java
View file

@ -44,8 +44,8 @@ public final class MicroCache2 extends MicroCache
StatCoderContext bc = new StatCoderContext( dataBuffers.iobuffer );
TagValueCoder wayTagCoder = new TagValueCoder( bc, dataBuffers.tagbuf1, wayValidator );
TagValueCoder nodeTagCoder = new TagValueCoder( bc, dataBuffers.tagbuf1, null );
TagValueCoder wayTagCoder = new TagValueCoder( bc, dataBuffers, wayValidator );
TagValueCoder nodeTagCoder = new TagValueCoder( bc, dataBuffers, null );
NoisyDiffCoder nodeIdxDiff = new NoisyDiffCoder( bc );
NoisyDiffCoder nodeEleDiff = new NoisyDiffCoder( bc );
NoisyDiffCoder extLonDiff = new NoisyDiffCoder(bc);

14
brouter-codec/src/main/java/btools/codec/TagValueCoder.java
View file

@ -78,9 +78,9 @@ public final class TagValueCoder
this.bc = bc;
}
public TagValueCoder( BitCoderContext bc, byte[] buffer, TagValueValidator validator )
public TagValueCoder( BitCoderContext bc, DataBuffers buffers, TagValueValidator validator )
{
tree = decodeTree( bc, buffer, validator );
tree = decodeTree( bc, buffers, validator );
this.bc = bc;
}
@ -89,18 +89,20 @@ public final class TagValueCoder
identityMap = new HashMap<TagValueSet, TagValueSet>();
}
private Object decodeTree( BitCoderContext bc, byte[] buffer, TagValueValidator validator )
private Object decodeTree( BitCoderContext bc, DataBuffers buffers, TagValueValidator validator )
{
boolean isNode = bc.decodeBit();
if ( isNode )
{
TreeNode node = new TreeNode();
node.child1 = decodeTree( bc, buffer, validator );
node.child2 = decodeTree( bc, buffer, validator );
node.child1 = decodeTree( bc, buffers, validator );
node.child2 = decodeTree( bc, buffers, validator );
return node;
}
BitCoderContext ctx = new BitCoderContext( buffer );
byte[] buffer = buffers.tagbuf1;
BitCoderContext ctx = buffers.bctx1;
ctx.reset( buffer );
int inum = 0;
int lastEncodedInum = 0;

6
brouter-core/src/main/java/btools/router/OsmPath.java
View file

@ -160,7 +160,11 @@ final class OsmPath implements OsmLinkHolder
cost += iicost;
}
OsmTransferNode transferNode = link.decodeFirsttransfer( p1 );
// OsmTransferNode transferNode = link.decodeGeometry( p1, rc.byteDataReaderGeometry, rc.transferNodeCache );
OsmTransferNode transferNode = link.geometry == null ? null
: rc.geometryDecoder.decodeGeometry( link.geometry, p1, link.targetNode, link.counterLinkWritten );
OsmNode targetNode = link.targetNode;
for(;;)
{

3
brouter-core/src/main/java/btools/router/RoutingContext.java
View file

@ -12,6 +12,7 @@ import java.util.List;
import btools.expressions.BExpressionContext;
import btools.expressions.BExpressionContextNode;
import btools.expressions.BExpressionContextWay;
import btools.mapaccess.GeometryDecoder;
public final class RoutingContext
{
@ -40,6 +41,8 @@ public final class RoutingContext
public BExpressionContextWay expctxWay;
public BExpressionContextNode expctxNode;
public GeometryDecoder geometryDecoder = new GeometryDecoder();
public int memoryclass = 64;

17
brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java
View file

@ -44,6 +44,8 @@ public abstract class BExpressionContext implements IByteArrayUnifier
private int[] lookupData = new int[0];
private byte[] abBuf = new byte[256];
private BitCoderContext ctxEndode = new BitCoderContext( abBuf );
private BitCoderContext ctxDecode = new BitCoderContext( new byte[0] );
private Map<String,Integer> variableNumbers = new HashMap<String,Integer>();
@ -121,11 +123,13 @@ public abstract class BExpressionContext implements IByteArrayUnifier
public byte[] encode( int[] ld )
{
// start with first bit hardwired ("reversedirection")
BitCoderContext ctx = new BitCoderContext( abBuf );
BitCoderContext ctx = ctxEndode;
ctx.reset();
int skippedTags = 0;
int nonNullTags= 0;
int skippedTags = 0;
int nonNullTags= 0;
// (skip first bit ("reversedirection") )
// all others are generic
for( int inum = 1; inum < lookupValues.size(); inum++ ) // loop over lookup names
@ -174,13 +178,16 @@ public abstract class BExpressionContext implements IByteArrayUnifier
decode( lookupData, false, ab );
lookupDataValid = true;
}
/**
* decode a byte-array into a lookup data array
*/
private void decode( int[] ld, boolean inverseDirection, byte[] ab )
{
BitCoderContext ctx = new BitCoderContext(ab);
BitCoderContext ctx = ctxDecode;
ctx.reset( ab );
// start with first bit hardwired ("reversedirection")
ld[0] = inverseDirection ? 2 : 0;

69
brouter-mapaccess/src/main/java/btools/mapaccess/GeometryDecoder.java Normal file
View file

@ -0,0 +1,69 @@
/**
* Container for link between two Osm nodes
*
* @author ab
*/
package btools.mapaccess;
import btools.util.ByteDataReader;
public final class GeometryDecoder
{
private ByteDataReader r = new ByteDataReader( null );
private OsmTransferNode[] cachedNodes;
private int nCachedNodes = 128;
public GeometryDecoder()
{
// create some caches
cachedNodes = new OsmTransferNode[nCachedNodes];
for( int i=0; i<nCachedNodes; i++ )
{
cachedNodes[i] = new OsmTransferNode();
}
}
public OsmTransferNode decodeGeometry( byte[] geometry, OsmNode sourceNode, OsmNode targetNode, boolean reverseLink )
{
OsmTransferNode firstTransferNode = null;
OsmTransferNode lastTransferNode = null;
OsmNode startnode = reverseLink ? targetNode : sourceNode;
r.reset( geometry );
int olon = startnode.ilon;
int olat = startnode.ilat;
int oselev = startnode.selev;
int idx = 0;
while ( r.hasMoreData() )
{
OsmTransferNode trans = idx < nCachedNodes ? cachedNodes[idx++] : new OsmTransferNode();
trans.ilon = olon + r.readVarLengthSigned();
trans.ilat = olat + r.readVarLengthSigned();
trans.selev = (short)(oselev + r.readVarLengthSigned());
olon = trans.ilon;
olat = trans.ilat;
oselev = trans.selev;
if ( reverseLink ) // reverse chaining
{
trans.next = firstTransferNode;
firstTransferNode = trans;
}
else
{
trans.next = null;
if ( lastTransferNode == null )
{
firstTransferNode = trans;
}
else
{
lastTransferNode.next = trans;
}
lastTransferNode = trans;
}
}
return firstTransferNode;
}
}

40
brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java
View file

@ -7,9 +7,7 @@ package btools.mapaccess;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import btools.codec.DataBuffers;
import btools.codec.MicroCache;
@ -43,10 +41,15 @@ public final class NodesCache
private boolean garbageCollectionEnabled = false;
private boolean ghostCleaningDone = false;
private long cacheSumClean = 0;
private long ghostSum = 0;
private long ghostWakeup = 0;
public String formatStatus()
{
return "collecting=" + garbageCollectionEnabled + " noGhosts=" + ghostCleaningDone + " cacheSum=" + cacheSum;
return "collecting=" + garbageCollectionEnabled + " noGhosts=" + ghostCleaningDone + " cacheSum=" + cacheSum + " cacheSumClean=" + cacheSumClean + " ghostSum=" + ghostSum + " ghostWakeup=" + ghostWakeup ;
}
public NodesCache( String segmentDir, OsmNodesMap nodesMap, BExpressionContextWay ctxWay, boolean forceSecondaryData, long maxmem, NodesCache oldCache )
@ -90,6 +93,7 @@ public final class NodesCache
dataBuffers = new DataBuffers();
secondarySegmentsDir = StorageConfigHelper.getSecondarySegmentDir( segmentDir );
}
ghostSum = cacheSum;
}
public void cleanNonVirgin()
@ -110,39 +114,28 @@ public final class NodesCache
private void checkEnableCacheCleaning()
{
if ( cacheSum < maxmem || ghostCleaningDone )
{
return;
}
for ( int i = 0; i < fileRows.length; i++ )
{
OsmFile[] fileRow = fileRows[i];
if ( fileRow == null )
{
continue;
int nghosts = 0;
}
for ( OsmFile osmf : fileRow )
{
if ( garbageCollectionEnabled )
{
if ( osmf.ghost )
{
nghosts++;
}
cacheSum -= osmf.cleanGhosts();
}
else
{
osmf.cleanAll();
cacheSum -= osmf.collectAll();
}
}
if ( nghosts == 0 )
continue;
int j = 0;
OsmFile[] frow = new OsmFile[fileRow.length - nghosts];
for ( OsmFile osmf : fileRow )
{
if ( osmf.ghost )
continue;
frow[j++] = osmf;
}
fileRows[i] = frow;
}
if ( garbageCollectionEnabled )
@ -150,7 +143,8 @@ public final class NodesCache
ghostCleaningDone = true;
}
else
{
{
cacheSumClean = cacheSum;
garbageCollectionEnabled = true;
}
}
@ -189,7 +183,6 @@ public final class NodesCache
newFileRow[ndegrees] = osmf;
fileRows[latDegree] = newFileRow;
}
osmf.ghost = false;
currentFileName = osmf.filename;
if ( !osmf.hasData() )
@ -208,6 +201,7 @@ public final class NodesCache
else if ( segment.ghost )
{
segment.unGhost();
ghostWakeup += segment.getDataSize();
}
return segment;
}
@ -240,7 +234,7 @@ public final class NodesCache
if ( garbageCollectionEnabled ) // garbage collection
{
segment.collect( segment.getSize() >> 2 ); // threshold = 1/4 of size is deleted
cacheSum -= segment.collect( segment.getSize() >> 1 ); // threshold = 1/2 of size is deleted
}
return !node.isHollow();

28
brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java
View file

@ -29,8 +29,6 @@ final class OsmFile
public String filename;
public boolean ghost = false;
private int divisor;
private int cellsize;
private int ncaches;
@ -162,7 +160,6 @@ final class OsmFile
long setGhostState()
{
long sum = 0;
ghost = true;
int nc = microCaches == null ? 0 : microCaches.length;
for ( int i = 0; i < nc; i++ )
{
@ -182,8 +179,26 @@ final class OsmFile
return sum;
}
void cleanAll()
long collectAll()
{
long deleted = 0;
int nc = microCaches == null ? 0 : microCaches.length;
for ( int i = 0; i < nc; i++ )
{
MicroCache mc = microCaches[i];
if ( mc == null )
continue;
if ( !mc.ghost )
{
deleted += mc.collect( 0 );
}
}
return deleted;
}
long cleanGhosts()
{
long deleted = 0;
int nc = microCaches == null ? 0 : microCaches.length;
for ( int i = 0; i < nc; i++ )
{
@ -194,11 +209,8 @@ final class OsmFile
{
microCaches[i] = null;
}
else
{
mc.collect( 0 );
}
}
return deleted;
}
void cleanNonVirgin()

42
brouter-mapaccess/src/main/java/btools/mapaccess/OsmLink.java
View file

@ -5,7 +5,6 @@
*/
package btools.mapaccess;
import btools.util.ByteDataReader;
public class OsmLink
@ -33,47 +32,6 @@ public class OsmLink
this.geometry = geometry;
}
final public OsmTransferNode decodeFirsttransfer( OsmNode sourceNode )
{
if ( geometry == null ) return null;
OsmTransferNode firstTransferNode = null;
OsmTransferNode lastTransferNode = null;
OsmNode startnode = counterLinkWritten ? targetNode : sourceNode;
ByteDataReader r = new ByteDataReader( geometry );
int olon = startnode.ilon;
int olat = startnode.ilat;
int oselev = startnode.selev;
while ( r.hasMoreData() )
{
OsmTransferNode trans = new OsmTransferNode();
trans.ilon = olon + r.readVarLengthSigned();
trans.ilat = olat + r.readVarLengthSigned();
trans.selev = (short)(oselev + r.readVarLengthSigned());
olon = trans.ilon;
olat = trans.ilat;
oselev = trans.selev;
if ( counterLinkWritten ) // reverse chaining
{
trans.next = firstTransferNode;
firstTransferNode = trans;
}
else
{
if ( lastTransferNode == null )
{
firstTransferNode = trans;
}
else
{
lastTransferNode.next = trans;
}
lastTransferNode = trans;
}
}
return firstTransferNode;
}
final public void addLinkHolder( OsmLinkHolder holder )
{
if ( firstlinkholder != null ) { holder.setNextForLink( firstlinkholder ); }

15
brouter-util/src/main/java/btools/util/BitCoderContext.java
View file

@ -15,6 +15,21 @@ public class BitCoderContext
this.ab = ab;
idxMax = ab.length-1;
}
public final void reset( byte[] ab )
{
this.ab = ab;
idxMax = ab.length-1;
reset();
}
public final void reset()
{
idx = -1;
bm = 0x100;
bits = 0;
b = 0;
}
/**
* encode a distance with a variable bit length

8
brouter-util/src/main/java/btools/util/ByteDataReader.java
View file

@ -24,6 +24,14 @@ public class ByteDataReader
aboffset = offset;
aboffsetEnd = ab == null ? 0 : ab.length;
}
public final void reset( byte[] byteArray )
{
ab = byteArray;
aboffset = 0;
aboffsetEnd = ab == null ? 0 : ab.length;
}
public final int readInt()
{