Fabio/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

direct weaving/escape-analysis

Browse source
This commit is contained in:
Arndt Brenschede 2019-06-23 18:57:46 +02:00
parent 661a09817a
commit 9f6878f891
9 changed files with 482 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
brouter-codec/src/main/java/btools/codec/MicroCache.java
View file

@ -40,6 +40,13 @@ public class MicroCache extends ByteDataWriter
super( ab );
}
public final static MicroCache emptyNonVirgin = new MicroCache( null );
static
{
emptyNonVirgin.virgin = false;
}
public static MicroCache emptyCache()
{
return new MicroCache( null ); // TODO: singleton?

129
brouter-core/src/main/java/btools/router/RoutingEngine.java
View file

@ -62,6 +62,8 @@ public class RoutingEngine extends Thread
private Object[] extract;
private boolean directWeaving = Boolean.getBoolean( "directWeaving" );
public RoutingEngine( String outfileBase, String logfileBase, String segmentDir,
List<OsmNodeNamed> waypoints, RoutingContext rc )
{
@ -575,7 +577,7 @@ public class RoutingEngine extends Thread
{
logInfo( "NodesCache status before reset=" + nodesCache.formatStatus() );
}
long maxmem = routingContext.memoryclass * 131072L; // 1/8 of total
long maxmem = routingContext.memoryclass * 1024L *1024L; // in MB
nodesCache = new NodesCache(segmentDir, routingContext.expctxWay, routingContext.forceSecondaryData, maxmem, nodesCache, detailed );
islandNodePairs.clearTempPairs();
@ -695,6 +697,7 @@ public class RoutingEngine extends Thread
{
boolean detailed = guideTrack != null;
resetCache( detailed );
nodesCache.nodesMap.cleanupMode = detailed ? 0 : ( routingContext.considerTurnRestrictions ? 2 : 1 );
return _findTrack( operationName, startWp, endWp, costCuttingTrack, refTrack, fastPartialRecalc );
}
finally
@ -708,7 +711,7 @@ public class RoutingEngine extends Thread
{
boolean verbose = guideTrack != null;
int maxTotalCost = 1000000000;
int maxTotalCost = guideTrack != null ? guideTrack.cost + 5000 : 1000000000;
int firstMatchCost = 1000000000;
logInfo( "findtrack with airDistanceCostFactor=" + airDistanceCostFactor );
@ -717,28 +720,37 @@ public class RoutingEngine extends Thread
matchPath = null;
int nodesVisited = 0;
long endNodeId1 = endWp == null ? -1L : endWp.node1.getIdFromPos();
long endNodeId2 = endWp == null ? -1L : endWp.node2.getIdFromPos();
long startNodeId1 = startWp.node1.getIdFromPos();
long startNodeId2 = startWp.node2.getIdFromPos();
OsmNodeNamed endPos = endWp == null ? null : new OsmNodeNamed( endWp.crosspoint );
long endNodeId1 = endWp == null ? -1L : endWp.node1.getIdFromPos();
long endNodeId2 = endWp == null ? -1L : endWp.node2.getIdFromPos();
OsmNode end1 = null;
OsmNode end2 = null;
OsmNodeNamed endPos = null;
boolean sameSegmentSearch = ( startNodeId1 == endNodeId1 && startNodeId2 == endNodeId2 )
|| ( startNodeId1 == endNodeId2 && startNodeId2 == endNodeId1 );
OsmNode start1 = nodesCache.getStartNode( startNodeId1 );
if ( start1 == null ) return null;
OsmNode start2 = null;
for( OsmLink link = start1.firstlink; link != null; link = link.getNext( start1 ) )
boolean sameSegmentSearch = false;
OsmNode start1 = nodesCache.getGraphNode( startWp.node1 );
OsmNode start2 = nodesCache.getGraphNode( startWp.node2 );
if ( endWp != null )
{
if ( link.getTarget( start1 ).getIdFromPos() == startNodeId2 )
{
start2 = link.getTarget( start1 );
break;
}
end1 = nodesCache.getGraphNode( endWp.node1 );
end2 = nodesCache.getGraphNode( endWp.node2 );
nodesCache.nodesMap.endNode1 = end1;
nodesCache.nodesMap.endNode2 = end2;
endPos = new OsmNodeNamed( endWp.crosspoint );
sameSegmentSearch = ( start1 == end1 && start2 == end2 ) || ( start1 == end2 && start2 == end1 );
}
if ( start2 == null ) return null;
if ( !nodesCache.obtainNonHollowNode( start1 ) )
{
return null;
}
nodesCache.expandHollowLinkTargets( start1 );
if ( !nodesCache.obtainNonHollowNode( start2 ) )
{
return null;
}
nodesCache.expandHollowLinkTargets( start2 );
routingContext.startDirectionValid = routingContext.forceUseStartDirection || fastPartialRecalc;
routingContext.startDirectionValid &= routingContext.startDirection != null && !routingContext.inverseDirection;
@ -770,6 +782,10 @@ public class RoutingEngine extends Thread
addToOpenset( startPath1 );
addToOpenset( startPath2 );
}
ArrayList<OsmPath> openBorderList = new ArrayList<OsmPath>();
boolean memoryPanicMode = false;
boolean needNonPanicProcessing = false;
for(;;)
{
if ( terminated )
@ -793,12 +809,72 @@ public class RoutingEngine extends Thread
{
path = openSet.popLowestKeyValue();
}
if ( path == null ) break;
if ( path == null )
{
if ( openBorderList.isEmpty() )
{
break;
}
for( OsmPath p : openBorderList )
{
openSet.add( p.cost + (int)(p.airdistance*airDistanceCostFactor), p );
}
openBorderList.clear();
memoryPanicMode = false;
needNonPanicProcessing = true;
continue;
}
if ( path.airdistance == -1 )
{
path.unregisterUpTree( routingContext );
continue;
}
if ( directWeaving && nodesCache.hasHollowLinkTargets( path.getTargetNode() ) )
{
if ( !memoryPanicMode )
{
if ( !nodesCache.nodesMap.isInMemoryBounds( openSet.getSize() ) )
{
int nodesBefore = nodesCache.nodesMap.nodesCreated;
int pathsBefore = openSet.getSize();
nodesCache.nodesMap.collectOutreachers();
for(;;)
{
OsmPath p3 = openSet.popLowestKeyValue();
if ( p3 == null ) break;
if ( p3.airdistance != -1 && nodesCache.nodesMap.canEscape( p3.getTargetNode() ) )
{
openBorderList.add( p3 );
}
}
for( OsmPath p : openBorderList )
{
openSet.add( p.cost + (int)(p.airdistance*airDistanceCostFactor), p );
}
openBorderList.clear();
System.out.println( "collected, nodes/paths before=" + nodesBefore + "/" + pathsBefore + " after=" + nodesCache.nodesMap.nodesCreated + "/" + openSet.getSize() );
if ( !nodesCache.nodesMap.isInMemoryBounds( openSet.getSize()) ) // TODO
{
if ( maxTotalCost < 1000000000 || needNonPanicProcessing )
{
throw new IllegalArgumentException( "memory limit reached" );
}
memoryPanicMode = true;
System.out.println( "************************ memory limit reached, enabled memory panic mode *************************" );
}
}
}
if ( memoryPanicMode )
{
openBorderList.add( path );
continue;
}
}
needNonPanicProcessing = false;
if ( fastPartialRecalc && matchPath != null && path.cost > 30L*firstMatchCost && !costCuttingTrack.isDirty )
{
@ -832,6 +908,12 @@ public class RoutingEngine extends Thread
OsmNode sourceNode = path.getSourceNode();
OsmNode currentNode = path.getTargetNode();
if ( currentLink.isLinkUnused() )
{
path.unregisterUpTree( routingContext );
continue;
}
long currentNodeId = currentNode.getIdFromPos();
long sourceNodeId = sourceNode.getIdFromPos();
@ -911,6 +993,10 @@ public class RoutingEngine extends Thread
continue;
}
nodesCache.nodesMap.currentMaxCost = maxTotalCost;
nodesCache.nodesMap.currentPathCost = path.cost;
nodesCache.nodesMap.destination = endPos;
routingContext.firstPrePath = null;
for( OsmLink link = currentNode.firstlink; link != null; link = link.getNext( currentNode) )
@ -1162,9 +1248,10 @@ public class RoutingEngine extends Thread
private OsmTrack mergeTrack( OsmPathElement match, OsmTrack oldTrack )
{
System.out.println( "**************** merging match=" + match.cost + " with oldTrack=" + oldTrack.cost );
OsmPathElement element = match;
OsmTrack track = new OsmTrack();
track.cost = oldTrack.cost;
while ( element != null )
{

29
brouter-mapaccess/src/main/java/btools/mapaccess/DirectWeaver.java
View file

@ -11,8 +11,8 @@ import btools.util.ByteDataWriter;
import btools.util.IByteArrayUnifier;
/**
* MicroCache2 is the new format that uses statistical encoding and
* is able to do access filtering and waypoint matching during encoding
* DirectWeaver does the same decoding as MicroCache2, but decodes directly
* into the instance-graph, not into the intermediate nodes-cache
*/
public final class DirectWeaver extends ByteDataWriter
{
@ -20,8 +20,7 @@ public final class DirectWeaver extends ByteDataWriter
private int latBase;
private int cellsize;
protected int[] faid;
protected int size = 0;
private int size = 0;
private static boolean debug = false;
@ -43,8 +42,9 @@ public final class DirectWeaver extends ByteDataWriter
NoisyDiffCoder transEleDiff = new NoisyDiffCoder( bc );
size = bc.decodeNoisyNumber( 5 );
faid = size > dataBuffers.ibuf2.length ? new int[size] : dataBuffers.ibuf2;
int[] faid = size > dataBuffers.ibuf2.length ? new int[size] : dataBuffers.ibuf2;
if ( debug ) System.out.println( "*** decoding cache of size=" + size + " for lonIdx=" + lonIdx + " latIdx=" + latIdx );
bc.decodeSortedArray( faid, 0, size, 0x20000000, 0 );
@ -59,9 +59,11 @@ public final class DirectWeaver extends ByteDataWriter
if ( node == null )
{
node = new OsmNode( ilon, ilat );
node.visitID = 0;
}
else
{
node.visitID = 1;
hollowNodes.remove( node );
}
nodes[n] = node;
@ -189,7 +191,11 @@ public final class DirectWeaver extends ByteDataWriter
if ( nodeIdx != n ) // valid internal (forward-) link
{
OsmNode node2 = nodes[nodeIdx];
OsmLink link = node.isLinkUnused() ? node : ( node2.isLinkUnused() ? node2 : new OsmLink() );
OsmLink link = node.isLinkUnused() ? node : ( node2.isLinkUnused() ? node2 : null );
if ( link == null )
{
link = new OsmLink();
}
link.descriptionBitmap = wayTags.data;
link.geometry = geometry;
node.addLink( link, isReverse, node2 );
@ -197,14 +203,13 @@ public final class DirectWeaver extends ByteDataWriter
else // weave external link
{
node.addLink( linklon, linklat, wayTags.data, geometry, hollowNodes, isReverse );
node.visitID = 1;
}
}
} // ... loop over links
} // ... loop over nodes
}
}
hollowNodes.cleanupAndCount( nodes );
}
public long expandId( int id32 )

39
brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java
View file

@ -20,7 +20,7 @@ public final class NodesCache
private File segmentDir;
private File secondarySegmentsDir = null;
private OsmNodesMap nodesMap;
public OsmNodesMap nodesMap;
private BExpressionContextWay expCtxWay;
private int lookupVersion;
private int lookupMinorVersion;
@ -38,7 +38,7 @@ public final class NodesCache
public String first_file_access_name;
private long cacheSum = 0;
private long maxmem;
private long maxmemtiles;
private boolean detailed;
private boolean garbageCollectionEnabled = false;
@ -58,13 +58,14 @@ public final class NodesCache
public NodesCache( String segmentDir, BExpressionContextWay ctxWay, boolean forceSecondaryData, long maxmem, NodesCache oldCache, boolean detailed )
{
this.maxmemtiles = maxmem / 8;
this.segmentDir = new File( segmentDir );
this.nodesMap = new OsmNodesMap();
this.nodesMap.maxmem = (2L*maxmem) / 3L;
this.expCtxWay = ctxWay;
this.lookupVersion = ctxWay.meta.lookupVersion;
this.lookupMinorVersion = ctxWay.meta.lookupMinorVersion;
this.forceSecondaryData = forceSecondaryData;
this.maxmem = maxmem;
this.detailed = detailed;
if ( ctxWay != null )
@ -130,7 +131,7 @@ public final class NodesCache
// clean all ghosts and enable garbage collection
private void checkEnableCacheCleaning()
{
if ( cacheSum < maxmem )
if ( cacheSum < maxmemtiles )
{
return;
}
@ -158,7 +159,7 @@ public final class NodesCache
if ( garbageCollectionEnabled )
{
ghostCleaningDone = true;
maxmem *= 2;
maxmemtiles *= 2;
}
else
{
@ -282,6 +283,21 @@ public final class NodesCache
}
}
/**
* make sure all link targets of the given node are non-hollow
*/
public boolean hasHollowLinkTargets( OsmNode n )
{
for( OsmLink link = n.firstlink; link != null; link = link.getNext( n ) )
{
if ( link.getTarget( n ).isHollow() )
{
return true;
}
}
return false;
}
/**
* get a node for the given id with all link-targets also non-hollow
*
@ -307,6 +323,19 @@ public final class NodesCache
return n;
}
public OsmNode getGraphNode( OsmNode template )
{
OsmNode graphNode = new OsmNode( template.ilon, template.ilat );
graphNode.setHollow();
OsmNode existing = nodesMap.put( graphNode );
if ( existing == null )
{
return graphNode;
}
nodesMap.put( existing );
return existing;
}
public void matchWaypointsToNodes( List<MatchedWaypoint> unmatchedWaypoints, double maxDistance, OsmNodePairSet islandNodePairs )
{
waypointMatcher = new WaypointMatcherImpl( unmatchedWaypoints, 250., islandNodePairs );

4
brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java
View file

@ -164,9 +164,7 @@ final class OsmFile
return new MicroCache2( dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher );
}
new DirectWeaver( dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher, hollowNodes );
MicroCache dummy = MicroCache.emptyCache();
dummy.virgin = false;
return dummy;
return MicroCache.emptyNonVirgin;
}
throw new IOException( "checkum error" );
}

30
brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java
View file

@ -34,6 +34,8 @@ public class OsmNode extends OsmLink implements OsmPos
public byte[] nodeDescription;
public TurnRestriction firstRestriction;
public int visitID;
public void addTurnRestriction( TurnRestriction tr )
{
@ -234,6 +236,28 @@ public class OsmNode extends OsmLink implements OsmPos
{
return ( (long) ilon ) << 32 | ilat;
}
public void vanish()
{
if ( !isHollow() )
{
OsmLink l = firstlink;
while( l != null )
{
OsmNode target = l.getTarget( this );
OsmLink nextLink = l.getNext( this );
if ( !target.isHollow() )
{
unlinkLink( l );
if ( !l.isLinkUnused() )
{
target.unlinkLink( l );
}
}
l = nextLink;
}
}
}
public final void unlinkLink( OsmLink link )
{
@ -258,7 +282,7 @@ public class OsmNode extends OsmLink implements OsmPos
}
l = nl;
}
else
else if ( l.n2 != this && l.n2 != null )
{
OsmLink nl = l.next;
if ( nl == link )
@ -268,6 +292,10 @@ public class OsmNode extends OsmLink implements OsmPos
}
l = nl;
}
else
{
throw new IllegalArgumentException( "unlinkLink: unknown source" );
}
}
}

5
brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodePairSet.java
View file

@ -96,6 +96,11 @@ public class OsmNodePairSet
return tempNodes;
}
public int getMaxTmpNodes()
{
return maxTempNodes;
}
public int getFreezeCount()
{
return freezecount;

279
brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodesMap.java
View file

@ -5,9 +5,11 @@
*/
package btools.mapaccess;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import btools.util.ByteArrayUnifier;
import btools.util.SortedHeap;
public final class OsmNodesMap
{
@ -16,6 +18,222 @@ public final class OsmNodesMap
private ByteArrayUnifier abUnifier = new ByteArrayUnifier( 16384, false );
private OsmNode testKey = new OsmNode();
public int nodesCreated;
public long maxmem;
public int lastVisitID = 1000;
public int baseID = 1000;
public OsmNode destination;
public int currentPathCost;
public int currentMaxCost = 1000000000;
public OsmNode endNode1;
public OsmNode endNode2;
public int cleanupMode = 0;
public void cleanupAndCount( OsmNode[] nodes )
{
if ( cleanupMode == 0 )
{
justCount( nodes );
}
else
{
cleanupPeninsulas( nodes );
}
}
private void justCount( OsmNode[] nodes )
{
for( int i=0; i<nodes.length; i++ )
{
OsmNode n = nodes[i];
if ( n.firstlink != null )
{
nodesCreated++;
}
}
}
private void cleanupPeninsulas( OsmNode[] nodes )
{
baseID = lastVisitID++;
for( int i=0; i<nodes.length; i++ ) // loop over nodes again just for housekeeping
{
OsmNode n = nodes[i];
if ( n.firstlink != null )
{
if ( n.visitID == 1 )
{
try
{
minVisitIdInSubtree( null, n );
}
catch( StackOverflowError soe )
{
System.out.println( "+++++++++++++++ StackOverflowError ++++++++++++++++" );
}
}
}
}
}
private int minVisitIdInSubtree( OsmNode source, OsmNode n )
{
if ( n.visitID == 1 ) n.visitID = baseID; // border node
else n.visitID = lastVisitID++;
int minId = n.visitID;
nodesCreated++;
OsmLink nextLink = null;
for( OsmLink l = n.firstlink; l != null; l = nextLink )
{
nextLink = l.getNext( n );
OsmNode t = l.getTarget( n );
if ( t == source ) continue;
if ( t.isHollow() ) continue;
int minIdSub = t.visitID;
if ( minIdSub == 1 )
{
minIdSub = baseID;
}
else if ( minIdSub == 0 )
{
int nodesCreatedUntilHere = nodesCreated;
minIdSub = minVisitIdInSubtree( n, t);
if ( minIdSub > n.visitID ) // peninsula ?
{
nodesCreated = nodesCreatedUntilHere;
n.unlinkLink( l );
t.unlinkLink( l );
}
}
else if ( minIdSub < baseID )
{
continue;
}
else if ( cleanupMode == 2 )
{
minIdSub = baseID; // in tree-mode, hitting anything is like a gateway
}
if ( minIdSub < minId ) minId = minIdSub;
}
return minId;
}
public boolean isInMemoryBounds( int npaths )
{
// long total = nodesCreated * 76L + linksCreated * 48L;
long total = nodesCreated * 95L + npaths * 200L;
return total <= maxmem;
}
private void addActiveNode( ArrayList<OsmNode> nodes2check, OsmNode n )
{
n.visitID = lastVisitID;
nodesCreated++;
nodes2check.add( n );
}
// is there an escape from this node
// to a hollow node (or destination node) ?
public boolean canEscape( OsmNode n0 )
{
boolean sawLowIDs = false;
lastVisitID++;
ArrayList<OsmNode> nodes2check = new ArrayList<OsmNode>();
nodes2check.add( n0 );
while ( !nodes2check.isEmpty() )
{
OsmNode n = nodes2check.remove( nodes2check.size()-1 );
if ( n.visitID < baseID )
{
n.visitID = lastVisitID;
nodesCreated++;
for( OsmLink l = n.firstlink; l != null; l = l.getNext( n ) )
{
OsmNode t = l.getTarget( n );
nodes2check.add( t );
}
}
else if ( n.visitID < lastVisitID )
{
sawLowIDs = true;
}
}
if ( sawLowIDs )
{
return true;
}
nodes2check.add( n0 );
while ( !nodes2check.isEmpty() )
{
OsmNode n = nodes2check.remove( nodes2check.size()-1 );
if ( n.visitID == lastVisitID )
{
n.visitID = lastVisitID;
nodesCreated--;
for( OsmLink l = n.firstlink; l != null; l = l.getNext( n ) )
{
OsmNode t = l.getTarget( n );
nodes2check.add( t );
}
n.vanish();
}
}
return false;
}
public void collectOutreachers()
{
nodesCreated=0;
System.out.println( "collectOutreachers, currentMaxCost=" + currentMaxCost );
ArrayList<OsmNode> nodes2check = new ArrayList<OsmNode>();
for( OsmNode n : hmap.values() )
{
addActiveNode( nodes2check, n );
}
lastVisitID++;
baseID = lastVisitID;
while ( !nodes2check.isEmpty() )
{
OsmNode n = nodes2check.remove( nodes2check.size()-1 );
n.visitID = lastVisitID;
for( OsmLink l = n.firstlink; l != null; l = l.getNext( n ) )
{
OsmNode t = l.getTarget( n );
if ( t.visitID != lastVisitID )
{
addActiveNode( nodes2check, t );
}
}
if ( destination != null && currentMaxCost < 1000000000 )
{
int distance = n.calcDistance( destination );
if ( distance > currentMaxCost - currentPathCost + 100 )
{
n.vanish();
}
}
if ( n.firstlink == null )
{
nodesCreated--;
}
}
}
public ByteArrayUnifier getByteArrayUnifier()
{
@ -36,7 +254,10 @@ public final class OsmNodesMap
public void remove( OsmNode node )
{
hmap.remove( node );
if ( node != endNode1 && node != endNode2 ) // keep endnodes in hollow-map even when loaded
{ // (needed for escape analysis)
hmap.remove( node );
}
}
/**
@ -48,4 +269,58 @@ public final class OsmNodesMap
return hmap.put( node, node );
}
// ********************** test cleanup **********************
private static void addLinks( OsmNode[] nodes, int idx, boolean isBorder, int[] links )
{
OsmNode n = nodes[idx];
n.visitID = isBorder ? 1 : 0;
n.selev = (short)idx;
for( int i : links )
{
OsmNode t = nodes[i];
OsmLink link = n.isLinkUnused() ? n : ( t.isLinkUnused() ? t : null );
if ( link == null )
{
link = new OsmLink();
}
n.addLink( link, false, t );
}
}
public static void main( String[] args )
{
OsmNode[] nodes = new OsmNode[12];
for( int i=0; i<nodes.length; i++ )
{
nodes[i]= new OsmNode( (i+1000)*1000,(i+1000)*1000 );
}
addLinks( nodes, 0, true , new int[]{1,5} ); // 0
addLinks( nodes, 1, true , new int[]{} ); // 1
addLinks( nodes, 2, false, new int[]{3,4} ); // 2
addLinks( nodes, 3, false, new int[]{4} ); // 3
addLinks( nodes, 4, false, new int[]{} ); // 4
addLinks( nodes, 5, true , new int[]{6,9} ); // 5
addLinks( nodes, 6, false, new int[]{7,8} ); // 6
addLinks( nodes, 7, false, new int[]{} ); // 7
addLinks( nodes, 8, false, new int[]{} ); // 8
addLinks( nodes, 9, false, new int[]{10,11} ); // 9
addLinks( nodes, 10, false, new int[]{11} ); // 10
addLinks( nodes, 11, false, new int[]{} ); // 11
OsmNodesMap nm = new OsmNodesMap();
nm.cleanupMode = 2;
nm.cleanupAndCount( nodes );
System.out.println( "nodesCreated=" + nm.nodesCreated );
nm.cleanupAndCount( nodes );
System.out.println( "nodesCreated=" + nm.nodesCreated );
}
}

5
brouter-server/src/main/java/btools/server/BRouter.java
View file

@ -126,8 +126,11 @@ public class BRouter
else
{
wplist.add( readPosition( args, 3, "to" ) );
re = new RoutingEngine( "mytrack", "mylog", args[0], wplist, readRoutingContext(args) );
RoutingContext rc = readRoutingContext(args);
rc.memoryclass = 16;
re = new RoutingEngine( "mytrack", "mylog", args[0], wplist, rc );
re.doRun( 0 );
}
if ( re.getErrorMessage() != null )
{