From 9f6878f891e6c1febc9987353df13078926e8677 Mon Sep 17 00:00:00 2001 From: Arndt Brenschede Date: Sun, 23 Jun 2019 18:57:46 +0200 Subject: [PATCH] direct weaving/escape-analysis --- .../main/java/btools/codec/MicroCache.java | 7 + .../java/btools/router/RoutingEngine.java | 129 ++++++-- .../java/btools/mapaccess/DirectWeaver.java | 29 +- .../java/btools/mapaccess/NodesCache.java | 39 ++- .../main/java/btools/mapaccess/OsmFile.java | 4 +- .../main/java/btools/mapaccess/OsmNode.java | 30 +- .../java/btools/mapaccess/OsmNodePairSet.java | 5 + .../java/btools/mapaccess/OsmNodesMap.java | 279 +++++++++++++++++- .../src/main/java/btools/server/BRouter.java | 5 +- 9 files changed, 482 insertions(+), 45 deletions(-) diff --git a/brouter-codec/src/main/java/btools/codec/MicroCache.java b/brouter-codec/src/main/java/btools/codec/MicroCache.java index b95f772..2cb4a8c 100644 --- a/brouter-codec/src/main/java/btools/codec/MicroCache.java +++ b/brouter-codec/src/main/java/btools/codec/MicroCache.java @@ -40,6 +40,13 @@ public class MicroCache extends ByteDataWriter super( ab ); } + public final static MicroCache emptyNonVirgin = new MicroCache( null ); + + static + { + emptyNonVirgin.virgin = false; + } + public static MicroCache emptyCache() { return new MicroCache( null ); // TODO: singleton? diff --git a/brouter-core/src/main/java/btools/router/RoutingEngine.java b/brouter-core/src/main/java/btools/router/RoutingEngine.java index a9b6445..67bfb77 100644 --- a/brouter-core/src/main/java/btools/router/RoutingEngine.java +++ b/brouter-core/src/main/java/btools/router/RoutingEngine.java @@ -62,6 +62,8 @@ public class RoutingEngine extends Thread private Object[] extract; + private boolean directWeaving = Boolean.getBoolean( "directWeaving" ); + public RoutingEngine( String outfileBase, String logfileBase, String segmentDir, List waypoints, RoutingContext rc ) { @@ -575,7 +577,7 @@ public class RoutingEngine extends Thread { logInfo( "NodesCache status before reset=" + nodesCache.formatStatus() ); } - long maxmem = routingContext.memoryclass * 131072L; // 1/8 of total + long maxmem = routingContext.memoryclass * 1024L *1024L; // in MB nodesCache = new NodesCache(segmentDir, routingContext.expctxWay, routingContext.forceSecondaryData, maxmem, nodesCache, detailed ); islandNodePairs.clearTempPairs(); @@ -695,6 +697,7 @@ public class RoutingEngine extends Thread { boolean detailed = guideTrack != null; resetCache( detailed ); + nodesCache.nodesMap.cleanupMode = detailed ? 0 : ( routingContext.considerTurnRestrictions ? 2 : 1 ); return _findTrack( operationName, startWp, endWp, costCuttingTrack, refTrack, fastPartialRecalc ); } finally @@ -708,7 +711,7 @@ public class RoutingEngine extends Thread { boolean verbose = guideTrack != null; - int maxTotalCost = 1000000000; + int maxTotalCost = guideTrack != null ? guideTrack.cost + 5000 : 1000000000; int firstMatchCost = 1000000000; logInfo( "findtrack with airDistanceCostFactor=" + airDistanceCostFactor ); @@ -717,28 +720,37 @@ public class RoutingEngine extends Thread matchPath = null; int nodesVisited = 0; - long endNodeId1 = endWp == null ? -1L : endWp.node1.getIdFromPos(); - long endNodeId2 = endWp == null ? -1L : endWp.node2.getIdFromPos(); long startNodeId1 = startWp.node1.getIdFromPos(); long startNodeId2 = startWp.node2.getIdFromPos(); - - OsmNodeNamed endPos = endWp == null ? null : new OsmNodeNamed( endWp.crosspoint ); + long endNodeId1 = endWp == null ? -1L : endWp.node1.getIdFromPos(); + long endNodeId2 = endWp == null ? -1L : endWp.node2.getIdFromPos(); + OsmNode end1 = null; + OsmNode end2 = null; + OsmNodeNamed endPos = null; - boolean sameSegmentSearch = ( startNodeId1 == endNodeId1 && startNodeId2 == endNodeId2 ) - || ( startNodeId1 == endNodeId2 && startNodeId2 == endNodeId1 ); - - OsmNode start1 = nodesCache.getStartNode( startNodeId1 ); - if ( start1 == null ) return null; - OsmNode start2 = null; - for( OsmLink link = start1.firstlink; link != null; link = link.getNext( start1 ) ) + boolean sameSegmentSearch = false; + OsmNode start1 = nodesCache.getGraphNode( startWp.node1 ); + OsmNode start2 = nodesCache.getGraphNode( startWp.node2 ); + if ( endWp != null ) { - if ( link.getTarget( start1 ).getIdFromPos() == startNodeId2 ) - { - start2 = link.getTarget( start1 ); - break; - } + end1 = nodesCache.getGraphNode( endWp.node1 ); + end2 = nodesCache.getGraphNode( endWp.node2 ); + nodesCache.nodesMap.endNode1 = end1; + nodesCache.nodesMap.endNode2 = end2; + endPos = new OsmNodeNamed( endWp.crosspoint ); + sameSegmentSearch = ( start1 == end1 && start2 == end2 ) || ( start1 == end2 && start2 == end1 ); } - if ( start2 == null ) return null; + if ( !nodesCache.obtainNonHollowNode( start1 ) ) + { + return null; + } + nodesCache.expandHollowLinkTargets( start1 ); + if ( !nodesCache.obtainNonHollowNode( start2 ) ) + { + return null; + } + nodesCache.expandHollowLinkTargets( start2 ); + routingContext.startDirectionValid = routingContext.forceUseStartDirection || fastPartialRecalc; routingContext.startDirectionValid &= routingContext.startDirection != null && !routingContext.inverseDirection; @@ -770,6 +782,10 @@ public class RoutingEngine extends Thread addToOpenset( startPath1 ); addToOpenset( startPath2 ); } + ArrayList openBorderList = new ArrayList(); + boolean memoryPanicMode = false; + boolean needNonPanicProcessing = false; + for(;;) { if ( terminated ) @@ -793,12 +809,72 @@ public class RoutingEngine extends Thread { path = openSet.popLowestKeyValue(); } - if ( path == null ) break; + if ( path == null ) + { + if ( openBorderList.isEmpty() ) + { + break; + } + for( OsmPath p : openBorderList ) + { + openSet.add( p.cost + (int)(p.airdistance*airDistanceCostFactor), p ); + } + openBorderList.clear(); + memoryPanicMode = false; + needNonPanicProcessing = true; + continue; + } + if ( path.airdistance == -1 ) { path.unregisterUpTree( routingContext ); continue; } + + if ( directWeaving && nodesCache.hasHollowLinkTargets( path.getTargetNode() ) ) + { + if ( !memoryPanicMode ) + { + if ( !nodesCache.nodesMap.isInMemoryBounds( openSet.getSize() ) ) + { + int nodesBefore = nodesCache.nodesMap.nodesCreated; + int pathsBefore = openSet.getSize(); + + nodesCache.nodesMap.collectOutreachers(); + for(;;) + { + OsmPath p3 = openSet.popLowestKeyValue(); + if ( p3 == null ) break; + if ( p3.airdistance != -1 && nodesCache.nodesMap.canEscape( p3.getTargetNode() ) ) + { + openBorderList.add( p3 ); + } + } + for( OsmPath p : openBorderList ) + { + openSet.add( p.cost + (int)(p.airdistance*airDistanceCostFactor), p ); + } + openBorderList.clear(); +System.out.println( "collected, nodes/paths before=" + nodesBefore + "/" + pathsBefore + " after=" + nodesCache.nodesMap.nodesCreated + "/" + openSet.getSize() ); + if ( !nodesCache.nodesMap.isInMemoryBounds( openSet.getSize()) ) // TODO + { + if ( maxTotalCost < 1000000000 || needNonPanicProcessing ) + { + throw new IllegalArgumentException( "memory limit reached" ); + } + memoryPanicMode = true; + System.out.println( "************************ memory limit reached, enabled memory panic mode *************************" ); + } + } + } + if ( memoryPanicMode ) + { + openBorderList.add( path ); + continue; + } + } + needNonPanicProcessing = false; + if ( fastPartialRecalc && matchPath != null && path.cost > 30L*firstMatchCost && !costCuttingTrack.isDirty ) { @@ -832,6 +908,12 @@ public class RoutingEngine extends Thread OsmNode sourceNode = path.getSourceNode(); OsmNode currentNode = path.getTargetNode(); + if ( currentLink.isLinkUnused() ) + { + path.unregisterUpTree( routingContext ); + continue; + } + long currentNodeId = currentNode.getIdFromPos(); long sourceNodeId = sourceNode.getIdFromPos(); @@ -911,6 +993,10 @@ public class RoutingEngine extends Thread continue; } + nodesCache.nodesMap.currentMaxCost = maxTotalCost; + nodesCache.nodesMap.currentPathCost = path.cost; + nodesCache.nodesMap.destination = endPos; + routingContext.firstPrePath = null; for( OsmLink link = currentNode.firstlink; link != null; link = link.getNext( currentNode) ) @@ -1162,9 +1248,10 @@ public class RoutingEngine extends Thread private OsmTrack mergeTrack( OsmPathElement match, OsmTrack oldTrack ) { - +System.out.println( "**************** merging match=" + match.cost + " with oldTrack=" + oldTrack.cost ); OsmPathElement element = match; OsmTrack track = new OsmTrack(); + track.cost = oldTrack.cost; while ( element != null ) { diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/DirectWeaver.java b/brouter-mapaccess/src/main/java/btools/mapaccess/DirectWeaver.java index 5b9acd9..aa25238 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/DirectWeaver.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/DirectWeaver.java @@ -11,8 +11,8 @@ import btools.util.ByteDataWriter; import btools.util.IByteArrayUnifier; /** - * MicroCache2 is the new format that uses statistical encoding and - * is able to do access filtering and waypoint matching during encoding + * DirectWeaver does the same decoding as MicroCache2, but decodes directly + * into the instance-graph, not into the intermediate nodes-cache */ public final class DirectWeaver extends ByteDataWriter { @@ -20,8 +20,7 @@ public final class DirectWeaver extends ByteDataWriter private int latBase; private int cellsize; - protected int[] faid; - protected int size = 0; + private int size = 0; private static boolean debug = false; @@ -43,8 +42,9 @@ public final class DirectWeaver extends ByteDataWriter NoisyDiffCoder transEleDiff = new NoisyDiffCoder( bc ); size = bc.decodeNoisyNumber( 5 ); - faid = size > dataBuffers.ibuf2.length ? new int[size] : dataBuffers.ibuf2; - + + int[] faid = size > dataBuffers.ibuf2.length ? new int[size] : dataBuffers.ibuf2; + if ( debug ) System.out.println( "*** decoding cache of size=" + size + " for lonIdx=" + lonIdx + " latIdx=" + latIdx ); bc.decodeSortedArray( faid, 0, size, 0x20000000, 0 ); @@ -59,9 +59,11 @@ public final class DirectWeaver extends ByteDataWriter if ( node == null ) { node = new OsmNode( ilon, ilat ); + node.visitID = 0; } else { + node.visitID = 1; hollowNodes.remove( node ); } nodes[n] = node; @@ -189,7 +191,11 @@ public final class DirectWeaver extends ByteDataWriter if ( nodeIdx != n ) // valid internal (forward-) link { OsmNode node2 = nodes[nodeIdx]; - OsmLink link = node.isLinkUnused() ? node : ( node2.isLinkUnused() ? node2 : new OsmLink() ); + OsmLink link = node.isLinkUnused() ? node : ( node2.isLinkUnused() ? node2 : null ); + if ( link == null ) + { + link = new OsmLink(); + } link.descriptionBitmap = wayTags.data; link.geometry = geometry; node.addLink( link, isReverse, node2 ); @@ -197,14 +203,13 @@ public final class DirectWeaver extends ByteDataWriter else // weave external link { node.addLink( linklon, linklat, wayTags.data, geometry, hollowNodes, isReverse ); + node.visitID = 1; } } - + } // ... loop over links + } // ... loop over nodes - - } - } - + hollowNodes.cleanupAndCount( nodes ); } public long expandId( int id32 ) diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java b/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java index 9d25e70..93a9b1c 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java @@ -20,7 +20,7 @@ public final class NodesCache private File segmentDir; private File secondarySegmentsDir = null; - private OsmNodesMap nodesMap; + public OsmNodesMap nodesMap; private BExpressionContextWay expCtxWay; private int lookupVersion; private int lookupMinorVersion; @@ -38,7 +38,7 @@ public final class NodesCache public String first_file_access_name; private long cacheSum = 0; - private long maxmem; + private long maxmemtiles; private boolean detailed; private boolean garbageCollectionEnabled = false; @@ -58,13 +58,14 @@ public final class NodesCache public NodesCache( String segmentDir, BExpressionContextWay ctxWay, boolean forceSecondaryData, long maxmem, NodesCache oldCache, boolean detailed ) { + this.maxmemtiles = maxmem / 8; this.segmentDir = new File( segmentDir ); this.nodesMap = new OsmNodesMap(); + this.nodesMap.maxmem = (2L*maxmem) / 3L; this.expCtxWay = ctxWay; this.lookupVersion = ctxWay.meta.lookupVersion; this.lookupMinorVersion = ctxWay.meta.lookupMinorVersion; this.forceSecondaryData = forceSecondaryData; - this.maxmem = maxmem; this.detailed = detailed; if ( ctxWay != null ) @@ -130,7 +131,7 @@ public final class NodesCache // clean all ghosts and enable garbage collection private void checkEnableCacheCleaning() { - if ( cacheSum < maxmem ) + if ( cacheSum < maxmemtiles ) { return; } @@ -158,7 +159,7 @@ public final class NodesCache if ( garbageCollectionEnabled ) { ghostCleaningDone = true; - maxmem *= 2; + maxmemtiles *= 2; } else { @@ -282,6 +283,21 @@ public final class NodesCache } } + /** + * make sure all link targets of the given node are non-hollow + */ + public boolean hasHollowLinkTargets( OsmNode n ) + { + for( OsmLink link = n.firstlink; link != null; link = link.getNext( n ) ) + { + if ( link.getTarget( n ).isHollow() ) + { + return true; + } + } + return false; + } + /** * get a node for the given id with all link-targets also non-hollow * @@ -307,6 +323,19 @@ public final class NodesCache return n; } + public OsmNode getGraphNode( OsmNode template ) + { + OsmNode graphNode = new OsmNode( template.ilon, template.ilat ); + graphNode.setHollow(); + OsmNode existing = nodesMap.put( graphNode ); + if ( existing == null ) + { + return graphNode; + } + nodesMap.put( existing ); + return existing; + } + public void matchWaypointsToNodes( List unmatchedWaypoints, double maxDistance, OsmNodePairSet islandNodePairs ) { waypointMatcher = new WaypointMatcherImpl( unmatchedWaypoints, 250., islandNodePairs ); diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java index 993d311..e468fe3 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java @@ -164,9 +164,7 @@ final class OsmFile return new MicroCache2( dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher ); } new DirectWeaver( dataBuffers, lonIdx, latIdx, divisor, wayValidator, waypointMatcher, hollowNodes ); - MicroCache dummy = MicroCache.emptyCache(); - dummy.virgin = false; - return dummy; + return MicroCache.emptyNonVirgin; } throw new IOException( "checkum error" ); } diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java index dc30e10..3cf9ddb 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java @@ -34,6 +34,8 @@ public class OsmNode extends OsmLink implements OsmPos public byte[] nodeDescription; public TurnRestriction firstRestriction; + + public int visitID; public void addTurnRestriction( TurnRestriction tr ) { @@ -234,6 +236,28 @@ public class OsmNode extends OsmLink implements OsmPos { return ( (long) ilon ) << 32 | ilat; } + + public void vanish() + { + if ( !isHollow() ) + { + OsmLink l = firstlink; + while( l != null ) + { + OsmNode target = l.getTarget( this ); + OsmLink nextLink = l.getNext( this ); + if ( !target.isHollow() ) + { + unlinkLink( l ); + if ( !l.isLinkUnused() ) + { + target.unlinkLink( l ); + } + } + l = nextLink; + } + } + } public final void unlinkLink( OsmLink link ) { @@ -258,7 +282,7 @@ public class OsmNode extends OsmLink implements OsmPos } l = nl; } - else + else if ( l.n2 != this && l.n2 != null ) { OsmLink nl = l.next; if ( nl == link ) @@ -268,6 +292,10 @@ public class OsmNode extends OsmLink implements OsmPos } l = nl; } + else + { + throw new IllegalArgumentException( "unlinkLink: unknown source" ); + } } } diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodePairSet.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodePairSet.java index 7e930b3..b790e12 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodePairSet.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodePairSet.java @@ -96,6 +96,11 @@ public class OsmNodePairSet return tempNodes; } + public int getMaxTmpNodes() + { + return maxTempNodes; + } + public int getFreezeCount() { return freezecount; diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodesMap.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodesMap.java index c62a7f5..723a4cd 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodesMap.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNodesMap.java @@ -5,9 +5,11 @@ */ package btools.mapaccess; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; import btools.util.ByteArrayUnifier; +import btools.util.SortedHeap; public final class OsmNodesMap { @@ -16,6 +18,222 @@ public final class OsmNodesMap private ByteArrayUnifier abUnifier = new ByteArrayUnifier( 16384, false ); private OsmNode testKey = new OsmNode(); + + public int nodesCreated; + public long maxmem; + public int lastVisitID = 1000; + public int baseID = 1000; + + public OsmNode destination; + public int currentPathCost; + public int currentMaxCost = 1000000000; + + public OsmNode endNode1; + public OsmNode endNode2; + + public int cleanupMode = 0; + + public void cleanupAndCount( OsmNode[] nodes ) + { + if ( cleanupMode == 0 ) + { + justCount( nodes ); + } + else + { + cleanupPeninsulas( nodes ); + } + } + + private void justCount( OsmNode[] nodes ) + { + for( int i=0; i n.visitID ) // peninsula ? + { + nodesCreated = nodesCreatedUntilHere; + n.unlinkLink( l ); + t.unlinkLink( l ); + } + } + else if ( minIdSub < baseID ) + { + continue; + } + else if ( cleanupMode == 2 ) + { + minIdSub = baseID; // in tree-mode, hitting anything is like a gateway + } + if ( minIdSub < minId ) minId = minIdSub; + } + return minId; + } + + + + public boolean isInMemoryBounds( int npaths ) + { +// long total = nodesCreated * 76L + linksCreated * 48L; + long total = nodesCreated * 95L + npaths * 200L; + return total <= maxmem; + } + + private void addActiveNode( ArrayList nodes2check, OsmNode n ) + { + n.visitID = lastVisitID; + nodesCreated++; + nodes2check.add( n ); + } + + // is there an escape from this node + // to a hollow node (or destination node) ? + public boolean canEscape( OsmNode n0 ) + { + boolean sawLowIDs = false; + lastVisitID++; + ArrayList nodes2check = new ArrayList(); + nodes2check.add( n0 ); + while ( !nodes2check.isEmpty() ) + { + OsmNode n = nodes2check.remove( nodes2check.size()-1 ); + if ( n.visitID < baseID ) + { + n.visitID = lastVisitID; + nodesCreated++; + for( OsmLink l = n.firstlink; l != null; l = l.getNext( n ) ) + { + OsmNode t = l.getTarget( n ); + nodes2check.add( t ); + } + } + else if ( n.visitID < lastVisitID ) + { + sawLowIDs = true; + } + } + if ( sawLowIDs ) + { + return true; + } + + nodes2check.add( n0 ); + while ( !nodes2check.isEmpty() ) + { + OsmNode n = nodes2check.remove( nodes2check.size()-1 ); + if ( n.visitID == lastVisitID ) + { + n.visitID = lastVisitID; + nodesCreated--; + for( OsmLink l = n.firstlink; l != null; l = l.getNext( n ) ) + { + OsmNode t = l.getTarget( n ); + nodes2check.add( t ); + } + n.vanish(); + } + } + + return false; + } + + public void collectOutreachers() + { + nodesCreated=0; + +System.out.println( "collectOutreachers, currentMaxCost=" + currentMaxCost ); + ArrayList nodes2check = new ArrayList(); + for( OsmNode n : hmap.values() ) + { + addActiveNode( nodes2check, n ); + } + + lastVisitID++; + baseID = lastVisitID; + + while ( !nodes2check.isEmpty() ) + { + OsmNode n = nodes2check.remove( nodes2check.size()-1 ); + n.visitID = lastVisitID; + + for( OsmLink l = n.firstlink; l != null; l = l.getNext( n ) ) + { + OsmNode t = l.getTarget( n ); + if ( t.visitID != lastVisitID ) + { + addActiveNode( nodes2check, t ); + } + } + if ( destination != null && currentMaxCost < 1000000000 ) + { + int distance = n.calcDistance( destination ); + if ( distance > currentMaxCost - currentPathCost + 100 ) + { + n.vanish(); + } + } + if ( n.firstlink == null ) + { + nodesCreated--; + } + } + } + public ByteArrayUnifier getByteArrayUnifier() { @@ -36,7 +254,10 @@ public final class OsmNodesMap public void remove( OsmNode node ) { - hmap.remove( node ); + if ( node != endNode1 && node != endNode2 ) // keep endnodes in hollow-map even when loaded + { // (needed for escape analysis) + hmap.remove( node ); + } } /** @@ -48,4 +269,58 @@ public final class OsmNodesMap return hmap.put( node, node ); } + // ********************** test cleanup ********************** + + private static void addLinks( OsmNode[] nodes, int idx, boolean isBorder, int[] links ) + { + OsmNode n = nodes[idx]; + n.visitID = isBorder ? 1 : 0; + n.selev = (short)idx; + for( int i : links ) + { + OsmNode t = nodes[i]; + OsmLink link = n.isLinkUnused() ? n : ( t.isLinkUnused() ? t : null ); + if ( link == null ) + { + link = new OsmLink(); + } + n.addLink( link, false, t ); + } + } + + public static void main( String[] args ) + { + OsmNode[] nodes = new OsmNode[12]; + for( int i=0; i