diff --git a/brouter-core/src/main/java/btools/router/RoutingContext.java b/brouter-core/src/main/java/btools/router/RoutingContext.java index ae776a0..1bf3ef5 100644 --- a/brouter-core/src/main/java/btools/router/RoutingContext.java +++ b/brouter-core/src/main/java/btools/router/RoutingContext.java @@ -50,13 +50,13 @@ public final class RoutingContext implements DistanceChecker public void readGlobalConfig( BExpressionContext expctxGlobal ) { - downhillcostdiv = (int)expctxGlobal.getVariableValue( "downhillcost" ); - downhillcutoff = (int)(expctxGlobal.getVariableValue( "downhillcutoff" )*10000); - uphillcostdiv = (int)expctxGlobal.getVariableValue( "uphillcost" ); - uphillcutoff = (int)(expctxGlobal.getVariableValue( "uphillcutoff" )*10000); + downhillcostdiv = (int)expctxGlobal.getVariableValue( "downhillcost", 0.f ); + downhillcutoff = (int)(expctxGlobal.getVariableValue( "downhillcutoff", 0.f )*10000); + uphillcostdiv = (int)expctxGlobal.getVariableValue( "uphillcost", 0.f ); + uphillcutoff = (int)(expctxGlobal.getVariableValue( "uphillcutoff", 0.f )*10000); if ( downhillcostdiv != 0 ) downhillcostdiv = 1000000/downhillcostdiv; if ( uphillcostdiv != 0 ) uphillcostdiv = 1000000/uphillcostdiv; - carMode = 0.f != expctxGlobal.getVariableValue( "validForCars" ); + carMode = 0.f != expctxGlobal.getVariableValue( "validForCars", 0.f ); pass1coefficient = expctxGlobal.getVariableValue( "pass1coefficient", 1.5f ); pass2coefficient = expctxGlobal.getVariableValue( "pass2coefficient", 0.f ); } diff --git a/brouter-core/src/main/java/btools/router/RoutingEngine.java b/brouter-core/src/main/java/btools/router/RoutingEngine.java index c271fff..a48dd75 100644 --- a/brouter-core/src/main/java/btools/router/RoutingEngine.java +++ b/brouter-core/src/main/java/btools/router/RoutingEngine.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.List; import btools.expressions.BExpressionContext; +import btools.expressions.BExpressionMetaData; import btools.mapaccess.NodesCache; import btools.mapaccess.OsmLink; import btools.mapaccess.OsmLinkHolder; @@ -70,18 +71,20 @@ public class RoutingEngine extends Thread profileDir = new File( profileBaseDir ); profileFile = new File( profileDir, rc.localFunction + ".brf" ) ; } - BExpressionContext expctxGlobal = new BExpressionContext( "global" ); - expctxGlobal.readMetaData( new File( profileDir, "lookups.dat" ) ); + + BExpressionMetaData meta = new BExpressionMetaData(); + + BExpressionContext expctxGlobal = new BExpressionContext( "global", meta ); + rc.expctxWay = new BExpressionContext( "way", meta ); + rc.expctxNode = new BExpressionContext( "node", 1024, meta ); + + meta.readMetaData( new File( profileDir, "lookups.dat" ) ); + expctxGlobal.parseFile( profileFile, null ); expctxGlobal.evaluate( new int[0] ); rc.readGlobalConfig(expctxGlobal); - rc.expctxWay = new BExpressionContext( "way", 4096 ); - rc.expctxWay.readMetaData( new File( profileDir, "lookups.dat" ) ); rc.expctxWay.parseFile( profileFile, "global" ); - - rc.expctxNode = new BExpressionContext( "node", 1024 ); - rc.expctxNode.readMetaData( new File( profileDir, "lookups.dat" ) ); rc.expctxNode.parseFile( profileFile, "global" ); } } @@ -372,7 +375,6 @@ public class RoutingEngine extends Thread if ( matchPath != null ) { track = mergeTrack( matchPath, nearbyTrack ); - isDirty = true; } maxRunningTime += System.currentTimeMillis() - startTime; // reset timeout... } @@ -432,7 +434,8 @@ public class RoutingEngine extends Thread private void resetCache() { nodesMap = new OsmNodesMap(); - nodesCache = new NodesCache(segmentDir, nodesMap, routingContext.expctxWay.lookupVersion,routingContext.expctxWay.lookupMinorVersion, routingContext.carMode, nodesCache ); + BExpressionContext ctx = routingContext.expctxWay; + nodesCache = new NodesCache(segmentDir, nodesMap, ctx.meta.lookupVersion, ctx.meta.lookupMinorVersion, ctx.meta.readVarLength, routingContext.carMode, nodesCache ); } private OsmNode getStartNode( long startId ) @@ -566,11 +569,13 @@ public class RoutingEngine extends Thread } } - private OsmTrack findTrack( String operationName, MatchedWaypoint startWp, MatchedWaypoint endWp, OsmTrack costCuttingTrack, OsmTrack refTrack, boolean reducedTimeoutWhenUnmatched ) + private OsmTrack findTrack( String operationName, MatchedWaypoint startWp, MatchedWaypoint endWp, OsmTrack costCuttingTrack, OsmTrack refTrack, boolean fastPartialRecalc ) { boolean verbose = guideTrack != null; int maxTotalCost = 1000000000; + int firstMatchCost = 1000000000; + int firstEstimate = 1000000000; logInfo( "findtrack with maxTotalCost=" + maxTotalCost + " airDistanceCostFactor=" + airDistanceCostFactor ); @@ -607,7 +612,7 @@ public class RoutingEngine extends Thread { if ( maxRunningTime > 0 ) { - long timeout = ( matchPath == null && reducedTimeoutWhenUnmatched ) ? maxRunningTime/3 : maxRunningTime; + long timeout = ( matchPath == null && fastPartialRecalc ) ? maxRunningTime/3 : maxRunningTime; if ( System.currentTimeMillis() - startTime > timeout ) { throw new IllegalArgumentException( operationName + " timeout after " + (timeout/1000) + " seconds" ); @@ -627,6 +632,11 @@ public class RoutingEngine extends Thread } maxAdjCostFromQueue = path.adjustedCost; + if ( matchPath != null && fastPartialRecalc && firstMatchCost < 500 && path.cost > 30L*firstMatchCost ) + { + throw new IllegalArgumentException( "early exit for a close recalc" ); + } + nodesVisited++; linksProcessed++; @@ -645,6 +655,32 @@ public class RoutingEngine extends Thread logInfo( "found track at cost " + path.cost + " nodesVisited = " + nodesVisited ); return compileTrack( path, verbose ); } + + // check for a match with the cost-cutting-track + if ( costCuttingTrack != null ) + { + OsmPathElement pe = costCuttingTrack.getLink( sourceNodeId, currentNodeId ); + if ( pe != null ) + { + // remember first match cost for fast termination of partial recalcs + int parentcost = path.originElement == null ? 0 : path.originElement.cost; + if ( parentcost < firstMatchCost ) firstMatchCost = parentcost; + + int costEstimate = path.cost + + path.elevationCorrection( routingContext ) + + ( costCuttingTrack.cost - pe.cost ); + if ( costEstimate <= maxTotalCost ) + { + if ( matchPath == null ) firstEstimate = costEstimate; + matchPath = new OsmPathElement( path ); + } + if ( costEstimate < maxTotalCost ) + { + logInfo( "maxcost " + maxTotalCost + " -> " + costEstimate ); + maxTotalCost = costEstimate; + } + } + } } // recheck cutoff before doing expensive stuff @@ -743,27 +779,6 @@ public class RoutingEngine extends Thread int airDistance = isFinalLink ? 0 : nextNode.calcDistance( endPos ); bestPath.setAirDistanceCostAdjustment( (int)( airDistance * airDistanceCostFactor ) ); - // check for a match with the cost-cutting-track - if ( costCuttingTrack != null ) - { - OsmPathElement pe = costCuttingTrack.getLink( currentNodeId, targetNodeId ); - if ( pe != null ) - { - int costEstimate = bestPath.cost - + bestPath.elevationCorrection( routingContext ) - + ( costCuttingTrack.cost - pe.cost ); - if ( costEstimate <= maxTotalCost ) - { - matchPath = new OsmPathElement( bestPath ); - } - if ( costEstimate < maxTotalCost ) - { - logInfo( "maxcost " + maxTotalCost + " -> " + costEstimate + " airDistance=" + airDistance ); - maxTotalCost = costEstimate; - } - } - } - if ( isFinalLink || bestPath.cost + airDistance <= maxTotalCost + 10 ) { // add only if this may beat an existing path for that link diff --git a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java index b67584b..f7a5026 100644 --- a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java +++ b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java @@ -23,11 +23,8 @@ import btools.util.Crc32; public final class BExpressionContext { - private static final String CONTEXT_TAG = "---context:"; - private static final String VERSION_TAG = "---lookupversion:"; - private static final String MINOR_VERSION_TAG = "---minorversion:"; - private static final String VARLENGTH_TAG = "---readvarlength"; - + private static final String CONTEXT_TAG = "---context:"; + private String context; private boolean _inOurContext = false; private BufferedReader _br = null; @@ -55,7 +52,6 @@ public final class BExpressionContext private byte[][] _arrayBitmap; private int currentHashBucket = -1; private byte[] currentByteArray = null; - private boolean currentInversion = false; public List expressionList; @@ -79,13 +75,12 @@ public final class BExpressionContext private int linenr; - public short lookupVersion = -1; - public short lookupMinorVersion = -1; - public boolean readVarLength = false; + public BExpressionMetaData meta; + private boolean lookupDataValid = false; - public BExpressionContext( String context ) + public BExpressionContext( String context, BExpressionMetaData meta ) { - this( context, 4096 ); + this( context, 4096, meta ); } /** @@ -94,9 +89,15 @@ public final class BExpressionContext * @param context global, way or node - context of that instance * @param hashSize size of hashmap for result caching */ - public BExpressionContext( String context, int hashSize ) + public BExpressionContext( String context, int hashSize, BExpressionMetaData meta ) { - this.context = context; + this.context = context; + this.meta = meta; + + meta.registerListener(context, this ); + + if ( Boolean.getBoolean( "disableExpressionCache" ) ) hashSize = 1; + _arrayBitmap = new byte[hashSize][]; _arrayCostfactor = new float[hashSize]; @@ -105,18 +106,18 @@ public final class BExpressionContext _arrayNodeAccessGranted = new float[hashSize]; } - /** * encode internal lookup data to a byte array */ public byte[] encode() { + if ( !lookupDataValid ) throw new IllegalArgumentException( "internal error: encoding undefined data?" ); return encode( lookupData ); } public byte[] encode( int[] ld ) { - if ( !readVarLength ) return encodeFix( ld ); + if ( !meta.readVarLength ) return encodeFix( ld ); // start with first bit hardwired ("reversedirection") BitCoderContext ctx = new BitCoderContext( abBuf ); @@ -134,16 +135,16 @@ public final class BExpressionContext skippedTags++; continue; } - ctx.encodeDistance( skippedTags+1 ); + ctx.encodeVarBits( skippedTags+1 ); nonNullTags++; skippedTags = 0; // 0 excluded already, 1 (=unknown) we rotate up to 8 // to have the good code space for the popular values int dd = d < 2 ? 7 : ( d < 9 ? d - 2 : d - 1); - ctx.encodeDistance( dd ); + ctx.encodeVarBits( dd ); } - ctx.encodeDistance( 0 ); + ctx.encodeVarBits( 0 ); if ( nonNullTags == 0) return null; @@ -200,6 +201,7 @@ public final class BExpressionContext public void decode( byte[] ab ) { decode( lookupData, ab ); + lookupDataValid = true; } /** @@ -207,7 +209,7 @@ public final class BExpressionContext */ public void decode( int[] ld, byte[] ab ) { - if ( !readVarLength ) { decodeFix( ld, ab ); return; } + if ( !meta.readVarLength ) { decodeFix( ld, ab ); return; } BitCoderContext ctx = new BitCoderContext(ab); @@ -218,14 +220,14 @@ public final class BExpressionContext int inum = 1; for(;;) { - int delta = ctx.decodeDistance(); + int delta = ctx.decodeVarBits(); if ( delta == 0) break; if ( inum + delta > ld.length ) break; // higher minor version is o.k. while ( delta-- > 1 ) ld[inum++] = 0; // see encoder for value rotation - int dd = ctx.decodeDistance(); + int dd = ctx.decodeVarBits(); int d = dd == 7 ? 1 : ( dd < 7 ? dd + 2 : dd + 1); if ( d >= lookupValues.get(inum).length ) d = 1; // map out-of-range to unknown ld[inum++] = d; @@ -265,7 +267,7 @@ public final class BExpressionContext public String getCsvDescription( boolean inverseDirection, byte[] ab ) { - int inverseBitByteIndex = readVarLength ? 0 : 7; + int inverseBitByteIndex = meta.readVarLength ? 0 : 7; int abLen = ab.length; byte[] ab_copy = new byte[abLen]; System.arraycopy( ab, 0, ab_copy, 0 , abLen ); @@ -275,8 +277,10 @@ public final class BExpressionContext decode( lookupData, ab_copy ); for( int inum = 0; inum < lookupValues.size(); inum++ ) // loop over lookup names { - BExpressionLookupValue[] va = lookupValues.get(inum); - sb.append( '\t' ).append( va[lookupData[inum]].toString() ); + int idx = meta.readVarLength ? (inum+1)%lookupValues.size() : inum; // reversebit at the end.. + + BExpressionLookupValue[] va = lookupValues.get(idx); + sb.append( '\t' ).append( va[lookupData[idx]].toString() ); } return sb.toString(); } @@ -284,9 +288,10 @@ public final class BExpressionContext public String getCsvHeader() { StringBuilder sb = new StringBuilder( 200 ); - for( String name: lookupNames ) + for( int inum = 0; inum < lookupNames.size(); inum++ ) // loop over lookup names { - sb.append( '\t' ).append( name ); + int idx = meta.readVarLength ? (inum+1)%lookupValues.size() : inum; // reversebit at the end.. + sb.append( '\t' ).append( lookupNames.get(idx) ); } return sb.toString(); } @@ -307,42 +312,11 @@ public final class BExpressionContext return sb.toString(); } - public void readMetaData( File lookupsFile ) + private int parsedLines = 0; + private boolean fixTagsWritten = false; + + public void parseMetaLine( String line ) { - try - { - BufferedReader br = new BufferedReader( new FileReader( lookupsFile ) ); - - int parsedLines = 0; - boolean ourContext = false; - boolean fixTagsWritten = false; - for(;;) - { - String line = br.readLine(); - if ( line == null ) break; - line = line.trim(); - if ( line.length() == 0 || line.startsWith( "#" ) ) continue; - if ( line.startsWith( CONTEXT_TAG ) ) - { - ourContext = line.substring( CONTEXT_TAG.length() ).equals( context ); - continue; - } - if ( line.startsWith( VERSION_TAG ) ) - { - lookupVersion = Short.parseShort( line.substring( VERSION_TAG.length() ) ); - continue; - } - if ( line.startsWith( MINOR_VERSION_TAG ) ) - { - lookupMinorVersion = Short.parseShort( line.substring( MINOR_VERSION_TAG.length() ) ); - continue; - } - if ( line.startsWith( VARLENGTH_TAG ) ) - { - readVarLength = true; - continue; - } - if ( !ourContext ) continue; parsedLines++; StringTokenizer tk = new StringTokenizer( line, " " ); String name = tk.nextToken(); @@ -350,7 +324,7 @@ public final class BExpressionContext int idx = name.indexOf( ';' ); if ( idx >= 0 ) name = name.substring( 0, idx ); - if ( readVarLength ) + if ( meta.readVarLength ) { if ( !fixTagsWritten ) { @@ -358,28 +332,24 @@ public final class BExpressionContext if ( "way".equals( context ) ) addLookupValue( "reversedirection", "yes", null ); else if ( "node".equals( context ) ) addLookupValue( "nodeaccessgranted", "yes", null ); } - if ( "reversedirection".equals( name ) ) continue; // this is hardcoded - if ( "nodeaccessgranted".equals( name ) ) continue; // this is hardcoded + if ( "reversedirection".equals( name ) ) return; // this is hardcoded + if ( "nodeaccessgranted".equals( name ) ) return; // this is hardcoded } BExpressionLookupValue newValue = addLookupValue( name, value, null ); // add aliases while( newValue != null && tk.hasMoreTokens() ) newValue.addAlias( tk.nextToken() ); - } - br.close(); + } + + public void finishMetaParsing() + { if ( parsedLines == 0 && !"global".equals(context) ) { - throw new IllegalArgumentException( lookupsFile.getAbsolutePath() - + " does not contain data for context " + context + " (old version?)" ); + throw new IllegalArgumentException( "lookup table does not contain data for context " + context + " (old version?)" ); } // post-process metadata: lookupDataFrozen = true; - } - catch( Exception e ) - { - throw new RuntimeException( e ); - } } public void evaluate( int[] lookupData2 ) @@ -400,7 +370,9 @@ public final class BExpressionContext */ public boolean evaluate( boolean inverseDirection, byte[] ab, BExpressionReceiver receiver ) { - int inverseBitByteIndex = readVarLength ? 0 : 7; + lookupDataValid = false; // this is an assertion for a nasty pifall + + int inverseBitByteIndex = meta.readVarLength ? 0 : 7; int abLen = ab.length; boolean equalsCurrent = currentHashBucket >= 0 && abLen == currentByteArray.length; @@ -685,18 +657,12 @@ public final class BExpressionContext return num == null ? defaultValue : getVariableValue( num.intValue() ); } - public float getVariableValue( String name ) - { - Integer num = variableNumbers.get( name ); - return num == null ? 0.f : getVariableValue( num.intValue() ); - } - - public float getVariableValue( int variableIdx ) + float getVariableValue( int variableIdx ) { return variableData[variableIdx]; } - public int getVariableIdx( String name, boolean create ) + int getVariableIdx( String name, boolean create ) { Integer num = variableNumbers.get( name ); if ( num == null ) @@ -714,12 +680,12 @@ public final class BExpressionContext return num.intValue(); } - public int getMinWriteIdx() + int getMinWriteIdx() { return minWriteIdx; } - public float getLookupMatch( int nameIdx, int valueIdx ) + float getLookupMatch( int nameIdx, int valueIdx ) { return lookupData[nameIdx] == valueIdx ? 1.0f : 0.0f; } @@ -730,7 +696,7 @@ public final class BExpressionContext return num == null ? -1 : num.intValue(); } - public int getLookupValueIdx( int nameIdx, String value ) + int getLookupValueIdx( int nameIdx, String value ) { BExpressionLookupValue[] values = lookupValues.get( nameIdx ); for( int i=0; i< values.length; i++ ) @@ -741,7 +707,7 @@ public final class BExpressionContext } - public String parseToken() throws Exception + String parseToken() throws Exception { for(;;) { @@ -790,13 +756,13 @@ public final class BExpressionContext } } - public float assign( int variableIdx, float value ) + float assign( int variableIdx, float value ) { variableData[variableIdx] = value; return value; } - public void expressionWarning( String message ) + void expressionWarning( String message ) { _arrayBitmap[currentHashBucket] = null; // no caching if warnings if ( _receiver != null ) _receiver.expressionWarning( context, message ); diff --git a/brouter-expressions/src/main/java/btools/expressions/BExpressionMetaData.java b/brouter-expressions/src/main/java/btools/expressions/BExpressionMetaData.java new file mode 100644 index 0000000..d50c498 --- /dev/null +++ b/brouter-expressions/src/main/java/btools/expressions/BExpressionMetaData.java @@ -0,0 +1,91 @@ +// context for simple expression +// context means: +// - the local variables +// - the local variable names +// - the lookup-input variables + +package btools.expressions; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.StringTokenizer; +import java.util.TreeMap; + +import btools.util.BitCoderContext; +import btools.util.Crc32; + + +public final class BExpressionMetaData +{ + private static final String CONTEXT_TAG = "---context:"; + private static final String VERSION_TAG = "---lookupversion:"; + private static final String MINOR_VERSION_TAG = "---minorversion:"; + private static final String VARLENGTH_TAG = "---readvarlength"; + + public short lookupVersion = -1; + public short lookupMinorVersion = -1; + public boolean readVarLength = false; + + private HashMap listeners = new HashMap(); + + public void registerListener( String context, BExpressionContext ctx ) + { + listeners.put( context, ctx ); + } + + public void readMetaData( File lookupsFile ) + { + try + { + BufferedReader br = new BufferedReader( new FileReader( lookupsFile ) ); + + BExpressionContext ctx = null; + + for(;;) + { + String line = br.readLine(); + if ( line == null ) break; + line = line.trim(); + if ( line.length() == 0 || line.startsWith( "#" ) ) continue; + if ( line.startsWith( CONTEXT_TAG ) ) + { + ctx = listeners.get( line.substring( CONTEXT_TAG.length() ) ); + continue; + } + if ( line.startsWith( VERSION_TAG ) ) + { + lookupVersion = Short.parseShort( line.substring( VERSION_TAG.length() ) ); + continue; + } + if ( line.startsWith( MINOR_VERSION_TAG ) ) + { + lookupMinorVersion = Short.parseShort( line.substring( MINOR_VERSION_TAG.length() ) ); + continue; + } + if ( line.startsWith( VARLENGTH_TAG ) ) + { + readVarLength = true; + continue; + } + if ( ctx != null ) ctx.parseMetaLine( line ); + } + br.close(); + + for( BExpressionContext c : listeners.values() ) + { + c.finishMetaParsing(); + } + + } + catch( Exception e ) + { + throw new RuntimeException( e ); + } + } +} diff --git a/brouter-expressions/src/test/java/btools/expressions/EncodeDecodeTest.java b/brouter-expressions/src/test/java/btools/expressions/EncodeDecodeTest.java index 16e919a..aeda4e8 100644 --- a/brouter-expressions/src/test/java/btools/expressions/EncodeDecodeTest.java +++ b/brouter-expressions/src/test/java/btools/expressions/EncodeDecodeTest.java @@ -18,8 +18,9 @@ public class EncodeDecodeTest File lookupFile = new File( profileDir, "lookups.dat" ); // read lookup.dat + trekking.brf - BExpressionContext expctxWay = new BExpressionContext("way"); - expctxWay.readMetaData( lookupFile ); + BExpressionMetaData meta = new BExpressionMetaData(); + BExpressionContext expctxWay = new BExpressionContext("way", 4096, meta ); + meta.readMetaData( lookupFile ); expctxWay.parseFile( new File( profileDir, "trekking.brf" ), "global" ); String[] tags = { "highway=residential", "oneway=yes", "reversedirection=yes" }; diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/MapCreatorBase.java b/brouter-map-creator/src/main/java/btools/mapcreator/MapCreatorBase.java index 7040004..7255970 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/MapCreatorBase.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/MapCreatorBase.java @@ -15,9 +15,11 @@ import java.io.FileOutputStream; import java.io.IOException; import java.util.HashMap; +import btools.util.DiffCoderDataOutputStream; + public abstract class MapCreatorBase implements WayListener, NodeListener, RelationListener { - private DataOutputStream[] tileOutStreams; + private DiffCoderDataOutputStream[] tileOutStreams; protected File outTileDir; protected HashMap tags; @@ -102,16 +104,16 @@ public abstract class MapCreatorBase implements WayListener, NodeListener, Relat return new DataInputStream( new BufferedInputStream ( new FileInputStream( inFile ) ) ); } - protected DataOutputStream createOutStream( File outFile ) throws IOException + protected DiffCoderDataOutputStream createOutStream( File outFile ) throws IOException { - return new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) ); + return new DiffCoderDataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) ); } - protected DataOutputStream getOutStreamForTile( int tileIndex ) throws Exception + protected DiffCoderDataOutputStream getOutStreamForTile( int tileIndex ) throws Exception { if ( tileOutStreams == null ) { - tileOutStreams = new DataOutputStream[64]; + tileOutStreams = new DiffCoderDataOutputStream[64]; } if ( tileOutStreams[tileIndex] == null ) diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/NodeData.java b/brouter-map-creator/src/main/java/btools/mapcreator/NodeData.java index b8937b6..515ac16 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/NodeData.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/NodeData.java @@ -1,7 +1,7 @@ package btools.mapcreator; -import java.io.DataInputStream; -import java.io.DataOutputStream; +import btools.util.DiffCoderDataInputStream; +import btools.util.DiffCoderDataOutputStream; /** * Container for node data on the preprocessor level @@ -23,21 +23,21 @@ public class NodeData extends MapCreatorBase ilon = (int)( ( lon + 180. )*1000000. + 0.5); } - public NodeData( DataInputStream dis ) throws Exception + public NodeData( DiffCoderDataInputStream dis ) throws Exception { - nid = readId( dis ); - ilon = dis.readInt(); - ilat = dis.readInt(); + nid = dis.readDiffed( 0 ); + ilon = (int)dis.readDiffed( 1 ); + ilat = (int)dis.readDiffed( 2 ); int mode = dis.readByte(); if ( ( mode & 1 ) != 0 ) { int dlen = dis.readByte(); description = new byte[dlen]; dis.readFully( description ); } if ( ( mode & 2 ) != 0 ) selev = dis.readShort(); } - public void writeTo( DataOutputStream dos ) throws Exception + public void writeTo( DiffCoderDataOutputStream dos ) throws Exception { - writeId( dos, nid ); - dos.writeInt( ilon ); - dos.writeInt( ilat ); + dos.writeDiffed( nid, 0 ); + dos.writeDiffed( ilon, 1 ); + dos.writeDiffed( ilat, 2 ); int mode = ( description == null ? 0 : 1 ) | ( selev == Short.MIN_VALUE ? 0 : 2 ); dos.writeByte( (byte)mode ); if ( ( mode & 1 ) != 0 ) { dos.writeByte( description.length ); dos.write( description ); } diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/NodeFilter.java b/brouter-map-creator/src/main/java/btools/mapcreator/NodeFilter.java index c113c59..cf85c82 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/NodeFilter.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/NodeFilter.java @@ -1,11 +1,11 @@ package btools.mapcreator; import java.io.BufferedOutputStream; -import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import btools.util.DenseLongMap; +import btools.util.DiffCoderDataOutputStream; import btools.util.TinyDenseLongMap; /** @@ -17,7 +17,7 @@ import btools.util.TinyDenseLongMap; */ public class NodeFilter extends MapCreatorBase { - private DataOutputStream nodesOutStream; + private DiffCoderDataOutputStream nodesOutStream; private File nodeTilesOut; protected DenseLongMap nodebitmap; @@ -61,7 +61,7 @@ public class NodeFilter extends MapCreatorBase String filename = nodefile.getName(); filename = filename.substring( 0, filename.length() - 3 ) + "tlf"; File outfile = new File( nodeTilesOut, filename ); - nodesOutStream = new DataOutputStream( new BufferedOutputStream ( new FileOutputStream( outfile ) ) ); + nodesOutStream = new DiffCoderDataOutputStream( new BufferedOutputStream ( new FileOutputStream( outfile ) ) ); } @Override diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/NodeIterator.java b/brouter-map-creator/src/main/java/btools/mapcreator/NodeIterator.java index d1f6115..c523f92 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/NodeIterator.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/NodeIterator.java @@ -1,11 +1,12 @@ package btools.mapcreator; import java.io.BufferedInputStream; -import java.io.DataInputStream; import java.io.EOFException; import java.io.File; import java.io.FileInputStream; +import btools.util.DiffCoderDataInputStream; + /** * Iterate over a singe nodefile or a directory * of nodetiles and feed the nodes to the callback listener @@ -48,7 +49,7 @@ public class NodeIterator extends MapCreatorBase listener.nodeFileStart( nodefile ); - DataInputStream di = new DataInputStream( new BufferedInputStream ( new FileInputStream( nodefile ) ) ); + DiffCoderDataInputStream di = new DiffCoderDataInputStream( new BufferedInputStream ( new FileInputStream( nodefile ) ) ); try { for(;;) diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java b/brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java index 0d2085e..0769a7b 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java @@ -13,6 +13,7 @@ import java.io.File; import java.io.FileOutputStream; import btools.expressions.BExpressionContext; +import btools.expressions.BExpressionMetaData; public class OsmCutter extends MapCreatorBase { @@ -47,7 +48,7 @@ public class OsmCutter extends MapCreatorBase private BExpressionContext _expctxNode; private BExpressionContext _expctxWayStat; - private BExpressionContext _expctxNodeStat; + // private BExpressionContext _expctxNodeStat; public void process (File lookupFile, File outTileDir, File wayFile, File relFile, File mapFile) throws Exception { @@ -56,13 +57,13 @@ public class OsmCutter extends MapCreatorBase throw new IllegalArgumentException( "lookup-file: " + lookupFile + " does not exist" ); } - _expctxWay = new BExpressionContext("way"); - _expctxWay.readMetaData( lookupFile ); -// _expctxWayStat = new BExpressionContext("way"); + BExpressionMetaData meta = new BExpressionMetaData(); - _expctxNode = new BExpressionContext("node"); - _expctxNode.readMetaData( lookupFile ); -// _expctxNodeStat = new BExpressionContext("node"); + _expctxWay = new BExpressionContext("way", meta ); + _expctxNode = new BExpressionContext("node", meta ); + meta.readMetaData( lookupFile ); +// _expctxWayStat = new BExpressionContext("way", null ); +// _expctxNodeStat = new BExpressionContext("node", null ); this.outTileDir = outTileDir; if ( !outTileDir.isDirectory() ) throw new RuntimeException( "out tile directory " + outTileDir + " does not exist" ); @@ -122,8 +123,7 @@ public class OsmCutter extends MapCreatorBase int tileIndex = getTileIndex( n.ilon, n.ilat ); if ( tileIndex >= 0 ) { - DataOutputStream dos = getOutStreamForTile( tileIndex ); - n.writeTo( dos ); + n.writeTo( getOutStreamForTile( tileIndex ) ); } } diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/OsmNodeP.java b/brouter-map-creator/src/main/java/btools/mapcreator/OsmNodeP.java index 1e45f11..0d5fb69 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/OsmNodeP.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/OsmNodeP.java @@ -9,10 +9,14 @@ import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; +import sun.security.pkcs.SigningCertificateInfo; + +import btools.util.ByteDataWriter; + public class OsmNodeP implements Comparable { - public static final int EXTERNAL_BITMASK = 0x80; - public static final int VARIABLEDESC_BITMASK = 0x40; + public static final int SIGNLON_BITMASK = 0x80; + public static final int SIGNLAT_BITMASK = 0x40; public static final int TRANSFERNODE_BITMASK = 0x20; public static final int WRITEDESC_BITMASK = 0x10; public static final int SKIPDETAILS_BITMASK = 0x08; @@ -42,10 +46,12 @@ public class OsmNodeP implements Comparable public boolean isBorder = false; - public final static int BRIDGE_AND_BIT = 1; - public final static int TUNNEL_AND_BIT = 2; - public byte wayAndBits = -1; // use for bridge/tunnel logic - + public final static int NO_BRIDGE_BIT = 1; + public final static int NO_TUNNEL_BIT = 2; + public final static int LCN_BIT = 4; + public final static int CR_BIT = 8; + + public byte wayBits = 0; // interface OsmPos public int getILat() @@ -61,7 +67,7 @@ public class OsmNodeP implements Comparable public short getSElev() { // if all bridge or all tunnel, elevation=no-data - return (wayAndBits & ( BRIDGE_AND_BIT | TUNNEL_AND_BIT ) ) == 0 ? selev : Short.MIN_VALUE; + return ( wayBits & NO_BRIDGE_BIT ) == 0 || ( wayBits & NO_TUNNEL_BIT ) == 0 ? Short.MIN_VALUE : selev; } public double getElev() @@ -82,22 +88,23 @@ public class OsmNodeP implements Comparable return null; } - public void writeNodeData( DataOutputStream os, boolean writeVarLength ) throws IOException + public void writeNodeData( ByteDataWriter os, boolean writeVarLength, byte[] abBuf ) throws IOException { int lonIdx = ilon/62500; int latIdx = ilat/62500; // buffer the body to first calc size - ByteArrayOutputStream bos = new ByteArrayOutputStream( ); - DataOutputStream os2 = new DataOutputStream( bos ); - + ByteDataWriter os2 = new ByteDataWriter( abBuf ); os2.writeShort( getSElev() ); - + // hack: write node-desc as link tag (copy cycleway-bits) byte[] nodeDescription = getNodeDecsription(); for( OsmLinkP link0 = firstlink; link0 != null; link0 = link0.next ) { + int ilonref = ilon; + int ilatref = ilat; + OsmLinkP link = link0; OsmNodeP origin = this; int skipDetailBit = link0.counterLinkWritten() ? SKIPDETAILS_BITMASK : 0; @@ -160,32 +167,24 @@ public class OsmNodeP implements Comparable } } - int targetLonIdx = target.ilon/62500; - int targetLatIdx = target.ilat/62500; int bm = tranferbit | writedescbit | nodedescbit | skipDetailBit; - if ( writeVarLength ) bm |= VARIABLEDESC_BITMASK; + int dlon = target.ilon - ilonref; + int dlat = target.ilat - ilatref; + ilonref = target.ilon; + ilatref = target.ilat; + if ( dlon < 0 ) { bm |= SIGNLON_BITMASK; dlon = - dlon; } + if ( dlat < 0 ) { bm |= SIGNLAT_BITMASK; dlat = - dlat; } + os2.writeByte( bm ); + + int blon = os2.writeVarLengthUnsigned( dlon ); + int blat = os2.writeVarLengthUnsigned( dlat ); - if ( targetLonIdx == lonIdx && targetLatIdx == latIdx ) - { - // reduced position for internal target - os2.writeByte( bm ); - os2.writeShort( (short)(target.ilon - lonIdx*62500 - 31250) ); - os2.writeShort( (short)(target.ilat - latIdx*62500 - 31250) ); - } - else - { - // full position for external target - os2.writeByte( bm | EXTERNAL_BITMASK ); - os2.writeInt( target.ilon ); - os2.writeInt( target.ilat ); - } if ( writedescbit != 0 ) { // write the way description, code direction into the first bit - int len = lastDescription.length; - if ( writeVarLength ) os2.writeByte( len ); - os2.write( lastDescription, 0, len ); + if ( writeVarLength ) os2.writeByte( lastDescription.length ); + os2.write( lastDescription ); } if ( nodedescbit != 0 ) { @@ -199,7 +198,7 @@ public class OsmNodeP implements Comparable target.markLinkWritten( origin ); break; } - os2.writeShort( target.getSElev() ); + os2.writeVarLengthSigned( target.getSElev() -getSElev() ); // next link is the one (of two), does does'nt point back for( link = target.firstlink; link != null; link = link.next ) { @@ -211,12 +210,15 @@ public class OsmNodeP implements Comparable } // calculate the body size - int bodySize = bos.size(); + int bodySize = os2.size(); + + os.ensureCapacity( bodySize + 8 ); os.writeShort( (short)(ilon - lonIdx*62500 - 31250) ); os.writeShort( (short)(ilat - latIdx*62500 - 31250) ); - os.writeInt( bodySize ); - bos.writeTo( os ); + + os.writeVarLengthUnsigned( bodySize ); + os.write( abBuf, 0, bodySize ); } public String toString2() diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/OsmNodePT.java b/brouter-map-creator/src/main/java/btools/mapcreator/OsmNodePT.java index 989c4d4..f8eda68 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/OsmNodePT.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/OsmNodePT.java @@ -10,8 +10,6 @@ public class OsmNodePT extends OsmNodeP { public byte[] descriptionBits; - public byte wayOrBits = 0; // used to propagate bike networks to nodes - public OsmNodePT() { } @@ -25,7 +23,6 @@ public class OsmNodePT extends OsmNodeP public final byte[] getNodeDecsription() { return descriptionBits; - // return descriptionBits | (long)( (wayOrBits & 6) >> 1 ); TODO !!!!!!!!!!1 } @Override diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/PosUnifier.java b/brouter-map-creator/src/main/java/btools/mapcreator/PosUnifier.java index 0aa79df..205a5b9 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/PosUnifier.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/PosUnifier.java @@ -7,6 +7,7 @@ import java.io.File; import java.util.HashMap; import btools.util.CompactLongSet; +import btools.util.DiffCoderDataOutputStream; import btools.util.FrozenLongSet; /** @@ -21,8 +22,8 @@ import btools.util.FrozenLongSet; */ public class PosUnifier extends MapCreatorBase { - private DataOutputStream nodesOutStream; - private DataOutputStream borderNodesOut; + private DiffCoderDataOutputStream nodesOutStream; + private DiffCoderDataOutputStream borderNodesOut; private File nodeTilesOut; private CompactLongSet positionSet; diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java b/brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java index 601d195..39557b9 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/RelationMerger.java @@ -7,6 +7,7 @@ import java.io.File; import java.util.HashMap; import btools.expressions.BExpressionContext; +import btools.expressions.BExpressionMetaData; import btools.util.CompactLongSet; import btools.util.FrozenLongSet; @@ -29,7 +30,7 @@ public class RelationMerger extends MapCreatorBase public static void main(String[] args) throws Exception { - System.out.println("*** RelationMerger: merge relation bits into ways" ); + System.out.println("*** RelationMerger: merge relations into ways" ); if (args.length != 6) { System.out.println("usage: java RelationMerger " ); @@ -42,11 +43,15 @@ public class RelationMerger extends MapCreatorBase public void process( File wayFileIn, File wayFileOut, File relationFileIn, File lookupFile, File reportProfile, File checkProfile ) throws Exception { // read lookup + profile for relation access-check - expctxReport = new BExpressionContext("way"); - expctxReport.readMetaData( lookupFile ); + BExpressionMetaData metaReport = new BExpressionMetaData(); + expctxReport = new BExpressionContext("way", metaReport ); + metaReport.readMetaData( lookupFile ); + + BExpressionMetaData metaCheck = new BExpressionMetaData(); + expctxCheck = new BExpressionContext("way", metaCheck ); + metaCheck.readMetaData( lookupFile ); + expctxReport.parseFile( reportProfile, "global" ); - expctxCheck = new BExpressionContext("way"); - expctxCheck.readMetaData( lookupFile ); expctxCheck.parseFile( checkProfile, "global" ); // expctxStat = new BExpressionContext("way"); @@ -63,6 +68,7 @@ public class RelationMerger extends MapCreatorBase String network = dis.readUTF(); String tagname = "route_" + route + "_" + network; + CompactLongSet routeset = null; if ( expctxCheck.getLookupNameIdx(tagname) >= 0 ) { @@ -127,6 +133,7 @@ public class RelationMerger extends MapCreatorBase if ( ok ) { + expctxReport.decode( data.description ); for( String tagname : routesets.keySet() ) { CompactLongSet routeset = routesets.get( tagname ); diff --git a/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java b/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java index 317ec31..2fdb6d4 100644 --- a/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java +++ b/brouter-map-creator/src/main/java/btools/mapcreator/WayLinker.java @@ -8,7 +8,9 @@ import java.util.Collections; import java.util.List; import btools.expressions.BExpressionContext; +import btools.expressions.BExpressionMetaData; import btools.util.ByteArrayUnifier; +import btools.util.ByteDataWriter; import btools.util.CompactLongMap; import btools.util.CompactLongSet; import btools.util.Crc32; @@ -45,6 +47,7 @@ public class WayLinker extends MapCreatorBase private long creationTimeStamp; private BExpressionContext expctxWay; + private BExpressionContext expctxNode; private ByteArrayUnifier abUnifier; @@ -77,13 +80,19 @@ public class WayLinker extends MapCreatorBase this.borderFileIn = borderFileIn; this.dataTilesSuffix = dataTilesSuffix; + BExpressionMetaData meta = new BExpressionMetaData(); + // read lookup + profile for lookup-version + access-filter - expctxWay = new BExpressionContext("way"); - expctxWay.readMetaData( lookupFile ); - lookupVersion = expctxWay.lookupVersion; - lookupMinorVersion = expctxWay.lookupMinorVersion; - writeVarLength = expctxWay.readVarLength; + expctxWay = new BExpressionContext("way", meta); + expctxNode = new BExpressionContext("node", meta); + meta.readMetaData( lookupFile ); + + lookupVersion = meta.lookupVersion; + lookupMinorVersion = meta.lookupMinorVersion; + writeVarLength = meta.readVarLength; + expctxWay.parseFile( profileFile, "global" ); + expctxNode.parseFile( profileFile, "global" ); creationTimeStamp = System.currentTimeMillis(); @@ -158,11 +167,11 @@ public class WayLinker extends MapCreatorBase ok |= expctxWay.getCostfactor() < 10000.; if ( !ok ) return; - byte bridgeTunnel = 0; + byte wayBits = 0; expctxWay.decode( description ); - if ( expctxWay.getBooleanLookupValue( "bridge" ) ) bridgeTunnel |= OsmNodeP.BRIDGE_AND_BIT; - if ( expctxWay.getBooleanLookupValue( "tunnel" ) ) bridgeTunnel |= OsmNodeP.TUNNEL_AND_BIT; - + if ( !expctxWay.getBooleanLookupValue( "bridge" ) ) wayBits |= OsmNodeP.NO_BRIDGE_BIT; + if ( !expctxWay.getBooleanLookupValue( "tunnel" ) ) wayBits |= OsmNodeP.NO_TUNNEL_BIT; + OsmNodeP n1 = null; OsmNodeP n2 = null; for (int i=0; i size / 2 ) // garbage collection @@ -252,53 +268,6 @@ final class MicroCache return positions; } - public int readInt() - { - int i3 = ab[aboffset++]& 0xff; - int i2 = ab[aboffset++]& 0xff; - int i1 = ab[aboffset++]& 0xff; - int i0 = ab[aboffset++]& 0xff; - return (i3 << 24) + (i2 << 16) + (i1 << 8) + i0; - } - - public long readLong() - { - long i7 = ab[aboffset++]& 0xff; - long i6 = ab[aboffset++]& 0xff; - long i5 = ab[aboffset++]& 0xff; - long i4 = ab[aboffset++]& 0xff; - long i3 = ab[aboffset++]& 0xff; - long i2 = ab[aboffset++]& 0xff; - long i1 = ab[aboffset++]& 0xff; - long i0 = ab[aboffset++]& 0xff; - return (i7 << 56) + (i6 << 48) + (i5 << 40) + (i4 << 32) + (i3 << 24) + (i2 << 16) + (i1 << 8) + i0; - } - - public boolean readBoolean() - { - int i0 = ab[aboffset++]& 0xff; - return i0 != 0; - } - - public byte readByte() - { - int i0 = ab[aboffset++] & 0xff; - return (byte)(i0); - } - - public short readShort() - { - int i1 = ab[aboffset++] & 0xff; - int i0 = ab[aboffset++] & 0xff; - return (short)( (i1 << 8) | i0); - } - - public void readFully( byte[] ta ) - { - System.arraycopy( ab, aboffset, ta, 0, ta.length ); - aboffset += ta.length; - } - public boolean hasMoreData() { return aboffset < aboffsetEnd; diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java b/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java index 95da78d..612ca1f 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java @@ -17,6 +17,7 @@ public final class NodesCache private OsmNodesMap nodesMap; private int lookupVersion; private int lookupMinorVersion; + private boolean readVarLength; private boolean carMode; private String currentFileName; @@ -33,12 +34,13 @@ public final class NodesCache private long cacheSum = 0; private boolean garbageCollectionEnabled = false; - public NodesCache( String segmentDir, OsmNodesMap nodesMap, int lookupVersion, int lookupMinorVersion, boolean carMode, NodesCache oldCache ) + public NodesCache( String segmentDir, OsmNodesMap nodesMap, int lookupVersion, int minorVersion, boolean varLen, boolean carMode, NodesCache oldCache ) { this.segmentDir = segmentDir; this.nodesMap = nodesMap; this.lookupVersion = lookupVersion; - this.lookupMinorVersion = lookupMinorVersion; + this.lookupMinorVersion = minorVersion; + this.readVarLength = varLen; this.carMode = carMode; if ( oldCache != null ) @@ -145,7 +147,7 @@ public final class NodesCache checkEnableCacheCleaning(); - segment = new MicroCache( osmf, lonIdx80, latIdx80, iobuffer ); + segment = new MicroCache( osmf, lonIdx80, latIdx80, iobuffer, readVarLength ); cacheSum += segment.getDataSize(); osmf.microCaches[subIdx] = segment; segmentList.add( segment ); diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java index 9868119..22726ff 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java @@ -7,6 +7,8 @@ package btools.mapaccess; import java.io.IOException; import java.io.RandomAccessFile; + +import btools.util.ByteDataReader; import btools.util.Crc32; final class OsmFile diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java index 2f24233..fbd35ad 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmNode.java @@ -11,8 +11,9 @@ import btools.util.ByteArrayUnifier; public class OsmNode implements OsmPos { - public static final int EXTERNAL_BITMASK = 0x80; - public static final int VARIABLEDESC_BITMASK = 0x40; + public static final int EXTERNAL_BITMASK = 0x80; // old semantic + public static final int SIGNLON_BITMASK = 0x80; + public static final int SIGNLAT_BITMASK = 0x40; public static final int TRANSFERNODE_BITMASK = 0x20; public static final int WRITEDESC_BITMASK = 0x10; public static final int SKIPDETAILS_BITMASK = 0x08; @@ -111,10 +112,10 @@ public class OsmNode implements OsmPos } - public void parseNodeBody( MicroCache is, OsmNodesMap hollowNodes, DistanceChecker dc ) + public void parseNodeBody( MicroCache is, OsmNodesMap hollowNodes, DistanceChecker dc, boolean readVarLength ) { ByteArrayUnifier abUnifier = hollowNodes.getByteArrayUnifier(); - + selev = is.readShort(); OsmLink lastlink = null; @@ -124,6 +125,9 @@ public class OsmNode implements OsmPos while( is.hasMoreData() ) { + int ilonref = ilon; + int ilatref = ilat; + OsmLink link = new OsmLink(); OsmTransferNode firstTransferNode = null; OsmTransferNode lastTransferNode = null; @@ -133,31 +137,45 @@ public class OsmNode implements OsmPos for(;;) { int bitField = is.readByte(); - if ( (bitField & EXTERNAL_BITMASK) != 0 ) + // System.out.println( "parseNodeBody: var=" + readVarLength + " bitField=" + bitField ); + if ( readVarLength ) { - // full position for external target - linklon = is.readInt(); - linklat = is.readInt(); + int dlon = is.readVarLengthUnsigned(); + int dlat = is.readVarLengthUnsigned(); + if ( (bitField & SIGNLON_BITMASK) != 0 ) { dlon = -dlon;} + if ( (bitField & SIGNLAT_BITMASK) != 0 ) { dlat = -dlat;} + linklon = ilonref + dlon; + linklat = ilatref + dlat; + ilonref = linklon; + ilatref = linklat; } else { - // reduced position for internal target - linklon = is.readShort(); - linklat = is.readShort(); - linklon += lonIdx*62500 + 31250; - linklat += latIdx*62500 + 31250; + if ( (bitField & EXTERNAL_BITMASK) != 0 ) + { + // full position for external target + linklon = is.readInt(); + linklat = is.readInt(); + } + else + { + // reduced position for internal target + linklon = is.readShort(); + linklat = is.readShort(); + linklon += lonIdx*62500 + 31250; + linklat += latIdx*62500 + 31250; + } } // read variable length or old 8 byte fixed, and ensure that 8 bytes is only fixed - boolean readFix8 = (bitField & VARIABLEDESC_BITMASK ) == 0; // old, fix length format if ( (bitField & WRITEDESC_BITMASK ) != 0 ) { - byte[] ab = new byte[readFix8 ? 8 : is.readByte()]; + byte[] ab = new byte[readVarLength ? is.readByte() : 8 ]; is.readFully( ab ); description = abUnifier.unify( ab ); } if ( (bitField & NODEDESC_BITMASK ) != 0 ) { - byte[] ab = new byte[readFix8 ? 8 : is.readByte()]; + byte[] ab = new byte[readVarLength ? is.readByte() : 8 ]; is.readFully( ab ); nodeDescription = abUnifier.unify( ab ); } @@ -185,7 +203,7 @@ public class OsmNode implements OsmPos trans.ilon = linklon; trans.ilat = linklat; trans.descriptionBitmap = description; - trans.selev = is.readShort(); + trans.selev = readVarLength ? (short)(selev + is.readVarLengthSigned()) : is.readShort(); if ( lastTransferNode == null ) { firstTransferNode = trans; diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmTransferNode.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmTransferNode.java index 5f916ee..50ad8e0 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmTransferNode.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmTransferNode.java @@ -5,6 +5,9 @@ */ package btools.mapaccess; +import btools.util.ByteDataReader; +import btools.util.ByteDataWriter; + public final class OsmTransferNode diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/PhysicalFile.java b/brouter-mapaccess/src/main/java/btools/mapaccess/PhysicalFile.java index 11276b4..8ecaa77 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/PhysicalFile.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/PhysicalFile.java @@ -6,6 +6,8 @@ package btools.mapaccess; import java.io.*; + +import btools.util.ByteDataReader; import btools.util.Crc32; final public class PhysicalFile @@ -37,7 +39,7 @@ final public class PhysicalFile if ( osmf.microCaches != null ) for( int lonIdx80=0; lonIdx80<80; lonIdx80++ ) for( int latIdx80=0; latIdx80<80; latIdx80++ ) - new MicroCache( osmf, lonIdx80, latIdx80, iobuffer ); + new MicroCache( osmf, lonIdx80, latIdx80, iobuffer, true ); // TODO: readVarLength ? } } catch( IllegalArgumentException iae ) diff --git a/brouter-routing-app/AndroidManifest.xml b/brouter-routing-app/AndroidManifest.xml index f4ec7ab..7e66d62 100644 --- a/brouter-routing-app/AndroidManifest.xml +++ b/brouter-routing-app/AndroidManifest.xml @@ -1,8 +1,8 @@ + android:versionCode="3" + android:versionName="1.0.1" package="btools.routingapp"> 3 + following 2-bit word ( 3..6 ) // 0001 -> 7 + following 3-bit word ( 7..14 ) etc. - public void encodeDistance( int value ) + public void encodeVarBits( int value ) { int range = 0; while ( value > range ) @@ -33,7 +33,7 @@ package btools.util; } // twin to encodeDistance - public int decodeDistance() + public int decodeVarBits() { int range = 0; int value = 0; diff --git a/brouter-util/src/main/java/btools/util/ByteArrayUnifier.java b/brouter-util/src/main/java/btools/util/ByteArrayUnifier.java index 5a9c692..a43388d 100644 --- a/brouter-util/src/main/java/btools/util/ByteArrayUnifier.java +++ b/brouter-util/src/main/java/btools/util/ByteArrayUnifier.java @@ -9,7 +9,11 @@ public final class ByteArrayUnifier public ByteArrayUnifier( int size, boolean validateImmutability ) { this.size = size; - byteArrayCache = new byte[size][]; + + if ( !Boolean.getBoolean( "disableByteArrayUnifification" ) ) + { + byteArrayCache = new byte[size][]; + } if ( validateImmutability ) crcCrosscheck = new int[size]; } @@ -22,6 +26,8 @@ public final class ByteArrayUnifier */ public byte[] unify( byte[] ab ) { + if ( byteArrayCache == null ) return ab; + int n = ab.length; int crc = Crc32.crc( ab, 0, n ); int idx = (crc & 0xfffffff) % size; diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/ByteDataReader.java b/brouter-util/src/main/java/btools/util/ByteDataReader.java similarity index 67% rename from brouter-mapaccess/src/main/java/btools/mapaccess/ByteDataReader.java rename to brouter-util/src/main/java/btools/util/ByteDataReader.java index 7ad5e25..7bc2dd8 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/ByteDataReader.java +++ b/brouter-util/src/main/java/btools/util/ByteDataReader.java @@ -1,75 +1,95 @@ -/** - * fast data-reading from a byte-array - * - * @author ab - */ -package btools.mapaccess; - - -final class ByteDataReader -{ - private byte[] ab; - private int aboffset; - - public ByteDataReader( byte[] byteArray ) - { - ab = byteArray; - } - - public int readInt() - { - int i3 = ab[aboffset++]& 0xff; - int i2 = ab[aboffset++]& 0xff; - int i1 = ab[aboffset++]& 0xff; - int i0 = ab[aboffset++]& 0xff; - return (i3 << 24) + (i2 << 16) + (i1 << 8) + i0; - } - - public long readLong() - { - long i7 = ab[aboffset++]& 0xff; - long i6 = ab[aboffset++]& 0xff; - long i5 = ab[aboffset++]& 0xff; - long i4 = ab[aboffset++]& 0xff; - long i3 = ab[aboffset++]& 0xff; - long i2 = ab[aboffset++]& 0xff; - long i1 = ab[aboffset++]& 0xff; - long i0 = ab[aboffset++]& 0xff; - return (i7 << 56) + (i6 << 48) + (i5 << 40) + (i4 << 32) + (i3 << 24) + (i2 << 16) + (i1 << 8) + i0; - } - - public boolean readBoolean() - { - int i0 = ab[aboffset++]& 0xff; - return i0 != 0; - } - - public byte readByte() - { - int i0 = ab[aboffset++] & 0xff; - return (byte)(i0); - } - - public short readShort() - { - int i1 = ab[aboffset++] & 0xff; - int i0 = ab[aboffset++] & 0xff; - return (short)( (i1 << 8) | i0); - } - - public void readFully( byte[] ta ) - { - System.arraycopy( ab, aboffset, ta, 0, ta.length ); - aboffset += ta.length; - } - - @Override - public String toString() - { - StringBuilder sb = new StringBuilder( "[" ); - for( int i=0; i> 1 : -(v >> 1 ); + } + + public final int readVarLengthUnsigned() + { + int v = 0; + int shift = 0; + for(;;) + { + int i7 = ab[aboffset++] & 0xff; + v |= (( i7 & 0x7f ) << shift); + if ( ( i7 & 0x80 ) == 0 ) break; + shift += 7; + } + return v; + } + + public final void readFully( byte[] ta ) + { + System.arraycopy( ab, aboffset, ta, 0, ta.length ); + aboffset += ta.length; + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder( "[" ); + for( int i=0; i> 24) & 0xff ); - ab[aboffset++] = (byte)( (v >> 16) & 0xff ); - ab[aboffset++] = (byte)( (v >> 8) & 0xff ); - ab[aboffset++] = (byte)( (v ) & 0xff ); - } - - public void writeLong( long v ) - { - ab[aboffset++] = (byte)( (v >> 56) & 0xff ); - ab[aboffset++] = (byte)( (v >> 48) & 0xff ); - ab[aboffset++] = (byte)( (v >> 40) & 0xff ); - ab[aboffset++] = (byte)( (v >> 32) & 0xff ); - ab[aboffset++] = (byte)( (v >> 24) & 0xff ); - ab[aboffset++] = (byte)( (v >> 16) & 0xff ); - ab[aboffset++] = (byte)( (v >> 8) & 0xff ); - ab[aboffset++] = (byte)( (v ) & 0xff ); - } - - public void writeBoolean( boolean v) - { - ab[aboffset++] = (byte)( v ? 1 : 0 ); - } - - public void writeByte( int v ) - { - ab[aboffset++] = (byte)( (v ) & 0xff ); - } - - public void writeShort( int v ) - { - ab[aboffset++] = (byte)( (v >> 8) & 0xff ); - ab[aboffset++] = (byte)( (v ) & 0xff ); - } - - public void write( byte[] sa ) - { - System.arraycopy( sa, 0, ab, aboffset, sa.length ); - aboffset += sa.length; - } - - @Override - public String toString() - { - StringBuilder sb = new StringBuilder( "[" ); - for( int i=0; i> 24) & 0xff ); + ab[aboffset++] = (byte)( (v >> 16) & 0xff ); + ab[aboffset++] = (byte)( (v >> 8) & 0xff ); + ab[aboffset++] = (byte)( (v ) & 0xff ); + } + + public void writeLong( long v ) + { + ab[aboffset++] = (byte)( (v >> 56) & 0xff ); + ab[aboffset++] = (byte)( (v >> 48) & 0xff ); + ab[aboffset++] = (byte)( (v >> 40) & 0xff ); + ab[aboffset++] = (byte)( (v >> 32) & 0xff ); + ab[aboffset++] = (byte)( (v >> 24) & 0xff ); + ab[aboffset++] = (byte)( (v >> 16) & 0xff ); + ab[aboffset++] = (byte)( (v >> 8) & 0xff ); + ab[aboffset++] = (byte)( (v ) & 0xff ); + } + + public void writeBoolean( boolean v) + { + ab[aboffset++] = (byte)( v ? 1 : 0 ); + } + + public void writeByte( int v ) + { + ab[aboffset++] = (byte)( (v ) & 0xff ); + } + + public void writeShort( int v ) + { + ab[aboffset++] = (byte)( (v >> 8) & 0xff ); + ab[aboffset++] = (byte)( (v ) & 0xff ); + } + + public void write( byte[] sa ) + { + System.arraycopy( sa, 0, ab, aboffset, sa.length ); + aboffset += sa.length; + } + + public void write( byte[] sa, int offset, int len ) + { + System.arraycopy( sa, offset, ab, aboffset, len ); + aboffset += len; + } + + public void ensureCapacity( int len ) + { + // TODO + } + + public byte[] toByteArray() + { + byte[] c = new byte[aboffset]; + System.arraycopy( ab, 0, c, 0, aboffset ); + return c; + } + + public int writeVarLengthSigned( int v ) + { + return writeVarLengthUnsigned( v < 0 ? ( (-v) << 1 ) | 1 : v << 1 ); + } + + public int writeVarLengthUnsigned( int v ) + { + int start = aboffset; + do + { + int i7 = v & 0x7f; + v >>= 7; + if ( v != 0 ) i7 |= 0x80; + ab[aboffset++] = (byte)( i7 & 0xff ); + } + while( v != 0 ); + return aboffset - start; + } + + public int size() + { + return aboffset; + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder( "[" ); + for( int i=0; i>> 8) ^ crctable[(crc ^ ab[j]) & 0xff]; } diff --git a/brouter-util/src/main/java/btools/util/DiffCoderDataInputStream.java b/brouter-util/src/main/java/btools/util/DiffCoderDataInputStream.java new file mode 100644 index 0000000..9fba290 --- /dev/null +++ b/brouter-util/src/main/java/btools/util/DiffCoderDataInputStream.java @@ -0,0 +1,49 @@ +/** + * DataInputStream extended by varlength diff coding + * + * @author ab + */ +package btools.util; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; + + +public final class DiffCoderDataInputStream extends DataInputStream +{ + private long[] lastValues = new long[10]; + + public DiffCoderDataInputStream( InputStream is ) + { + super( is ); + } + + public long readDiffed( int idx ) throws IOException + { + long d = readSigned(); + long v = lastValues[idx] + d; + lastValues[idx] = v; + return v; + } + + public long readSigned() throws IOException + { + long v = readUnsigned(); + return ( v & 1 ) == 0 ? v >> 1 : -(v >> 1 ); + } + + public long readUnsigned() throws IOException + { + long v = 0; + int shift = 0; + for(;;) + { + long i7 = readByte() & 0xff; + v |= (( i7 & 0x7f ) << shift); + if ( ( i7 & 0x80 ) == 0 ) break; + shift += 7; + } + return v; + } +} diff --git a/brouter-util/src/main/java/btools/util/DiffCoderDataOutputStream.java b/brouter-util/src/main/java/btools/util/DiffCoderDataOutputStream.java new file mode 100644 index 0000000..b3e8e75 --- /dev/null +++ b/brouter-util/src/main/java/btools/util/DiffCoderDataOutputStream.java @@ -0,0 +1,45 @@ +/** + * DataOutputStream extended by varlength diff coding + * + * @author ab + */ +package btools.util; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; + + +public final class DiffCoderDataOutputStream extends DataOutputStream +{ + private long[] lastValues = new long[10]; + + public DiffCoderDataOutputStream( OutputStream os ) + { + super( os ); + } + + public void writeDiffed( long v, int idx ) throws IOException + { + long d = v - lastValues[idx]; + lastValues[idx] = v; + writeSigned( d ); + } + + public void writeSigned( long v ) throws IOException + { + writeUnsigned( v < 0 ? ( (-v) << 1 ) | 1 : v << 1 ); + } + + public void writeUnsigned( long v ) throws IOException + { + do + { + long i7 = v & 0x7f; + v >>= 7; + if ( v != 0 ) i7 |= 0x80; + writeByte( (byte)( i7 & 0xff ) ); + } + while( v != 0 ); + } +} diff --git a/brouter-util/src/test/java/btools/util/BitCoderContextTest.java b/brouter-util/src/test/java/btools/util/BitCoderContextTest.java index 735f0c0..36b3b0a 100644 --- a/brouter-util/src/test/java/btools/util/BitCoderContextTest.java +++ b/brouter-util/src/test/java/btools/util/BitCoderContextTest.java @@ -9,19 +9,19 @@ import org.junit.Test; public class BitCoderContextTest { @Test - public void distanceEncodeDecodeTest() + public void varBitsEncodeDecodeTest() { byte[] ab = new byte[4000]; BitCoderContext ctx = new BitCoderContext( ab ); for( int i=0; i<1000; i++ ) { - ctx.encodeDistance( i ); + ctx.encodeVarBits( i ); } ctx = new BitCoderContext( ab ); for( int i=0; i<1000; i++ ) { - int value = ctx.decodeDistance(); + int value = ctx.decodeVarBits(); Assert.assertTrue( "distance value mismatch", value == i ); } } diff --git a/brouter-util/src/test/java/btools/util/ByteDataIOTest.java b/brouter-util/src/test/java/btools/util/ByteDataIOTest.java new file mode 100644 index 0000000..b6f1474 --- /dev/null +++ b/brouter-util/src/test/java/btools/util/ByteDataIOTest.java @@ -0,0 +1,28 @@ +package btools.util; + +import java.util.Random; +import java.util.HashSet; + +import org.junit.Assert; +import org.junit.Test; + +public class ByteDataIOTest +{ + @Test + public void varLengthEncodeDecodeTest() + { + byte[] ab = new byte[4000]; + ByteDataWriter w = new ByteDataWriter( ab ); + for( int i=0; i<1000; i++ ) + { + w.writeVarLengthUnsigned( i ); + } + ByteDataReader r = new ByteDataReader( ab ); + + for( int i=0; i<1000; i++ ) + { + int value = r.readVarLengthUnsigned(); + Assert.assertTrue( "value mismatch", value == i ); + } + } +}