diff --git a/brouter-codec/src/main/java/btools/codec/MicroCache.java b/brouter-codec/src/main/java/btools/codec/MicroCache.java index 2cb4a8c..8c01988 100644 --- a/brouter-codec/src/main/java/btools/codec/MicroCache.java +++ b/brouter-codec/src/main/java/btools/codec/MicroCache.java @@ -314,4 +314,56 @@ public class MicroCache extends ByteDataWriter } return null; } + + public void calcDelta( MicroCache mc1, MicroCache mc2 ) + { + int idx1 = 0; + int idx2 = 0; + + while( idx1 < mc1.size || idx2 < mc2.size ) + { + int id1 = idx1 < mc1.size ? mc1.faid[idx1] : Integer.MAX_VALUE; + int id2 = idx2 < mc2.size ? mc2.faid[idx2] : Integer.MAX_VALUE; + int id; + if ( id1 >= id2 ) + { + id = id2; + int start2 = idx2 > 0 ? mc2.fapos[idx2 - 1] : 0; + int len2 = mc2.fapos[idx2++] - start2; + + if ( id1 == id2 ) + { + // id exists in both caches, compare data + int start1 = idx1 > 0 ? mc1.fapos[idx1 - 1] : 0; + int len1 = mc1.fapos[idx1++] - start1; + if ( len1 == len2 ) + { + int i = 0; + while( i do nothing + } + } + } + write( mc2.ab, start2, len2 ); + } + else + { + idx1++; + id = id1; // deleted node + } + fapos[size] = aboffset; + faid[size] = id; + size++; + } + } + } diff --git a/brouter-codec/src/main/java/btools/codec/MicroCache2.java b/brouter-codec/src/main/java/btools/codec/MicroCache2.java index b4178fe..5c2bc83 100644 --- a/brouter-codec/src/main/java/btools/codec/MicroCache2.java +++ b/brouter-codec/src/main/java/btools/codec/MicroCache2.java @@ -87,10 +87,14 @@ public final class MicroCache2 extends MicroCache // future escapes (turn restrictions?) short trExceptions = 0; - for(;;) + int featureId = bc.decodeVarBits(); + if ( featureId == 13 ) + { + fapos[n] = aboffset; + continue; // empty node escape (delta files only) + } + while( featureId != 0 ) { - int featureId = bc.decodeVarBits(); - if ( featureId == 0 ) break; int bitsize = bc.decodeNoisyNumber( 5 ); if ( featureId == 2 ) // exceptions to turn-restriction @@ -113,6 +117,7 @@ public final class MicroCache2 extends MicroCache { for( int i=0; i< bitsize; i++ ) bc.decodeBit(); // unknown feature, just skip } + featureId = bc.decodeVarBits(); } writeBoolean( false ); @@ -147,7 +152,8 @@ public final class MicroCache2 extends MicroCache TagValueWrapper wayTags = wayTagCoder.decodeTagValueSet(); - if ( wayTags != null ) + boolean linkValid = wayTags != null || wayValidator == null; + if ( linkValid ) { int startPointer = aboffset; sizeoffset = writeSizePlaceHolder(); @@ -162,7 +168,7 @@ public final class MicroCache2 extends MicroCache finaldatasize += 1 + aboffset-startPointer; // reserve place for reverse validBits[ nodeIdx >> 5 ] |= 1 << nodeIdx; // mark target-node valid } - writeModeAndDesc( isReverse, wayTags.data ); + writeModeAndDesc( isReverse, wayTags == null ? null : wayTags.data ); } if ( !isReverse ) // write geometry for forward links only @@ -200,7 +206,7 @@ public final class MicroCache2 extends MicroCache } if ( matcher != null ) matcher.end(); } - if ( wayTags != null ) + if ( linkValid ) { injectSize( sizeoffset ); } @@ -375,6 +381,12 @@ public final class MicroCache2 extends MicroCache int ilon = (int)(id64 >> 32); int ilat = (int)(id64 & 0xffffffff); + if ( aboffset == aboffsetEnd ) + { + bc.encodeVarBits( 13 ); // empty node escape (delta files only) + continue; + } + // write turn restrictions while( readBoolean() ) { @@ -430,7 +442,10 @@ public final class MicroCache2 extends MicroCache readFully( description ); } - boolean isInternal = isInternal( ilonlink, ilatlink ); + long link64 = ((long)ilonlink)<<32 | ilatlink; + Integer idx = idMap.get( Long.valueOf( link64 ) ); + boolean isInternal = idx != null; + if ( isReverse && isInternal ) { if ( dodebug ) System.out.println( "*** NOT encoding link reverse=" + isReverse + " internal=" + isInternal ); @@ -442,9 +457,6 @@ public final class MicroCache2 extends MicroCache if ( isInternal ) { - long link64 = ((long)ilonlink)<<32 | ilatlink; - Integer idx = idMap.get( Long.valueOf( link64 ) ); - if ( idx == null ) throw new RuntimeException( "ups: internal not found?" ); int nodeIdx = idx.intValue(); if ( dodebug ) System.out.println( "*** target nodeIdx=" + nodeIdx ); if ( nodeIdx == n ) throw new RuntimeException( "ups: self ref?" ); diff --git a/brouter-codec/src/main/java/btools/codec/TagValueCoder.java b/brouter-codec/src/main/java/btools/codec/TagValueCoder.java index a733085..1118000 100644 --- a/brouter-codec/src/main/java/btools/codec/TagValueCoder.java +++ b/brouter-codec/src/main/java/btools/codec/TagValueCoder.java @@ -62,6 +62,11 @@ public final class TagValueCoder { if ( ++pass == 3 ) { + if ( identityMap.size() == 0 ) + { + TagValueSet dummy = new TagValueSet(); + identityMap.put( dummy, dummy ); + } PriorityQueue queue = new PriorityQueue(2*identityMap.size(), new TagValueSet.FrequencyComparator()); queue.addAll(identityMap.values()); while (queue.size() > 1) diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java new file mode 100644 index 0000000..937fd51 --- /dev/null +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java @@ -0,0 +1,130 @@ +/** + * Proof of concept for delta rd5's + * + * @author ab + */ +package btools.mapaccess; + +import java.io.File; + +import btools.codec.DataBuffers; +import btools.codec.MicroCache; +import btools.codec.MicroCache2; +import btools.codec.StatCoderContext; + +final public class Rd5DiffTool +{ + public static void main( String[] args ) throws Exception + { + diff2files( new File( args[0] ),new File( args[1] ) ); + } + + /** + * Compute the delta between 2 RD5 files and + * show statistics on the expected size of the delta file + */ + public static void diff2files( File f1, File f2 ) throws Exception + { + byte[] abBuf1 = new byte[10 * 1024 * 1024]; + byte[] abBuf2 = new byte[10 * 1024 * 1024]; + + int nodesTotal = 0; + int nodesDiff = 0; + + long bytesDiff = 0L; + + PhysicalFile pf1 = null; + PhysicalFile pf2 = null; + try + { + DataBuffers dataBuffers = new DataBuffers(); + pf1 = new PhysicalFile( f1, dataBuffers, -1, -1 ); + pf2 = new PhysicalFile( f2, dataBuffers, -1, -1 ); + int div = pf1.divisor; + for ( int lonDegree = 0; lonDegree < 5; lonDegree++ ) // does'nt really matter.. + { + for ( int latDegree = 0; latDegree < 5; latDegree++ ) // ..where on earth we are + { + OsmFile osmf1 = new OsmFile( pf1, lonDegree, latDegree, dataBuffers ); + OsmFile osmf2 = new OsmFile( pf2, lonDegree, latDegree, dataBuffers ); + for ( int lonIdx = 0; lonIdx < div; lonIdx++ ) + { + for ( int latIdx = 0; latIdx < div; latIdx++ ) + { + int lonIdxDiv = lonDegree * div + lonIdx; + int latIdxDiv = latDegree * div + latIdx; + + + MicroCache mc1 = osmf1.hasData() ? + osmf1.createMicroCache( lonIdxDiv, latIdxDiv, dataBuffers, null, null, true, null ) + : MicroCache.emptyCache(); + MicroCache mc2 = osmf2.hasData() ? + osmf2.createMicroCache( lonIdxDiv, latIdxDiv, dataBuffers, null, null, true, null ) + : MicroCache.emptyCache(); + + MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, lonIdxDiv, latIdxDiv, div ); + mc.calcDelta( mc1, mc2 ); + + nodesTotal += mc2.getSize(); + + if ( latIdx == 15 ) + { + // System.out.println( "hier!" ); + } + + if ( mc.getSize() > 0 ) + { + int len = mc.encodeMicroCache( abBuf1 ); + byte[] bytes = new byte[len]; + System.arraycopy( abBuf1, 0, bytes, 0, len ); + + bytesDiff += len; + nodesDiff += mc.getSize(); + + // cross-check the encoding: re-instantiate the cache + MicroCache mcCheck = new MicroCache2( new StatCoderContext( bytes ), new DataBuffers( null ), lonIdxDiv, latIdxDiv, div, null, null ); + + // ..and check if still the same + if ( mc.size() != mcCheck.size() ) + { + // mc.compareWith finds link-ordering differences, + // so we compare only if there's also a size missmatch... + + String diffMessage = mc.compareWith( mcCheck ); + if ( diffMessage != null ) + { + throw new RuntimeException( "files differ: " + diffMessage ); + } + } + } + } + } + } + } + System.out.println( "nodesTotal=" + nodesTotal + " nodesDiff=" + nodesDiff + " bytesDiff=" + bytesDiff ); + } + finally + { + if ( pf1 != null ) + { + try + { + pf1.ra.close(); + } + catch (Exception ee) + { + } + } + if ( pf2 != null ) + { + try + { + pf2.ra.close(); + } + catch (Exception ee) + { + } + } + } + } +}