diff --git a/brouter-codec/src/main/java/btools/codec/MicroCache.java b/brouter-codec/src/main/java/btools/codec/MicroCache.java index 8c01988..7e22e3b 100644 --- a/brouter-codec/src/main/java/btools/codec/MicroCache.java +++ b/brouter-codec/src/main/java/btools/codec/MicroCache.java @@ -366,4 +366,38 @@ public class MicroCache extends ByteDataWriter } } + public void addDelta( MicroCache mc1, MicroCache mc2, boolean keepEmptyNodes ) + { + int idx1 = 0; + int idx2 = 0; + + while( idx1 < mc1.size || idx2 < mc2.size ) + { + int id1 = idx1 < mc1.size ? mc1.faid[idx1] : Integer.MAX_VALUE; + int id2 = idx2 < mc2.size ? mc2.faid[idx2] : Integer.MAX_VALUE; + if ( id1 >= id2 ) // data from diff file wins + { + int start2 = idx2 > 0 ? mc2.fapos[idx2 - 1] : 0; + int len2 = mc2.fapos[idx2++] - start2; + if ( keepEmptyNodes || len2 > 0 ) + { + write( mc2.ab, start2, len2 ); + fapos[size] = aboffset; + faid[size++] = id2; + } + if ( id1 == id2 ) // // id exists in both caches + { + idx1++; + } + } + else // use data from base file + { + int start1 = idx1 > 0 ? mc1.fapos[idx1 - 1] : 0; + int len1 = mc1.fapos[idx1++] - start1; + write( mc1.ab, start1, len1 ); + fapos[size] = aboffset; + faid[size++] = id1; + } + } + } } diff --git a/brouter-codec/src/main/java/btools/codec/TagValueCoder.java b/brouter-codec/src/main/java/btools/codec/TagValueCoder.java index 1118000..77c6bac 100644 --- a/brouter-codec/src/main/java/btools/codec/TagValueCoder.java +++ b/brouter-codec/src/main/java/btools/codec/TagValueCoder.java @@ -266,6 +266,25 @@ public final class TagValueCoder return -1; if ( tvs1.frequency > tvs2.frequency ) return 1; + + // to avoid ordering instability, decide on the data + // if frequency is equal + int l1 = tvs1.data == null ? 0 : tvs1.data.length; + int l2 = tvs2.data == null ? 0 : tvs2.data.length; + + if ( l1 < l2 ) + return -1; + if ( l1 > l2 ) + return 1; + for( int i=0; i b2 ) + return 1; + } return 0; } } diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java index f6d5bee..265c61d 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java @@ -5,83 +5,193 @@ */ package btools.mapaccess; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.Arrays; import btools.codec.DataBuffers; import btools.codec.MicroCache; import btools.codec.MicroCache2; import btools.codec.StatCoderContext; +import btools.util.Crc32; final public class Rd5DiffTool { public static void main( String[] args ) throws Exception { - diff2files( new File( args[0] ),new File( args[1] ) ); + if ( args[1].endsWith( ".rd5diff" ) ) + { + if ( args[0].endsWith( ".rd5diff" ) ) + { + addDeltas( new File( args[0] ),new File( args[1] ), new File( args[2] ) ); + } + else + { + recoverFromDelta( new File( args[0] ),new File( args[1] ), new File( args[2] ) /*, new File( args[3] ) */ ); + } + } + else + { + diff2files( new File( args[0] ),new File( args[1] ), new File( args[2] ) ); + } + } + + private static long[] readFileIndex( DataInputStream dis, DataOutputStream dos ) throws Exception + { + long[] fileIndex = new long[25]; + for( int i=0; i<25; i++ ) + { + long lv = dis.readLong(); + fileIndex[i] = lv & 0xffffffffffffL; + if ( dos != null ) + { + dos.writeLong( lv ); + } + } + return fileIndex; + } + + private static long getTileStart( long[] index, int tileIndex ) + { + return tileIndex > 0 ? index[tileIndex - 1] : 200L; + } + + private static long getTileEnd( long[] index, int tileIndex ) + { + return index[tileIndex]; + } + + private static int[] readPosIndex( DataInputStream dis, DataOutputStream dos ) throws Exception + { + int[] posIndex = new int[1024]; + for( int i=0; i<1024; i++ ) + { + int iv = dis.readInt(); + posIndex[i] = iv; + if ( dos != null ) + { + dos.writeInt( iv ); + } + } + return posIndex; + } + + private static int getPosIdx( int[] posIdx, int idx ) + { + return idx == -1 ? 4096 : posIdx[idx]; + } + + private static byte[] createMicroCache( int[] posIdx, int tileIdx, DataInputStream dis, boolean deltaMode ) throws Exception + { + if ( posIdx == null ) + { + return null; + } + int size = getPosIdx( posIdx, tileIdx ) - getPosIdx( posIdx, tileIdx-1 ); + if ( size == 0 ) + { + return null; + } + if ( deltaMode ) + { + size = dis.readInt(); + } + byte[] ab = new byte[size]; + dis.readFully( ab ); + return ab; + } + + private static MicroCache createMicroCache( byte[] ab, DataBuffers dataBuffers ) throws Exception + { + if ( ab == null || ab.length == 0 ) + { + return MicroCache.emptyCache(); + } + StatCoderContext bc = new StatCoderContext( ab ); + return new MicroCache2( bc, dataBuffers, 0, 0, 32, null, null ); } /** * Compute the delta between 2 RD5 files and * show statistics on the expected size of the delta file */ - public static void diff2files( File f1, File f2 ) throws Exception + public static void diff2files( File f1, File f2, File outFile ) throws Exception { byte[] abBuf1 = new byte[10 * 1024 * 1024]; byte[] abBuf2 = new byte[10 * 1024 * 1024]; - int nodesTotal = 0; int nodesDiff = 0; int diffedTiles = 0; long bytesDiff = 0L; - long diffedTileSize = 0L; - PhysicalFile pf1 = null; - PhysicalFile pf2 = null; + DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) ); + DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) ); + DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) ); + + // copy header to outfile + long[] fileIndex1 = readFileIndex( dis1, null ); + long[] fileIndex2 = readFileIndex( dis2, dos ); + try { DataBuffers dataBuffers = new DataBuffers(); - pf1 = new PhysicalFile( f1, dataBuffers, -1, -1 ); - pf2 = new PhysicalFile( f2, dataBuffers, -1, -1 ); - int div = pf1.divisor; - for ( int lonDegree = 0; lonDegree < 5; lonDegree++ ) // does'nt really matter.. + for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ ) { - for ( int latDegree = 0; latDegree < 5; latDegree++ ) // ..where on earth we are - { - OsmFile osmf1 = new OsmFile( pf1, lonDegree, latDegree, dataBuffers ); - OsmFile osmf2 = new OsmFile( pf2, lonDegree, latDegree, dataBuffers ); - for ( int lonIdx = 0; lonIdx < div; lonIdx++ ) - { - for ( int latIdx = 0; latIdx < div; latIdx++ ) - { - int lonIdxDiv = lonDegree * div + lonIdx; - int latIdxDiv = latDegree * div + latIdx; + boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx ); + boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx ); + + int[] posIdx1 = hasData1 ? readPosIndex( dis1, null ) : null; + int[] posIdx2 = hasData2 ? readPosIndex( dis2, dos ) : null; + + for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ ) + { + byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false ); + byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, false ); + + if ( ab2 == null ) + { + continue; // empty target tile + } + MicroCache mc; + if ( Arrays.equals( ab1, ab2 ) ) + { + mc = MicroCache.emptyCache(); // empty diff + } + else // calc diff of the 2 tiles + { + MicroCache mc1 = createMicroCache( ab1, dataBuffers ); + MicroCache mc2 = createMicroCache( ab2, dataBuffers ); + mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 ); + mc.calcDelta( mc1, mc2 ); + } - - MicroCache mc1 = osmf1.hasData() ? - osmf1.createMicroCache( lonIdxDiv, latIdxDiv, dataBuffers, null, null, true, null ) - : MicroCache.emptyCache(); - MicroCache mc2 = osmf2.hasData() ? - osmf2.createMicroCache( lonIdxDiv, latIdxDiv, dataBuffers, null, null, true, null ) - : MicroCache.emptyCache(); - - MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, lonIdxDiv, latIdxDiv, div ); - mc.calcDelta( mc1, mc2 ); - - nodesTotal += mc2.getSize(); - - if ( mc.getSize() > 0 ) - { - int len = mc.encodeMicroCache( abBuf1 ); + if ( mc.getSize() == 0 ) + { + dos.writeInt( 0 ); + } + else + { + int len = mc.encodeMicroCache( abBuf1 ); + dos.writeInt( len ); + dos.write( abBuf1, 0, len ); + + // do some consistemcy checks on the encoding + byte[] bytes = new byte[len]; System.arraycopy( abBuf1, 0, bytes, 0, len ); bytesDiff += len; nodesDiff += mc.getSize(); diffedTiles++; - diffedTileSize += mc2.size(); // cross-check the encoding: decode again - MicroCache mcCheck = new MicroCache2( new StatCoderContext( bytes ), new DataBuffers( null ), lonIdxDiv, latIdxDiv, div, null, null ); + MicroCache mcCheck = new MicroCache2( new StatCoderContext( bytes ), new DataBuffers( null ), 0, 0, 32, null, null ); // due to link-order ambiguity, for decoded we can only compare node-count and datasize if ( mc.size() != mcCheck.size() ) @@ -110,30 +220,289 @@ final public class Rd5DiffTool throw new IllegalArgumentException( "decoded data mismatch at i=" + i ); } } - } - } } } } - System.out.println( "nodesTotal=" + nodesTotal + " nodesDiff=" + nodesDiff + " bytesDiff=" + bytesDiff + " diffedTiles=" + diffedTiles + " diffedTileSize=" + diffedTileSize ); + // write any remaining data to the output file + for(;;) + { + int len = dis2.read( abBuf1 ); + if (len < 0) + { + break; + } + dos.write( abBuf1, 0, len ); + } + System.out.println( "nodesDiff=" + nodesDiff + " bytesDiff=" + bytesDiff + " diffedTiles=" + diffedTiles ); } finally { - if ( pf1 != null ) + if ( dis1 != null ) { try { - pf1.ra.close(); + dis1.close(); } catch (Exception ee) { } } - if ( pf2 != null ) + if ( dis2 != null ) { try { - pf2.ra.close(); + dis2.close(); + } + catch (Exception ee) + { + } + } + if ( dos != null ) + { + try + { + dos.close(); + } + catch (Exception ee) + { + } + } + } + } + + + public static void recoverFromDelta( File f1, File f2, File outFile /* , File cmpFile */ ) throws Exception + { + byte[] abBuf1 = new byte[10 * 1024 * 1024]; + byte[] abBuf2 = new byte[10 * 1024 * 1024]; + + DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) ); + DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) ); +// DataInputStream disCmp = new DataInputStream( new BufferedInputStream( new FileInputStream( cmpFile ) ) ); + DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) ); + + // copy header to outfile + long[] fileIndex1 = readFileIndex( dis1, null ); + long[] fileIndex2 = readFileIndex( dis2, dos ); +// long[] fileIndexCmp = readFileIndex( disCmp, null ); + + try + { + DataBuffers dataBuffers = new DataBuffers(); + for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ ) + { + boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx ); + boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx ); + +// boolean hasDataCmp = getTileStart( fileIndexCmp, subFileIdx ) < getTileEnd( fileIndexCmp, subFileIdx ); + + int[] posIdx1 = hasData1 ? readPosIndex( dis1, null ) : null; + int[] posIdx2 = hasData2 ? readPosIndex( dis2, dos ) : null; + + // int[] posIdxCmp = hasDataCmp ? readPosIndex( disCmp, null ) : null; + + for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ ) + { + byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false ); + byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true ); + if ( ab2 == null ) + { + continue; // no target tile expected + } + +/* int targetSizeCmp = getPosIdx( posIdxCmp, tileIdx ) - getPosIdx( posIdxCmp, tileIdx-1 ); + if ( targetSizeCmp != targetSize ) throw new IllegalArgumentException( "target size mismatch: "+ targetSize + "," + targetSizeCmp ); + byte[] abCmp = new byte[targetSizeCmp]; + disCmp.readFully( abCmp ); +*/ + + // no-delta shortcut: just copy base data + if ( ab2.length == 0 ) + { + if ( ab1 != null ) + { + dos.write( ab1 ); + } + continue; + } + + // this is the real delta case (using decode->delta->encode ) + MicroCache mc1 = createMicroCache( ab1, dataBuffers ); + MicroCache mc2 = createMicroCache( ab2, dataBuffers ); + + MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 ); + mc.addDelta( mc1, mc2, false ); + + int len = mc.encodeMicroCache( abBuf1 ); + +/* System.out.println( "comparing for subFileIdx=" + subFileIdx + " tileIdx=" + tileIdx ); + boolean isequal = true; + for( int i=0; iadd->encode ) + MicroCache mc1 = createMicroCache( ab1, dataBuffers ); + MicroCache mc2 = createMicroCache( ab2, dataBuffers ); + + MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 ); + mc.addDelta( mc1, mc2, true ); + + int len = mc.encodeMicroCache( abBuf1 ); + + dos.writeInt( len+4 ); + dos.write( abBuf1, 0, len ); + dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 ); + } + } + // write any remaining data to the output file + for(;;) + { + int len = dis2.read( abBuf1 ); + if (len < 0) + { + break; + } + dos.write( abBuf1, 0, len ); + } + } + finally + { + if ( dis1 != null ) + { + try + { + dis1.close(); + } + catch (Exception ee) + { + } + } + if ( dis2 != null ) + { + try + { + dis2.close(); + } + catch (Exception ee) + { + } + } + if ( dos != null ) + { + try + { + dos.close(); } catch (Exception ee) { diff --git a/brouter-server/src/main/java/btools/server/SuspectManager.java b/brouter-server/src/main/java/btools/server/SuspectManager.java index 1055350..5d8f772 100644 --- a/brouter-server/src/main/java/btools/server/SuspectManager.java +++ b/brouter-server/src/main/java/btools/server/SuspectManager.java @@ -503,7 +503,7 @@ public class SuspectManager extends Thread { bw.write( filter + " suspect list for " + country + "\n" ); bw.write( "
see watchlist\n" ); - bw.write( "
back to country list

\n" ); + bw.write( "
back to country list

\n" ); int maxprio = 0; { for( int isuspect = 0; isuspect