rd5 delta progress

This commit is contained in:
Arndt Brenschede 2020-01-18 17:30:33 +01:00
parent c4f3e7dadf
commit 19a7555483
4 changed files with 468 additions and 46 deletions

View file

@ -366,4 +366,38 @@ public class MicroCache extends ByteDataWriter
}
}
public void addDelta( MicroCache mc1, MicroCache mc2, boolean keepEmptyNodes )
{
int idx1 = 0;
int idx2 = 0;
while( idx1 < mc1.size || idx2 < mc2.size )
{
int id1 = idx1 < mc1.size ? mc1.faid[idx1] : Integer.MAX_VALUE;
int id2 = idx2 < mc2.size ? mc2.faid[idx2] : Integer.MAX_VALUE;
if ( id1 >= id2 ) // data from diff file wins
{
int start2 = idx2 > 0 ? mc2.fapos[idx2 - 1] : 0;
int len2 = mc2.fapos[idx2++] - start2;
if ( keepEmptyNodes || len2 > 0 )
{
write( mc2.ab, start2, len2 );
fapos[size] = aboffset;
faid[size++] = id2;
}
if ( id1 == id2 ) // // id exists in both caches
{
idx1++;
}
}
else // use data from base file
{
int start1 = idx1 > 0 ? mc1.fapos[idx1 - 1] : 0;
int len1 = mc1.fapos[idx1++] - start1;
write( mc1.ab, start1, len1 );
fapos[size] = aboffset;
faid[size++] = id1;
}
}
}
}

View file

@ -266,6 +266,25 @@ public final class TagValueCoder
return -1;
if ( tvs1.frequency > tvs2.frequency )
return 1;
// to avoid ordering instability, decide on the data
// if frequency is equal
int l1 = tvs1.data == null ? 0 : tvs1.data.length;
int l2 = tvs2.data == null ? 0 : tvs2.data.length;
if ( l1 < l2 )
return -1;
if ( l1 > l2 )
return 1;
for( int i=0; i<l1; i++ )
{
byte b1 = tvs1.data[i];
byte b2 = tvs2.data[i];
if ( b1 < b2 )
return -1;
if ( b1 > b2 )
return 1;
}
return 0;
}
}

View file

@ -5,83 +5,193 @@
*/
package btools.mapaccess;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.Arrays;
import btools.codec.DataBuffers;
import btools.codec.MicroCache;
import btools.codec.MicroCache2;
import btools.codec.StatCoderContext;
import btools.util.Crc32;
final public class Rd5DiffTool
{
public static void main( String[] args ) throws Exception
{
diff2files( new File( args[0] ),new File( args[1] ) );
if ( args[1].endsWith( ".rd5diff" ) )
{
if ( args[0].endsWith( ".rd5diff" ) )
{
addDeltas( new File( args[0] ),new File( args[1] ), new File( args[2] ) );
}
else
{
recoverFromDelta( new File( args[0] ),new File( args[1] ), new File( args[2] ) /*, new File( args[3] ) */ );
}
}
else
{
diff2files( new File( args[0] ),new File( args[1] ), new File( args[2] ) );
}
}
private static long[] readFileIndex( DataInputStream dis, DataOutputStream dos ) throws Exception
{
long[] fileIndex = new long[25];
for( int i=0; i<25; i++ )
{
long lv = dis.readLong();
fileIndex[i] = lv & 0xffffffffffffL;
if ( dos != null )
{
dos.writeLong( lv );
}
}
return fileIndex;
}
private static long getTileStart( long[] index, int tileIndex )
{
return tileIndex > 0 ? index[tileIndex - 1] : 200L;
}
private static long getTileEnd( long[] index, int tileIndex )
{
return index[tileIndex];
}
private static int[] readPosIndex( DataInputStream dis, DataOutputStream dos ) throws Exception
{
int[] posIndex = new int[1024];
for( int i=0; i<1024; i++ )
{
int iv = dis.readInt();
posIndex[i] = iv;
if ( dos != null )
{
dos.writeInt( iv );
}
}
return posIndex;
}
private static int getPosIdx( int[] posIdx, int idx )
{
return idx == -1 ? 4096 : posIdx[idx];
}
private static byte[] createMicroCache( int[] posIdx, int tileIdx, DataInputStream dis, boolean deltaMode ) throws Exception
{
if ( posIdx == null )
{
return null;
}
int size = getPosIdx( posIdx, tileIdx ) - getPosIdx( posIdx, tileIdx-1 );
if ( size == 0 )
{
return null;
}
if ( deltaMode )
{
size = dis.readInt();
}
byte[] ab = new byte[size];
dis.readFully( ab );
return ab;
}
private static MicroCache createMicroCache( byte[] ab, DataBuffers dataBuffers ) throws Exception
{
if ( ab == null || ab.length == 0 )
{
return MicroCache.emptyCache();
}
StatCoderContext bc = new StatCoderContext( ab );
return new MicroCache2( bc, dataBuffers, 0, 0, 32, null, null );
}
/**
* Compute the delta between 2 RD5 files and
* show statistics on the expected size of the delta file
*/
public static void diff2files( File f1, File f2 ) throws Exception
public static void diff2files( File f1, File f2, File outFile ) throws Exception
{
byte[] abBuf1 = new byte[10 * 1024 * 1024];
byte[] abBuf2 = new byte[10 * 1024 * 1024];
int nodesTotal = 0;
int nodesDiff = 0;
int diffedTiles = 0;
long bytesDiff = 0L;
long diffedTileSize = 0L;
PhysicalFile pf1 = null;
PhysicalFile pf2 = null;
DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
// copy header to outfile
long[] fileIndex1 = readFileIndex( dis1, null );
long[] fileIndex2 = readFileIndex( dis2, dos );
try
{
DataBuffers dataBuffers = new DataBuffers();
pf1 = new PhysicalFile( f1, dataBuffers, -1, -1 );
pf2 = new PhysicalFile( f2, dataBuffers, -1, -1 );
int div = pf1.divisor;
for ( int lonDegree = 0; lonDegree < 5; lonDegree++ ) // does'nt really matter..
for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ )
{
for ( int latDegree = 0; latDegree < 5; latDegree++ ) // ..where on earth we are
{
OsmFile osmf1 = new OsmFile( pf1, lonDegree, latDegree, dataBuffers );
OsmFile osmf2 = new OsmFile( pf2, lonDegree, latDegree, dataBuffers );
for ( int lonIdx = 0; lonIdx < div; lonIdx++ )
{
for ( int latIdx = 0; latIdx < div; latIdx++ )
{
int lonIdxDiv = lonDegree * div + lonIdx;
int latIdxDiv = latDegree * div + latIdx;
boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx );
boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx );
int[] posIdx1 = hasData1 ? readPosIndex( dis1, null ) : null;
int[] posIdx2 = hasData2 ? readPosIndex( dis2, dos ) : null;
for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ )
{
byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false );
byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, false );
if ( ab2 == null )
{
continue; // empty target tile
}
MicroCache mc;
if ( Arrays.equals( ab1, ab2 ) )
{
mc = MicroCache.emptyCache(); // empty diff
}
else // calc diff of the 2 tiles
{
MicroCache mc1 = createMicroCache( ab1, dataBuffers );
MicroCache mc2 = createMicroCache( ab2, dataBuffers );
mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
mc.calcDelta( mc1, mc2 );
}
MicroCache mc1 = osmf1.hasData() ?
osmf1.createMicroCache( lonIdxDiv, latIdxDiv, dataBuffers, null, null, true, null )
: MicroCache.emptyCache();
MicroCache mc2 = osmf2.hasData() ?
osmf2.createMicroCache( lonIdxDiv, latIdxDiv, dataBuffers, null, null, true, null )
: MicroCache.emptyCache();
MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, lonIdxDiv, latIdxDiv, div );
mc.calcDelta( mc1, mc2 );
nodesTotal += mc2.getSize();
if ( mc.getSize() > 0 )
{
int len = mc.encodeMicroCache( abBuf1 );
if ( mc.getSize() == 0 )
{
dos.writeInt( 0 );
}
else
{
int len = mc.encodeMicroCache( abBuf1 );
dos.writeInt( len );
dos.write( abBuf1, 0, len );
// do some consistemcy checks on the encoding
byte[] bytes = new byte[len];
System.arraycopy( abBuf1, 0, bytes, 0, len );
bytesDiff += len;
nodesDiff += mc.getSize();
diffedTiles++;
diffedTileSize += mc2.size();
// cross-check the encoding: decode again
MicroCache mcCheck = new MicroCache2( new StatCoderContext( bytes ), new DataBuffers( null ), lonIdxDiv, latIdxDiv, div, null, null );
MicroCache mcCheck = new MicroCache2( new StatCoderContext( bytes ), new DataBuffers( null ), 0, 0, 32, null, null );
// due to link-order ambiguity, for decoded we can only compare node-count and datasize
if ( mc.size() != mcCheck.size() )
@ -110,30 +220,289 @@ final public class Rd5DiffTool
throw new IllegalArgumentException( "decoded data mismatch at i=" + i );
}
}
}
}
}
}
}
System.out.println( "nodesTotal=" + nodesTotal + " nodesDiff=" + nodesDiff + " bytesDiff=" + bytesDiff + " diffedTiles=" + diffedTiles + " diffedTileSize=" + diffedTileSize );
// write any remaining data to the output file
for(;;)
{
int len = dis2.read( abBuf1 );
if (len < 0)
{
break;
}
dos.write( abBuf1, 0, len );
}
System.out.println( "nodesDiff=" + nodesDiff + " bytesDiff=" + bytesDiff + " diffedTiles=" + diffedTiles );
}
finally
{
if ( pf1 != null )
if ( dis1 != null )
{
try
{
pf1.ra.close();
dis1.close();
}
catch (Exception ee)
{
}
}
if ( pf2 != null )
if ( dis2 != null )
{
try
{
pf2.ra.close();
dis2.close();
}
catch (Exception ee)
{
}
}
if ( dos != null )
{
try
{
dos.close();
}
catch (Exception ee)
{
}
}
}
}
public static void recoverFromDelta( File f1, File f2, File outFile /* , File cmpFile */ ) throws Exception
{
byte[] abBuf1 = new byte[10 * 1024 * 1024];
byte[] abBuf2 = new byte[10 * 1024 * 1024];
DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
// DataInputStream disCmp = new DataInputStream( new BufferedInputStream( new FileInputStream( cmpFile ) ) );
DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
// copy header to outfile
long[] fileIndex1 = readFileIndex( dis1, null );
long[] fileIndex2 = readFileIndex( dis2, dos );
// long[] fileIndexCmp = readFileIndex( disCmp, null );
try
{
DataBuffers dataBuffers = new DataBuffers();
for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ )
{
boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx );
boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx );
// boolean hasDataCmp = getTileStart( fileIndexCmp, subFileIdx ) < getTileEnd( fileIndexCmp, subFileIdx );
int[] posIdx1 = hasData1 ? readPosIndex( dis1, null ) : null;
int[] posIdx2 = hasData2 ? readPosIndex( dis2, dos ) : null;
// int[] posIdxCmp = hasDataCmp ? readPosIndex( disCmp, null ) : null;
for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ )
{
byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false );
byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true );
if ( ab2 == null )
{
continue; // no target tile expected
}
/* int targetSizeCmp = getPosIdx( posIdxCmp, tileIdx ) - getPosIdx( posIdxCmp, tileIdx-1 );
if ( targetSizeCmp != targetSize ) throw new IllegalArgumentException( "target size mismatch: "+ targetSize + "," + targetSizeCmp );
byte[] abCmp = new byte[targetSizeCmp];
disCmp.readFully( abCmp );
*/
// no-delta shortcut: just copy base data
if ( ab2.length == 0 )
{
if ( ab1 != null )
{
dos.write( ab1 );
}
continue;
}
// this is the real delta case (using decode->delta->encode )
MicroCache mc1 = createMicroCache( ab1, dataBuffers );
MicroCache mc2 = createMicroCache( ab2, dataBuffers );
MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
mc.addDelta( mc1, mc2, false );
int len = mc.encodeMicroCache( abBuf1 );
/* System.out.println( "comparing for subFileIdx=" + subFileIdx + " tileIdx=" + tileIdx );
boolean isequal = true;
for( int i=0; i<len;i++ )
{
if ( isequal && abCmp[i] != abBuf1[i] )
{
System.out.println( "data mismatch at i=" + i + " " + abCmp[i] + "!=" + abBuf1[i] + " targetSize=" + targetSize );
isequal = false;
MicroCache.debug = true;
System.out.println( "**** decoding original cache ****" );
createMicroCache( abCmp, dataBuffers );
System.out.println( "**** decoding reconstructed cache ****" );
createMicroCache( abBuf1, dataBuffers );
System.exit(1);
}
}
*/
dos.write( abBuf1, 0, len );
dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 );
}
}
// write any remaining data to the output file
for(;;)
{
int len = dis2.read( abBuf1 );
if (len < 0)
{
break;
}
dos.write( abBuf1, 0, len );
}
}
finally
{
if ( dis1 != null )
{
try
{
dis1.close();
}
catch (Exception ee)
{
}
}
if ( dis2 != null )
{
try
{
dis2.close();
}
catch (Exception ee)
{
}
}
if ( dos != null )
{
try
{
dos.close();
}
catch (Exception ee)
{
}
}
}
}
public static void addDeltas( File f1, File f2, File outFile ) throws Exception
{
byte[] abBuf1 = new byte[10 * 1024 * 1024];
byte[] abBuf2 = new byte[10 * 1024 * 1024];
DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
// copy header to outfile
long[] fileIndex1 = readFileIndex( dis1, null );
long[] fileIndex2 = readFileIndex( dis2, dos );
try
{
DataBuffers dataBuffers = new DataBuffers();
for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ )
{
boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx );
boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx );
int[] posIdx1 = hasData1 ? readPosIndex( dis1, null ) : null;
int[] posIdx2 = hasData2 ? readPosIndex( dis2, dos ) : null;
for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ )
{
byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, true );
byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true );
if ( ab2 == null )
{
continue; // no target tile expected
}
// no-delta shortcut: just copy base data
if ( ab2.length == 0 )
{
if ( ab1 == null )
{
dos.writeInt( 0 );
}
else
{
dos.writeInt( ab1.length );
dos.write( ab1 );
}
continue;
}
// this is the real delta case (using decode->add->encode )
MicroCache mc1 = createMicroCache( ab1, dataBuffers );
MicroCache mc2 = createMicroCache( ab2, dataBuffers );
MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
mc.addDelta( mc1, mc2, true );
int len = mc.encodeMicroCache( abBuf1 );
dos.writeInt( len+4 );
dos.write( abBuf1, 0, len );
dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 );
}
}
// write any remaining data to the output file
for(;;)
{
int len = dis2.read( abBuf1 );
if (len < 0)
{
break;
}
dos.write( abBuf1, 0, len );
}
}
finally
{
if ( dis1 != null )
{
try
{
dis1.close();
}
catch (Exception ee)
{
}
}
if ( dis2 != null )
{
try
{
dis2.close();
}
catch (Exception ee)
{
}
}
if ( dos != null )
{
try
{
dos.close();
}
catch (Exception ee)
{

View file

@ -503,7 +503,7 @@ public class SuspectManager extends Thread
{
bw.write( filter + " suspect list for " + country + "\n" );
bw.write( "<br><a href=\"/brouter/suspects" + challenge + country + "/" + filter + "/watchlist\">see watchlist</a>\n" );
bw.write( "<br><a href=\"/brouter/suspects\">back to country list</a><br><br>\n" );
bw.write( "<br><a href=\"/brouter/suspects" + challenge + "\">back to country list</a><br><br>\n" );
int maxprio = 0;
{
for( int isuspect = 0; isuspect<suspects.cnt; isuspect++ )