diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java new file mode 100644 index 0000000..76607fc --- /dev/null +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java @@ -0,0 +1,119 @@ +/** + * Proof of concept for delta rd5's + * + * @author ab + */ +package btools.mapaccess; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.security.DigestInputStream; +import java.security.MessageDigest; + +final public class Rd5DiffManager +{ + public static void main( String[] args ) throws Exception + { + calcDiffs( new File( args[0] ),new File( args[1] ) ); + } + + /** + * Compute diffs for all RD5 files + */ + public static void calcDiffs( File oldDir, File newDir ) throws Exception + { + File oldDiffDir = new File( oldDir, "diff" ); + File newDiffDir = new File( newDir, "diff" ); + + File[] filesNew = newDir.listFiles(); + + for( File fn : filesNew ) + { + String name = fn.getName(); + if ( !name.endsWith( ".rd5" ) ) + { + continue; + } + if ( fn.length() < 1024*1024 ) + { + continue; // exclude very small files from diffing + } + String basename = name.substring( 0, name.length() - 4 ); + File fo = new File( oldDir, name ); + if ( !fo.isFile() ) + { + continue; + } + + // calculate MD5 of old file + String md5 = getMD5( fo ); + + System.out.println( "name=" + name + " md5=" + md5 ); + + File specificNewDiffs = new File( newDiffDir, basename ); + specificNewDiffs.mkdirs(); + + String diffFileName = md5 + ".rd5diff"; + + File diffFile = new File( specificNewDiffs, diffFileName ); + + // calc the new diff + Rd5DiffTool.diff2files( fo, fn, diffFile ); + + // ... and add that to old diff files + File specificOldDiffs = new File( oldDiffDir, basename ); + if ( specificOldDiffs.isDirectory() ) + { + File[] oldDiffs = specificOldDiffs.listFiles(); + for( File od : oldDiffs ) + { + if ( !od.getName().endsWith( ".rd5diff" ) ) + { + continue; + } + if ( System.currentTimeMillis() - od.lastModified() > 31*86400000L ) + { + continue; // limit diff history to 31 days + } + + File updatedDiff = new File( specificNewDiffs, od.getName() ); + + Rd5DiffTool.addDeltas( od, diffFile, updatedDiff ); + updatedDiff.setLastModified( od.lastModified() ); + } + } + } + } + + public static String getMD5( File f ) throws Exception + { + MessageDigest md = MessageDigest.getInstance("MD5"); + BufferedInputStream bis = new BufferedInputStream( new FileInputStream( f ) ); + DigestInputStream dis = new DigestInputStream(bis, md); + byte[] buf = new byte[8192]; + for(;;) + { + int len = dis.read( buf ); + if ( len <= 0 ) + { + break; + } + } + dis.close(); + byte[] bytes = md.digest(); + + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < bytes.length; j++) + { + int v = bytes[j] & 0xff; + sb.append( hexChar( v >>> 4 ) ).append( hexChar( v & 0xf ) ); + } + return sb.toString(); + } + + private static char hexChar( int v ) + { + return (char) ( v > 9 ? 'a' + (v-10) : '0' + v ); + } +} diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java index 265c61d..5c744bd 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java @@ -138,6 +138,8 @@ final public class Rd5DiffTool long[] fileIndex1 = readFileIndex( dis1, null ); long[] fileIndex2 = readFileIndex( dis2, dos ); + long t0 = System.currentTimeMillis(); + try { DataBuffers dataBuffers = new DataBuffers(); @@ -181,14 +183,14 @@ final public class Rd5DiffTool dos.writeInt( len ); dos.write( abBuf1, 0, len ); - // do some consistemcy checks on the encoding + bytesDiff += len; + nodesDiff += mc.getSize(); + diffedTiles++; + +/* // do some consistemcy checks on the encoding byte[] bytes = new byte[len]; System.arraycopy( abBuf1, 0, bytes, 0, len ); - - bytesDiff += len; - nodesDiff += mc.getSize(); - diffedTiles++; // cross-check the encoding: decode again MicroCache mcCheck = new MicroCache2( new StatCoderContext( bytes ), new DataBuffers( null ), 0, 0, 32, null, null ); @@ -220,6 +222,7 @@ final public class Rd5DiffTool throw new IllegalArgumentException( "decoded data mismatch at i=" + i ); } } + */ } } } @@ -233,7 +236,8 @@ final public class Rd5DiffTool } dos.write( abBuf1, 0, len ); } - System.out.println( "nodesDiff=" + nodesDiff + " bytesDiff=" + bytesDiff + " diffedTiles=" + diffedTiles ); + long t1 = System.currentTimeMillis(); + System.out.println( "nodesDiff=" + nodesDiff + " bytesDiff=" + bytesDiff + " diffedTiles=" + diffedTiles + " took " + (t1-t0) + "ms" ); } finally { @@ -416,6 +420,8 @@ final public class Rd5DiffTool long[] fileIndex1 = readFileIndex( dis1, null ); long[] fileIndex2 = readFileIndex( dis2, dos ); + long t0 = System.currentTimeMillis(); + try { DataBuffers dataBuffers = new DataBuffers(); @@ -475,6 +481,8 @@ final public class Rd5DiffTool } dos.write( abBuf1, 0, len ); } + long t1 = System.currentTimeMillis(); + System.out.println( "adding diffs took " + (t1-t0) + "ms" ); } finally {