rd5 delta progress

brouter-codec/src/main/java/btools/codec/MicroCache.java

@@ -366,4 +366,38 @@ public class MicroCache extends ByteDataWriter
      }
   }
 
+  public void addDelta( MicroCache mc1, MicroCache mc2, boolean keepEmptyNodes )
+  {
+     int idx1 = 0;
+     int idx2 = 0;
+
+     while( idx1 < mc1.size || idx2 < mc2.size )
+     {
+       int id1 = idx1 < mc1.size ? mc1.faid[idx1] : Integer.MAX_VALUE;
+       int id2 = idx2 < mc2.size ? mc2.faid[idx2] : Integer.MAX_VALUE;
+       if ( id1 >= id2 ) // data from diff file wins
+       {
+         int start2 = idx2 > 0 ? mc2.fapos[idx2 - 1] : 0;
+         int len2 = mc2.fapos[idx2++] - start2;
+         if ( keepEmptyNodes || len2 > 0 )
+         {
+           write( mc2.ab, start2, len2 );
+           fapos[size] = aboffset;
+           faid[size++] = id2;
+         }
+         if ( id1 == id2 ) // // id exists in both caches
+         {
+           idx1++;
+         }
+       }
+       else // use data from base file
+       {
+         int start1 = idx1 > 0 ? mc1.fapos[idx1 - 1] : 0;
+         int len1 = mc1.fapos[idx1++] - start1;
+         write( mc1.ab, start1, len1 );
+         fapos[size] = aboffset;
+         faid[size++] = id1;
+       }
+     }
+  }
 }

brouter-codec/src/main/java/btools/codec/TagValueCoder.java

@@ -266,6 +266,25 @@ public final class TagValueCoder
           return -1;
         if ( tvs1.frequency > tvs2.frequency )
           return 1;
+
+        // to avoid ordering instability, decide on the data
+        // if frequency is equal
+        int l1 = tvs1.data == null ? 0 : tvs1.data.length;
+        int l2 = tvs2.data == null ? 0 : tvs2.data.length;
+
+        if ( l1 < l2 )
+          return -1;
+        if ( l1 > l2 )
+          return 1;
+        for( int i=0; i<l1; i++ )
+        {
+          byte b1 = tvs1.data[i];
+          byte b2 = tvs2.data[i];
+          if ( b1 < b2 )
+            return -1;
+          if ( b1 > b2 )
+            return 1;
+        }
         return 0;
       }
     }

brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java

@@ -5,83 +5,193 @@
  */
 package btools.mapaccess;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.util.Arrays;
 
 import btools.codec.DataBuffers;
 import btools.codec.MicroCache;
 import btools.codec.MicroCache2;
 import btools.codec.StatCoderContext;
+import btools.util.Crc32;
 
 final public class Rd5DiffTool
 {
   public static void main( String[] args ) throws Exception
   {
-    diff2files( new File( args[0] ),new File( args[1] ) );
+    if ( args[1].endsWith( ".rd5diff" ) )
+    {
+      if ( args[0].endsWith( ".rd5diff" ) )
+      {
+        addDeltas( new File( args[0] ),new File( args[1] ), new File( args[2] ) );
+      }
+      else
+      {
+        recoverFromDelta( new File( args[0] ),new File( args[1] ), new File( args[2] ) /*, new File( args[3] ) */ );
+      }
+    }
+    else
+    {
+      diff2files( new File( args[0] ),new File( args[1] ), new File( args[2] ) );
+    }
+  }
+
+  private static long[] readFileIndex( DataInputStream dis, DataOutputStream dos ) throws Exception
+  {
+    long[] fileIndex = new long[25];
+    for( int i=0; i<25; i++ )
+    {
+      long lv = dis.readLong();
+      fileIndex[i] = lv & 0xffffffffffffL;
+      if ( dos != null )
+      {
+        dos.writeLong( lv );
+      }
+    }
+    return fileIndex;
+  }
+
+  private static long getTileStart( long[] index, int tileIndex )
+  {
+    return tileIndex > 0 ? index[tileIndex - 1] : 200L;
+  }
+
+  private static long getTileEnd( long[] index, int tileIndex )
+  {
+    return index[tileIndex];
+  }
+
+  private static int[] readPosIndex( DataInputStream dis, DataOutputStream dos ) throws Exception
+  {
+    int[] posIndex = new int[1024];
+    for( int i=0; i<1024; i++ )
+    {
+      int iv = dis.readInt();
+      posIndex[i] = iv;
+      if ( dos != null )
+      {
+        dos.writeInt( iv );
+      }
+    }
+    return posIndex;
+  }
+
+  private static int getPosIdx( int[] posIdx, int idx )
+  {
+    return idx == -1 ? 4096 : posIdx[idx];
+  }
+
+  private static byte[] createMicroCache( int[] posIdx, int tileIdx, DataInputStream dis, boolean deltaMode ) throws Exception
+  {
+    if ( posIdx == null )
+    {
+      return null;
+    }
+    int size = getPosIdx( posIdx, tileIdx ) - getPosIdx( posIdx, tileIdx-1 );
+    if ( size == 0 )
+    {
+      return null;
+    }
+    if ( deltaMode )
+    {
+      size = dis.readInt();
+    }
+    byte[] ab = new byte[size];
+    dis.readFully( ab );
+    return ab;
+  }
+
+  private static MicroCache createMicroCache( byte[] ab, DataBuffers dataBuffers ) throws Exception
+  {
+    if ( ab == null || ab.length == 0 )
+    {
+      return MicroCache.emptyCache();
+    }
+    StatCoderContext bc = new StatCoderContext( ab );
+    return new MicroCache2( bc, dataBuffers, 0, 0, 32, null, null );
   }
 
   /**
    * Compute the delta between 2 RD5 files and
    * show statistics on the expected size of the delta file
    */
-  public static void diff2files( File f1, File f2 ) throws Exception
+  public static void diff2files( File f1, File f2, File outFile ) throws Exception
   {
     byte[] abBuf1 = new byte[10 * 1024 * 1024];
     byte[] abBuf2 = new byte[10 * 1024 * 1024];
     
-    int nodesTotal = 0;
     int nodesDiff = 0;
     int diffedTiles = 0;
     
     long bytesDiff = 0L;
-    long diffedTileSize = 0L;
 
-    PhysicalFile pf1 = null;
+    DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
-    PhysicalFile pf2 = null;
+    DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
+    DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
+
+    // copy header to outfile
+    long[] fileIndex1 = readFileIndex( dis1, null );
+    long[] fileIndex2 = readFileIndex( dis2, dos );
+
     try
     {
       DataBuffers dataBuffers = new DataBuffers();
-      pf1 = new PhysicalFile( f1, dataBuffers, -1, -1 );
+      for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ )
-      pf2 = new PhysicalFile( f2, dataBuffers, -1, -1 );
-      int div = pf1.divisor;
-      for ( int lonDegree = 0; lonDegree < 5; lonDegree++ ) // does'nt really matter..
       {
-        for ( int latDegree = 0; latDegree < 5; latDegree++ ) // ..where on earth we are
+         boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx );
-        {
+         boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx );
-          OsmFile osmf1 = new OsmFile( pf1, lonDegree, latDegree, dataBuffers );
+
-          OsmFile osmf2 = new OsmFile( pf2, lonDegree, latDegree, dataBuffers );
+         int[] posIdx1 = hasData1 ? readPosIndex( dis1, null ) : null;
-          for ( int lonIdx = 0; lonIdx < div; lonIdx++ )
+         int[] posIdx2 = hasData2 ? readPosIndex( dis2, dos ) : null;
-          {
+
-            for ( int latIdx = 0; latIdx < div; latIdx++ )
+         for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ )
-            {
+         {
-              int lonIdxDiv = lonDegree * div + lonIdx;
+           byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false );
-              int latIdxDiv = latDegree * div + latIdx;
+           byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, false );
+
+           if ( ab2 == null )
+           {
+             continue; // empty target tile
+           }
+           MicroCache mc;
+           if ( Arrays.equals( ab1, ab2 ) )
+           {
+             mc = MicroCache.emptyCache(); // empty diff
+           }
+           else // calc diff of the 2 tiles
+           {
+             MicroCache mc1 = createMicroCache( ab1, dataBuffers );
+             MicroCache mc2 = createMicroCache( ab2, dataBuffers );
+             mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
+             mc.calcDelta( mc1, mc2 );
+           }
               
-            
+           if ( mc.getSize() == 0 )
-              MicroCache mc1 = osmf1.hasData() ? 
+           {
-                osmf1.createMicroCache( lonIdxDiv, latIdxDiv, dataBuffers, null, null, true, null )
+             dos.writeInt( 0 );
-              : MicroCache.emptyCache();
+           }
-              MicroCache mc2 = osmf2.hasData() ? 
+           else
-                osmf2.createMicroCache( lonIdxDiv, latIdxDiv, dataBuffers, null, null, true, null )
+           {
-              : MicroCache.emptyCache();
+             int len = mc.encodeMicroCache( abBuf1 );
-              
+             dos.writeInt( len );
-              MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, lonIdxDiv, latIdxDiv, div );
+             dos.write( abBuf1, 0, len );
-              mc.calcDelta( mc1, mc2 );
+
-              
+                 // do some consistemcy checks on the encoding
-              nodesTotal += mc2.getSize();
+
-              
-              if ( mc.getSize() > 0 )
-              {
-                 int len = mc.encodeMicroCache( abBuf1 );
                  byte[] bytes = new byte[len];
                  System.arraycopy( abBuf1, 0, bytes, 0, len );
 
                  bytesDiff += len;
                  nodesDiff += mc.getSize();
                  diffedTiles++;
-                 diffedTileSize += mc2.size();
                  
                  // cross-check the encoding: decode again
-                 MicroCache mcCheck = new MicroCache2( new StatCoderContext( bytes ), new DataBuffers( null ), lonIdxDiv, latIdxDiv, div, null, null );
+                 MicroCache mcCheck = new MicroCache2( new StatCoderContext( bytes ), new DataBuffers( null ), 0, 0, 32, null, null );
                  
                  // due to link-order ambiguity, for decoded we can only compare node-count and datasize
                  if ( mc.size() != mcCheck.size() )
@@ -110,30 +220,289 @@ final public class Rd5DiffTool
                      throw new IllegalArgumentException( "decoded data mismatch at i=" + i );
                    }
                  }
-              }
-            }
           }
         }
       }
-      System.out.println( "nodesTotal=" + nodesTotal + " nodesDiff=" + nodesDiff + " bytesDiff=" + bytesDiff + " diffedTiles=" + diffedTiles + " diffedTileSize=" + diffedTileSize );
+      // write any remaining data to the output file
+      for(;;)
+      {
+        int len = dis2.read( abBuf1 );
+        if (len < 0)
+        {
+          break;
+        }
+        dos.write( abBuf1, 0, len );
+      }
+      System.out.println( "nodesDiff=" + nodesDiff + " bytesDiff=" + bytesDiff + " diffedTiles=" + diffedTiles );
     }
     finally
     {
-      if ( pf1 != null )
+      if ( dis1 != null )
       {
         try
         {
-          pf1.ra.close();
+          dis1.close();
         }
         catch (Exception ee)
         {
         }
       }
-      if ( pf2 != null )
+      if ( dis2 != null )
       {
         try
         {
-          pf2.ra.close();
+          dis2.close();
+        }
+        catch (Exception ee)
+        {
+        }
+      }
+      if ( dos != null )
+      {
+        try
+        {
+          dos.close();
+        }
+        catch (Exception ee)
+        {
+        }
+      }
+    }
+  }
+
+
+  public static void recoverFromDelta( File f1, File f2, File outFile /* , File cmpFile */ ) throws Exception
+  {
+    byte[] abBuf1 = new byte[10 * 1024 * 1024];
+    byte[] abBuf2 = new byte[10 * 1024 * 1024];
+
+    DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
+    DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
+//    DataInputStream disCmp = new DataInputStream( new BufferedInputStream( new FileInputStream( cmpFile ) ) );
+    DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
+
+    // copy header to outfile
+    long[] fileIndex1 = readFileIndex( dis1, null );
+    long[] fileIndex2 = readFileIndex( dis2, dos );
+//    long[] fileIndexCmp = readFileIndex( disCmp, null );
+
+    try
+    {
+      DataBuffers dataBuffers = new DataBuffers();
+      for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ )
+      {
+         boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx );
+         boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx );
+
+//         boolean hasDataCmp = getTileStart( fileIndexCmp, subFileIdx ) < getTileEnd( fileIndexCmp, subFileIdx );
+
+         int[] posIdx1 = hasData1 ? readPosIndex( dis1, null ) : null;
+         int[] posIdx2 = hasData2 ? readPosIndex( dis2, dos ) : null;
+
+ //        int[] posIdxCmp = hasDataCmp ? readPosIndex( disCmp, null ) : null;
+
+         for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ )
+         {
+           byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false );
+           byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true );
+           if ( ab2 == null )
+           {
+             continue; // no target tile expected
+           }
+
+/*         int targetSizeCmp = getPosIdx( posIdxCmp, tileIdx ) - getPosIdx( posIdxCmp, tileIdx-1 );
+           if ( targetSizeCmp != targetSize ) throw new IllegalArgumentException( "target size mismatch: "+ targetSize + "," + targetSizeCmp );
+           byte[] abCmp = new byte[targetSizeCmp];
+           disCmp.readFully( abCmp );
+*/
+
+           // no-delta shortcut: just copy base data
+           if ( ab2.length == 0 )
+           {
+             if ( ab1 != null )
+             {
+               dos.write( ab1 );
+             }
+             continue;
+           }
+
+           // this is the real delta case (using decode->delta->encode )
+           MicroCache mc1 = createMicroCache( ab1, dataBuffers );
+           MicroCache mc2 = createMicroCache( ab2, dataBuffers );
+
+           MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
+           mc.addDelta( mc1, mc2, false );
+
+           int len = mc.encodeMicroCache( abBuf1 );
+
+/*           System.out.println( "comparing for subFileIdx=" + subFileIdx + " tileIdx=" + tileIdx );
+           boolean isequal = true;
+           for( int i=0; i<len;i++ )
+           {
+             if ( isequal && abCmp[i] != abBuf1[i] )
+             {
+               System.out.println( "data mismatch at i=" + i + " " + abCmp[i] + "!=" + abBuf1[i]  + " targetSize=" + targetSize );
+               isequal = false;
+
+               MicroCache.debug = true;
+               System.out.println( "**** decoding original cache ****" );
+               createMicroCache( abCmp, dataBuffers );
+               System.out.println( "**** decoding reconstructed cache ****" );
+               createMicroCache( abBuf1, dataBuffers );
+               System.exit(1);
+             }
+           }
+*/
+           dos.write( abBuf1, 0, len );
+           dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 );
+        }
+      }
+      // write any remaining data to the output file
+      for(;;)
+      {
+        int len = dis2.read( abBuf1 );
+        if (len < 0)
+        {
+          break;
+        }
+        dos.write( abBuf1, 0, len );
+      }
+    }
+    finally
+    {
+      if ( dis1 != null )
+      {
+        try
+        {
+          dis1.close();
+        }
+        catch (Exception ee)
+        {
+        }
+      }
+      if ( dis2 != null )
+      {
+        try
+        {
+          dis2.close();
+        }
+        catch (Exception ee)
+        {
+        }
+      }
+      if ( dos != null )
+      {
+        try
+        {
+          dos.close();
+        }
+        catch (Exception ee)
+        {
+        }
+      }
+    }
+  }
+
+  public static void addDeltas( File f1, File f2, File outFile ) throws Exception
+  {
+    byte[] abBuf1 = new byte[10 * 1024 * 1024];
+    byte[] abBuf2 = new byte[10 * 1024 * 1024];
+
+    DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
+    DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
+    DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
+
+    // copy header to outfile
+    long[] fileIndex1 = readFileIndex( dis1, null );
+    long[] fileIndex2 = readFileIndex( dis2, dos );
+
+    try
+    {
+      DataBuffers dataBuffers = new DataBuffers();
+      for ( int subFileIdx = 0; subFileIdx < 25; subFileIdx++ )
+      {
+         boolean hasData1 = getTileStart( fileIndex1, subFileIdx ) < getTileEnd( fileIndex1, subFileIdx );
+         boolean hasData2 = getTileStart( fileIndex2, subFileIdx ) < getTileEnd( fileIndex2, subFileIdx );
+
+         int[] posIdx1 = hasData1 ? readPosIndex( dis1, null ) : null;
+         int[] posIdx2 = hasData2 ? readPosIndex( dis2, dos ) : null;
+
+         for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ )
+         {
+           byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, true );
+           byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true );
+           if ( ab2 == null )
+           {
+             continue; // no target tile expected
+           }
+
+           // no-delta shortcut: just copy base data
+           if ( ab2.length == 0 )
+           {
+             if ( ab1 == null )
+             {
+               dos.writeInt( 0 );
+             }
+             else
+             {
+               dos.writeInt( ab1.length );
+               dos.write( ab1 );
+             }
+             continue;
+           }
+
+           // this is the real delta case (using decode->add->encode )
+           MicroCache mc1 = createMicroCache( ab1, dataBuffers );
+           MicroCache mc2 = createMicroCache( ab2, dataBuffers );
+
+           MicroCache mc = new MicroCache2( mc1.getSize() + mc2.getSize(), abBuf2, 0, 0, 32 );
+           mc.addDelta( mc1, mc2, true );
+
+           int len = mc.encodeMicroCache( abBuf1 );
+
+           dos.writeInt( len+4 );
+           dos.write( abBuf1, 0, len );
+           dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 );
+        }
+      }
+      // write any remaining data to the output file
+      for(;;)
+      {
+        int len = dis2.read( abBuf1 );
+        if (len < 0)
+        {
+          break;
+        }
+        dos.write( abBuf1, 0, len );
+      }
+    }
+    finally
+    {
+      if ( dis1 != null )
+      {
+        try
+        {
+          dis1.close();
+        }
+        catch (Exception ee)
+        {
+        }
+      }
+      if ( dis2 != null )
+      {
+        try
+        {
+          dis2.close();
+        }
+        catch (Exception ee)
+        {
+        }
+      }
+      if ( dos != null )
+      {
+        try
+        {
+          dos.close();
         }
         catch (Exception ee)
         {

brouter-server/src/main/java/btools/server/SuspectManager.java

@@ -503,7 +503,7 @@ public class SuspectManager extends Thread
     {
       bw.write( filter + " suspect list for " + country + "\n" );
       bw.write( "<br><a href=\"/brouter/suspects" + challenge + country + "/" + filter + "/watchlist\">see watchlist</a>\n" );
-      bw.write( "<br><a href=\"/brouter/suspects\">back to country list</a><br><br>\n" );
+      bw.write( "<br><a href=\"/brouter/suspects" + challenge + "\">back to country list</a><br><br>\n" );
       int maxprio = 0;
       {
         for( int isuspect = 0; isuspect<suspects.cnt; isuspect++ )