rd5 diff progress

brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java

@@ -48,6 +48,8 @@ final public class Rd5DiffManager
       
       // calculate MD5 of old file
       String md5 = getMD5( fo );
+
+      String md5New = getMD5( fn );
       
       System.out.println( "name=" + name + " md5=" + md5 );
       
@@ -55,9 +57,12 @@ final public class Rd5DiffManager
       specificNewDiffs.mkdirs();
       
       String diffFileName = md5 + ".rd5diff";
-      
       File diffFile = new File( specificNewDiffs, diffFileName );
       
+      String dummyDiffFileName = md5New + ".rd5diff";
+      File dummyDiffFile = new File( specificNewDiffs, dummyDiffFileName );
+      dummyDiffFile.createNewFile();
+
       // calc the new diff
       Rd5DiffTool.diff2files( fo, fn, diffFile  );
       
@@ -72,15 +77,17 @@ final public class Rd5DiffManager
           {
             continue;
           }
-          if ( System.currentTimeMillis() - od.lastModified() > 31*86400000L )
+          if ( System.currentTimeMillis() - od.lastModified() > 9*86400000L )
           {
-            continue; // limit diff history to 31 days
+            continue; // limit diff history to 9 days
           }
           
           File updatedDiff = new File( specificNewDiffs, od.getName() );
-          
+          if ( !updatedDiff.exists() )
-          Rd5DiffTool.addDeltas( od, diffFile, updatedDiff  );
+          {        
-          updatedDiff.setLastModified( od.lastModified() );
+            Rd5DiffTool.addDeltas( od, diffFile, updatedDiff  );
+            updatedDiff.setLastModified( od.lastModified() );
+          }
         }
       }
     }

brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java

@@ -19,8 +19,9 @@ import btools.codec.MicroCache;
 import btools.codec.MicroCache2;
 import btools.codec.StatCoderContext;
 import btools.util.Crc32;
+import btools.util.ProgressListener;
 
-final public class Rd5DiffTool
+final public class Rd5DiffTool implements ProgressListener
 {
   public static void main( String[] args ) throws Exception
   {
@@ -32,7 +33,7 @@ final public class Rd5DiffTool
       }
       else
       {
-        recoverFromDelta( new File( args[0] ),new File( args[1] ), new File( args[2] ) /*, new File( args[3] ) */ );
+        recoverFromDelta( new File( args[0] ),new File( args[1] ), new File( args[2] ), new Rd5DiffTool() /*, new File( args[3] ) */ );
       }
     }
     else
@@ -41,6 +42,18 @@ final public class Rd5DiffTool
     }
   }
 
+            @Override
+            public void updateProgress( String progress )
+            {
+              System.out.println( progress );
+            }
+  
+            @Override
+            public boolean isCanceled()
+            {
+              return false;
+            }
+
   private static long[] readFileIndex( DataInputStream dis, DataOutputStream dos ) throws Exception
   {
     long[] fileIndex = new long[25];
@@ -275,10 +288,20 @@ final public class Rd5DiffTool
   }
 
 
-  public static void recoverFromDelta( File f1, File f2, File outFile /* , File cmpFile */ ) throws Exception
+  public static void recoverFromDelta( File f1, File f2, File outFile, ProgressListener progress /* , File cmpFile */ ) throws Exception
   {
+    if ( f2.length() == 0L )
+    {
+      copyFile( f1, outFile, progress );
+      return;
+    }
+  
     byte[] abBuf1 = new byte[10 * 1024 * 1024];
     byte[] abBuf2 = new byte[10 * 1024 * 1024];
+    
+    boolean canceled = false;
+
+    long t0 = System.currentTimeMillis();
 
     DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
     DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
@@ -290,6 +313,8 @@ final public class Rd5DiffTool
     long[] fileIndex2 = readFileIndex( dis2, dos );
 //    long[] fileIndexCmp = readFileIndex( disCmp, null );
 
+    int lastPct = -1;
+
     try
     {
       DataBuffers dataBuffers = new DataBuffers();
@@ -307,6 +332,19 @@ final public class Rd5DiffTool
 
          for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ )
          {
+           if ( progress.isCanceled() )
+           {
+             canceled = true;
+             return;
+           }
+           double bytesProcessed = getTileStart( fileIndex1, subFileIdx ) + ( posIdx1 == null ? 0 : getPosIdx( posIdx1, tileIdx-1 ) );
+           int pct =  (int)(100. * bytesProcessed / getTileEnd( fileIndex1, 24 ) + 0.5 );
+           if ( pct != lastPct )
+           {
+             progress.updateProgress( "Applying delta: " + pct + "%" );
+             lastPct = pct;
+           }
+
            byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false );
            byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true );
            if ( ab2 == null )
@@ -359,6 +397,7 @@ final public class Rd5DiffTool
 */
            dos.write( abBuf1, 0, len );
            dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 );
+
         }
       }
       // write any remaining data to the output file
@@ -371,6 +410,8 @@ final public class Rd5DiffTool
         }
         dos.write( abBuf1, 0, len );
       }
+           long t1 = System.currentTimeMillis();
+           System.out.println( "recovering from diffs took " + (t1-t0) + "ms" );
     }
     finally
     {
@@ -403,6 +444,72 @@ final public class Rd5DiffTool
         catch (Exception ee)
         {
         }
+        if ( canceled )
+        {
+          outFile.delete();
+        }
+      }
+    }
+  }
+
+  public static void copyFile( File f1, File outFile, ProgressListener progress ) throws Exception
+  {
+    boolean canceled = false;
+    DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
+    DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
+    int lastPct = -1;
+    long sizeTotal = f1.length();
+    long sizeRead = 0L;
+    try
+    {
+      byte buf[] = new byte[65536];      
+      for (;;)
+      {
+        if ( progress.isCanceled() )
+        {
+          canceled = true;
+          return;
+        }
+        int pct =  (int)( (100. * sizeRead) / (sizeTotal+1) + 0.5 );
+        if ( pct != lastPct )
+        {
+          progress.updateProgress( "Copying: " + pct + "%" );
+          lastPct = pct;
+        }
+        int len = dis1.read( buf );
+        if ( len <= 0 )
+        {
+          break;
+        }
+        sizeRead += len;
+        dos.write( buf, 0, len );
+      }
+    }
+    finally
+    {
+      if ( dis1 != null )
+      {
+        try
+        {
+          dis1.close();
+        }
+        catch (Exception ee)
+        {
+        }
+      }
+      if ( dos != null )
+      {
+        try
+        {
+          dos.close();
+        }
+        catch (Exception ee)
+        {
+        }
+        if ( canceled )
+        {
+          outFile.delete();
+        }
       }
     }
   }

brouter-routing-app/src/main/java/btools/routingapp/BInstallerView.java

@@ -25,7 +25,10 @@ import android.view.MotionEvent;
 import android.view.View;
 import android.widget.Toast;
 import btools.mapaccess.PhysicalFile;
+import btools.mapaccess.Rd5DiffManager;
+import btools.mapaccess.Rd5DiffTool;
 import btools.router.RoutingHelper;
+import btools.util.ProgressListener;
 
 public class BInstallerView extends View
 {
@@ -61,7 +64,9 @@ public class BInstallerView extends View
 
     private long currentDownloadSize;
     private String currentDownloadFile = "";
+    private volatile String currentDownloadOperation = "";
     private String downloadAction = "";
+    private volatile String newDownloadAction = "";
 
     private long totalSize = 0;
     private long rd5Tiles = 0;
@@ -144,6 +149,7 @@ public class BInstallerView extends View
     	String namebase = baseNameForTile( tileIndex );
       String baseurl = "http://brouter.de/brouter/segments4/";
         currentDownloadFile = namebase + ".rd5";
+        currentDownloadOperation = "Checking";
         String url = baseurl + currentDownloadFile;
     	isDownloading = true;
     	downloadCanceled = false;
@@ -241,7 +247,7 @@ public class BInstallerView extends View
         	 tileStatus[tidx] |= MASK_INSTALLED_RD5;
         	 
         	 long age = System.currentTimeMillis() - new File( dir, fileName ).lastModified();
-        	 if ( age < 86400000 ) tileStatus[tidx] |= MASK_CURRENT_RD5;
+        	 if ( age < 300000 ) tileStatus[tidx] |= MASK_CURRENT_RD5; // 5 minutes
           }
         }
     }
@@ -368,7 +374,7 @@ public class BInstallerView extends View
            	{
               String sizeHint = currentDownloadSize > 0 ? " (" + ((currentDownloadSize + mb-1)/mb) + " MB)" : "";
               paint.setTextSize(30);
-              canvas.drawText( "Loading " + currentDownloadFile + sizeHint, 30, (imgh/3)*2-30, paint);
+              canvas.drawText( currentDownloadOperation +  " " + currentDownloadFile + sizeHint, 30, (imgh/3)*2-30, paint);
               canvas.drawText( downloadAction, 30, (imgh/3)*2, paint);
            	}
             if ( !tilesVisible )
@@ -573,7 +579,7 @@ float tx, ty;
         
         // usually, subclasses of AsyncTask are declared inside the activity class.
         // that way, you can easily modify the UI thread from here
-        private class DownloadTask extends AsyncTask<String, Integer, String> {
+        private class DownloadTask extends AsyncTask<String, Integer, String> implements ProgressListener {
 
             private Context context;
             private PowerManager.WakeLock mWakeLock;
@@ -583,6 +589,19 @@ float tx, ty;
             }
 
             @Override
+            public void updateProgress( String progress )
+            {
+              newDownloadAction = progress;
+              publishProgress( 0 );
+            }
+  
+            @Override
+            public boolean isCanceled()
+            {
+              return isDownloadCanceled();
+            }
+
+              @Override
             protected String doInBackground(String... sUrls)
             {
               InputStream input = null;
@@ -595,16 +614,48 @@ float tx, ty;
               {
                 try
                 {
-                    URL url = new URL(surl);
+                    int slidx = surl.lastIndexOf( "segments4/" );
+                    String name = surl.substring( slidx+10 );
+                    String surlBase = surl.substring( 0, slidx+10 );
+                    fname = baseDir + "/brouter/segments4/" + name;
 
-                    connection = (HttpURLConnection) url.openConnection();
+                    boolean delta = true;
-                    connection.connect();
 
+                    File targetFile = new File( fname );
+                    if ( targetFile.exists() )
+                    {
+                      updateProgress( "Calculating local checksum.." );
+                    
+                      // first check for a delta file
+                      String md5 = Rd5DiffManager.getMD5( targetFile );
+                      String surlDelta = surlBase + "diff/" + name.replace( ".rd5", "/" + md5 + ".rd5diff" );
+                      
+                      URL urlDelta = new URL(surlDelta);
+
+                      updateProgress( "Connecting.." );
+
+                      connection = (HttpURLConnection) urlDelta.openConnection();
+                      connection.connect();
+
+                      // 404 kind of expected here, means there's no delta file
+                      if (connection.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND )
+                      {
+                        connection = null;
+                      }
+                    }                   
+                
+                    if ( connection == null )
+                    {                
+                      delta = false;
+                      URL url = new URL(surl);
+                      connection = (HttpURLConnection) url.openConnection();
+                      connection.connect();
+                    }
                     // expect HTTP 200 OK, so we don't mistakenly save error report
                     // instead of the file
                     if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
                         return "Server returned HTTP " + connection.getResponseCode()
-                                + " " + connection.getResponseMessage();
+                              + " " + connection.getResponseMessage();
                     }
 
                     // this will be useful to display download percentage
@@ -612,13 +663,13 @@ float tx, ty;
                     int fileLength = connection.getContentLength();
                     currentDownloadSize = fileLength;
                     if ( availableSize >= 0 && fileLength > availableSize ) return "not enough space on sd-card";
+
+                    currentDownloadOperation = delta ? "Updating" : "Loading";
                     
                     // download the file
                     input = connection.getInputStream();
                     
-                    int slidx = surl.lastIndexOf( "segments4/" );
+                    tmp_file = new File( fname + ( delta ? "_diff" : "_tmp" ) );
-                    fname = baseDir + "/brouter/segments4/" + surl.substring( slidx+10 );
-                    tmp_file = new File( fname + "_tmp" );
                     output = new FileOutputStream( tmp_file );
 
                     byte data[] = new byte[4096];
@@ -632,7 +683,15 @@ float tx, ty;
                         total += count;
                         // publishing the progress....
                         if (fileLength > 0) // only if total length is known
-                            publishProgress((int) (total * 100 / fileLength));
+                        {
+                          int pct = (int) (total * 100 / fileLength);
+                          updateProgress( "Progress " + pct + "%" );
+                        }
+                        else
+                        {
+                          updateProgress( "Progress (unnown size)" );
+                        }
+
                         output.write(data, 0, count);
                         
                         // enforce < 2 Mbit/s
@@ -642,15 +701,33 @@ float tx, ty;
                         	try { Thread.sleep( dt ); } catch( InterruptedException ie ) {}
                         }
                     }
-                    publishProgress( 101 );
+                    output.close();
-                    String check_result = PhysicalFile.checkFileIntegrity( tmp_file );
+                    output = null;
-                    if ( check_result != null ) return check_result;
 
-                    if ( !tmp_file.renameTo( new File( fname ) ) )
+                    if ( delta )
                     {
-                      return "Could not rename to " + fname;
+                      updateProgress( "Applying delta.." );
+                      File diffFile = tmp_file;
+                      tmp_file = new File( fname + "_tmp" );
+                      Rd5DiffTool.recoverFromDelta( targetFile, diffFile, tmp_file, this );
+                      diffFile.delete();
+                    }
+                    if (isDownloadCanceled())
+                    {
+                      return "Canceled!";
+                    }
+                    if ( tmp_file != null )
+                    {
+                      updateProgress( "Verifying integrity.." );
+                      String check_result = PhysicalFile.checkFileIntegrity( tmp_file );
+                      if ( check_result != null ) return check_result;
+
+                      if ( !tmp_file.renameTo( targetFile ) )
+                      {
+                        return "Could not rename to " + targetFile;
+                      }
+                      deleteRawTracks(); // invalidate raw-tracks after data update
                     }
-                    deleteRawTracks(); // invalidate raw-tracks after data update
                     return null;
                 } catch (Exception e) {
                     return e.toString();
@@ -685,10 +762,9 @@ float tx, ty;
 
             @Override
             protected void onProgressUpdate(Integer... progress) {
-            	String newAction = progress[0] == 101 ? "Verifying.." : "Progress " + progress[0] + "%";
+            	if ( !newDownloadAction.equals( downloadAction ) )
-            	if ( !newAction.equals( downloadAction ) )
             	{
-            	  downloadAction = newAction;
+            	  downloadAction = newDownloadAction;
             	  invalidate();
             	}
             }

brouter-util/src/main/java/btools/util/ProgressListener.java

@@ -0,0 +1,8 @@
+package btools.util;
+
+public interface ProgressListener
+{
+  public void updateProgress( String progress );
+  
+  public boolean isCanceled();
+}