diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java index 76607fc..faa860b 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java @@ -48,6 +48,8 @@ final public class Rd5DiffManager // calculate MD5 of old file String md5 = getMD5( fo ); + + String md5New = getMD5( fn ); System.out.println( "name=" + name + " md5=" + md5 ); @@ -55,9 +57,12 @@ final public class Rd5DiffManager specificNewDiffs.mkdirs(); String diffFileName = md5 + ".rd5diff"; - File diffFile = new File( specificNewDiffs, diffFileName ); + String dummyDiffFileName = md5New + ".rd5diff"; + File dummyDiffFile = new File( specificNewDiffs, dummyDiffFileName ); + dummyDiffFile.createNewFile(); + // calc the new diff Rd5DiffTool.diff2files( fo, fn, diffFile ); @@ -72,15 +77,17 @@ final public class Rd5DiffManager { continue; } - if ( System.currentTimeMillis() - od.lastModified() > 31*86400000L ) + if ( System.currentTimeMillis() - od.lastModified() > 9*86400000L ) { - continue; // limit diff history to 31 days + continue; // limit diff history to 9 days } File updatedDiff = new File( specificNewDiffs, od.getName() ); - - Rd5DiffTool.addDeltas( od, diffFile, updatedDiff ); - updatedDiff.setLastModified( od.lastModified() ); + if ( !updatedDiff.exists() ) + { + Rd5DiffTool.addDeltas( od, diffFile, updatedDiff ); + updatedDiff.setLastModified( od.lastModified() ); + } } } } diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java index 5c744bd..ccf3367 100644 --- a/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java +++ b/brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java @@ -19,8 +19,9 @@ import btools.codec.MicroCache; import btools.codec.MicroCache2; import btools.codec.StatCoderContext; import btools.util.Crc32; +import btools.util.ProgressListener; -final public class Rd5DiffTool +final public class Rd5DiffTool implements ProgressListener { public static void main( String[] args ) throws Exception { @@ -32,7 +33,7 @@ final public class Rd5DiffTool } else { - recoverFromDelta( new File( args[0] ),new File( args[1] ), new File( args[2] ) /*, new File( args[3] ) */ ); + recoverFromDelta( new File( args[0] ),new File( args[1] ), new File( args[2] ), new Rd5DiffTool() /*, new File( args[3] ) */ ); } } else @@ -41,6 +42,18 @@ final public class Rd5DiffTool } } + @Override + public void updateProgress( String progress ) + { + System.out.println( progress ); + } + + @Override + public boolean isCanceled() + { + return false; + } + private static long[] readFileIndex( DataInputStream dis, DataOutputStream dos ) throws Exception { long[] fileIndex = new long[25]; @@ -275,10 +288,20 @@ final public class Rd5DiffTool } - public static void recoverFromDelta( File f1, File f2, File outFile /* , File cmpFile */ ) throws Exception + public static void recoverFromDelta( File f1, File f2, File outFile, ProgressListener progress /* , File cmpFile */ ) throws Exception { + if ( f2.length() == 0L ) + { + copyFile( f1, outFile, progress ); + return; + } + byte[] abBuf1 = new byte[10 * 1024 * 1024]; byte[] abBuf2 = new byte[10 * 1024 * 1024]; + + boolean canceled = false; + + long t0 = System.currentTimeMillis(); DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) ); DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) ); @@ -290,6 +313,8 @@ final public class Rd5DiffTool long[] fileIndex2 = readFileIndex( dis2, dos ); // long[] fileIndexCmp = readFileIndex( disCmp, null ); + int lastPct = -1; + try { DataBuffers dataBuffers = new DataBuffers(); @@ -307,6 +332,19 @@ final public class Rd5DiffTool for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ ) { + if ( progress.isCanceled() ) + { + canceled = true; + return; + } + double bytesProcessed = getTileStart( fileIndex1, subFileIdx ) + ( posIdx1 == null ? 0 : getPosIdx( posIdx1, tileIdx-1 ) ); + int pct = (int)(100. * bytesProcessed / getTileEnd( fileIndex1, 24 ) + 0.5 ); + if ( pct != lastPct ) + { + progress.updateProgress( "Applying delta: " + pct + "%" ); + lastPct = pct; + } + byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false ); byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true ); if ( ab2 == null ) @@ -359,6 +397,7 @@ final public class Rd5DiffTool */ dos.write( abBuf1, 0, len ); dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 ); + } } // write any remaining data to the output file @@ -371,6 +410,8 @@ final public class Rd5DiffTool } dos.write( abBuf1, 0, len ); } + long t1 = System.currentTimeMillis(); + System.out.println( "recovering from diffs took " + (t1-t0) + "ms" ); } finally { @@ -403,6 +444,72 @@ final public class Rd5DiffTool catch (Exception ee) { } + if ( canceled ) + { + outFile.delete(); + } + } + } + } + + public static void copyFile( File f1, File outFile, ProgressListener progress ) throws Exception + { + boolean canceled = false; + DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) ); + DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) ); + int lastPct = -1; + long sizeTotal = f1.length(); + long sizeRead = 0L; + try + { + byte buf[] = new byte[65536]; + for (;;) + { + if ( progress.isCanceled() ) + { + canceled = true; + return; + } + int pct = (int)( (100. * sizeRead) / (sizeTotal+1) + 0.5 ); + if ( pct != lastPct ) + { + progress.updateProgress( "Copying: " + pct + "%" ); + lastPct = pct; + } + int len = dis1.read( buf ); + if ( len <= 0 ) + { + break; + } + sizeRead += len; + dos.write( buf, 0, len ); + } + } + finally + { + if ( dis1 != null ) + { + try + { + dis1.close(); + } + catch (Exception ee) + { + } + } + if ( dos != null ) + { + try + { + dos.close(); + } + catch (Exception ee) + { + } + if ( canceled ) + { + outFile.delete(); + } } } } diff --git a/brouter-routing-app/src/main/java/btools/routingapp/BInstallerView.java b/brouter-routing-app/src/main/java/btools/routingapp/BInstallerView.java index efeb7c5..2302a5a 100644 --- a/brouter-routing-app/src/main/java/btools/routingapp/BInstallerView.java +++ b/brouter-routing-app/src/main/java/btools/routingapp/BInstallerView.java @@ -25,7 +25,10 @@ import android.view.MotionEvent; import android.view.View; import android.widget.Toast; import btools.mapaccess.PhysicalFile; +import btools.mapaccess.Rd5DiffManager; +import btools.mapaccess.Rd5DiffTool; import btools.router.RoutingHelper; +import btools.util.ProgressListener; public class BInstallerView extends View { @@ -61,7 +64,9 @@ public class BInstallerView extends View private long currentDownloadSize; private String currentDownloadFile = ""; + private volatile String currentDownloadOperation = ""; private String downloadAction = ""; + private volatile String newDownloadAction = ""; private long totalSize = 0; private long rd5Tiles = 0; @@ -144,6 +149,7 @@ public class BInstallerView extends View String namebase = baseNameForTile( tileIndex ); String baseurl = "http://brouter.de/brouter/segments4/"; currentDownloadFile = namebase + ".rd5"; + currentDownloadOperation = "Checking"; String url = baseurl + currentDownloadFile; isDownloading = true; downloadCanceled = false; @@ -241,7 +247,7 @@ public class BInstallerView extends View tileStatus[tidx] |= MASK_INSTALLED_RD5; long age = System.currentTimeMillis() - new File( dir, fileName ).lastModified(); - if ( age < 86400000 ) tileStatus[tidx] |= MASK_CURRENT_RD5; + if ( age < 300000 ) tileStatus[tidx] |= MASK_CURRENT_RD5; // 5 minutes } } } @@ -368,7 +374,7 @@ public class BInstallerView extends View { String sizeHint = currentDownloadSize > 0 ? " (" + ((currentDownloadSize + mb-1)/mb) + " MB)" : ""; paint.setTextSize(30); - canvas.drawText( "Loading " + currentDownloadFile + sizeHint, 30, (imgh/3)*2-30, paint); + canvas.drawText( currentDownloadOperation + " " + currentDownloadFile + sizeHint, 30, (imgh/3)*2-30, paint); canvas.drawText( downloadAction, 30, (imgh/3)*2, paint); } if ( !tilesVisible ) @@ -573,7 +579,7 @@ float tx, ty; // usually, subclasses of AsyncTask are declared inside the activity class. // that way, you can easily modify the UI thread from here - private class DownloadTask extends AsyncTask { + private class DownloadTask extends AsyncTask implements ProgressListener { private Context context; private PowerManager.WakeLock mWakeLock; @@ -583,6 +589,19 @@ float tx, ty; } @Override + public void updateProgress( String progress ) + { + newDownloadAction = progress; + publishProgress( 0 ); + } + + @Override + public boolean isCanceled() + { + return isDownloadCanceled(); + } + + @Override protected String doInBackground(String... sUrls) { InputStream input = null; @@ -595,16 +614,48 @@ float tx, ty; { try { - URL url = new URL(surl); + int slidx = surl.lastIndexOf( "segments4/" ); + String name = surl.substring( slidx+10 ); + String surlBase = surl.substring( 0, slidx+10 ); + fname = baseDir + "/brouter/segments4/" + name; - connection = (HttpURLConnection) url.openConnection(); - connection.connect(); + boolean delta = true; + File targetFile = new File( fname ); + if ( targetFile.exists() ) + { + updateProgress( "Calculating local checksum.." ); + + // first check for a delta file + String md5 = Rd5DiffManager.getMD5( targetFile ); + String surlDelta = surlBase + "diff/" + name.replace( ".rd5", "/" + md5 + ".rd5diff" ); + + URL urlDelta = new URL(surlDelta); + + updateProgress( "Connecting.." ); + + connection = (HttpURLConnection) urlDelta.openConnection(); + connection.connect(); + + // 404 kind of expected here, means there's no delta file + if (connection.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND ) + { + connection = null; + } + } + + if ( connection == null ) + { + delta = false; + URL url = new URL(surl); + connection = (HttpURLConnection) url.openConnection(); + connection.connect(); + } // expect HTTP 200 OK, so we don't mistakenly save error report // instead of the file if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { return "Server returned HTTP " + connection.getResponseCode() - + " " + connection.getResponseMessage(); + + " " + connection.getResponseMessage(); } // this will be useful to display download percentage @@ -612,13 +663,13 @@ float tx, ty; int fileLength = connection.getContentLength(); currentDownloadSize = fileLength; if ( availableSize >= 0 && fileLength > availableSize ) return "not enough space on sd-card"; + + currentDownloadOperation = delta ? "Updating" : "Loading"; // download the file input = connection.getInputStream(); - int slidx = surl.lastIndexOf( "segments4/" ); - fname = baseDir + "/brouter/segments4/" + surl.substring( slidx+10 ); - tmp_file = new File( fname + "_tmp" ); + tmp_file = new File( fname + ( delta ? "_diff" : "_tmp" ) ); output = new FileOutputStream( tmp_file ); byte data[] = new byte[4096]; @@ -632,7 +683,15 @@ float tx, ty; total += count; // publishing the progress.... if (fileLength > 0) // only if total length is known - publishProgress((int) (total * 100 / fileLength)); + { + int pct = (int) (total * 100 / fileLength); + updateProgress( "Progress " + pct + "%" ); + } + else + { + updateProgress( "Progress (unnown size)" ); + } + output.write(data, 0, count); // enforce < 2 Mbit/s @@ -642,15 +701,33 @@ float tx, ty; try { Thread.sleep( dt ); } catch( InterruptedException ie ) {} } } - publishProgress( 101 ); - String check_result = PhysicalFile.checkFileIntegrity( tmp_file ); - if ( check_result != null ) return check_result; + output.close(); + output = null; - if ( !tmp_file.renameTo( new File( fname ) ) ) + if ( delta ) { - return "Could not rename to " + fname; + updateProgress( "Applying delta.." ); + File diffFile = tmp_file; + tmp_file = new File( fname + "_tmp" ); + Rd5DiffTool.recoverFromDelta( targetFile, diffFile, tmp_file, this ); + diffFile.delete(); + } + if (isDownloadCanceled()) + { + return "Canceled!"; + } + if ( tmp_file != null ) + { + updateProgress( "Verifying integrity.." ); + String check_result = PhysicalFile.checkFileIntegrity( tmp_file ); + if ( check_result != null ) return check_result; + + if ( !tmp_file.renameTo( targetFile ) ) + { + return "Could not rename to " + targetFile; + } + deleteRawTracks(); // invalidate raw-tracks after data update } - deleteRawTracks(); // invalidate raw-tracks after data update return null; } catch (Exception e) { return e.toString(); @@ -685,10 +762,9 @@ float tx, ty; @Override protected void onProgressUpdate(Integer... progress) { - String newAction = progress[0] == 101 ? "Verifying.." : "Progress " + progress[0] + "%"; - if ( !newAction.equals( downloadAction ) ) + if ( !newDownloadAction.equals( downloadAction ) ) { - downloadAction = newAction; + downloadAction = newDownloadAction; invalidate(); } } diff --git a/brouter-util/src/main/java/btools/util/ProgressListener.java b/brouter-util/src/main/java/btools/util/ProgressListener.java new file mode 100644 index 0000000..eda4efe --- /dev/null +++ b/brouter-util/src/main/java/btools/util/ProgressListener.java @@ -0,0 +1,8 @@ +package btools.util; + +public interface ProgressListener +{ + public void updateProgress( String progress ); + + public boolean isCanceled(); +}