scripts to handle diff updates

brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffApplier.java

@@ -0,0 +1,93 @@
+/**
+ * Apply all rd5 diff-files
+ *
+ * @author vcoppe
+ */
+package btools.mapaccess;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Comparator;
+
+import btools.util.ProgressListener;
+
+final public class Rd5DiffApplier implements ProgressListener
+{
+  public static void main( String[] args ) throws Exception
+  {
+    applyDiffs( new File( args[0] ),new File( args[1] ),new File( args[2] ) );
+  }
+
+  /**
+   * Apply diffs for all RD5 files
+   */
+  public static void applyDiffs( File segmentsDir, File diffDir, File outDir ) throws Exception
+  {
+    Rd5DiffApplier progress = new Rd5DiffApplier();
+
+    outDir.mkdir();
+
+    File[] fileSegments = segmentsDir.listFiles();
+    Arrays.sort( fileSegments, Comparator.comparing(File::getName) );
+
+    for( File fs : fileSegments )
+    {
+      String name = fs.getName();
+      if ( !name.endsWith( ".rd5" ) )
+      {
+        continue;
+      }
+
+      File newSegment = new File( outDir, name );
+      newSegment.createNewFile();
+      Rd5DiffTool.copyFile( fs, newSegment, progress );
+      newSegment.setLastModified( fs.lastModified() );
+
+      String basename = name.substring( 0, name.length() - 4 );
+      File segmentDiffDir = new File( diffDir, basename );
+      if ( segmentDiffDir.isDirectory() )
+      {
+        File[] segmentDiffFiles = segmentDiffDir.listFiles();
+        Arrays.sort( segmentDiffFiles, Comparator.comparingLong(File::lastModified) );
+
+        for ( File segmentDiff : segmentDiffFiles )
+        {
+          String diffName = segmentDiff.getName();
+          if ( !diffName.endsWith( ".df5" ) )
+          {
+            continue;
+          }
+          if ( segmentDiff.lastModified() <= fs.lastModified() )
+          {
+            continue; // diff is older than segment
+          }
+          if ( segmentDiff.length() == 0L )
+          {
+            continue; // avoid changing date for empty diff
+          }
+
+          System.out.println( "Applying diff " + segmentDiff.getName() + " to segment: " + name );
+
+          File tmpSegment = new File( outDir, "tmp.rd5" );
+          tmpSegment.createNewFile();
+
+          Rd5DiffTool.recoverFromDelta( newSegment, segmentDiff, tmpSegment, progress );
+
+          newSegment.delete();
+
+          tmpSegment.renameTo( new File( outDir, name ) );
+          newSegment = new File( outDir, name );
+          newSegment.setLastModified( segmentDiff.lastModified() );
+        }
+      }
+    }
+  }
+
+  @Override
+  public void updateProgress( String progress ) { }
+
+  public boolean isCanceled()
+  {
+    return false;
+  }
+}

misc/scripts/cronjob.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+./download_segments_diff.sh # downloads diff directory in tmp/diff
+
+mkdir -p tmp/segments4
+
+java -cp ../../brouter-server/build/libs/brouter-1.6.3-all.jar btools.mapaccess.Rd5DiffApplier ../segments4 ../segments4_diff tmp/segments4
+
+mv ../segments4 tmp/segments4_old && mv tmp/segments4 ../segments4
+
+rm -rf tmp

misc/scripts/download_segments.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+out_dir="../segments4"
+
+curl http://brouter.de/brouter/segments4/ --silent | grep "[EW][0-9]*_[NS][0-9]*\.rd5" -o | uniq > segments
+
+mkdir -p $out_dir
+
+SECONDS=0
+
+<segments xargs -I{} -P8 curl "http://brouter.de/brouter/segments4/{}" --remote-time --output "$out_dir/{}" --silent
+
+echo "All segments downloaded in ${SECONDS}s"
+
+rm segments

misc/scripts/download_segments_diff.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+out_dir="../segments4_diff"
+
+mkdir -p $out_dir
+
+find $out_dir -type f -mtime +10 -delete
+
+SECONDS=0
+
+folders=( `curl http://brouter.de/brouter/segments4/diff/ --silent | grep "[EW][0-9]*_[NS][0-9]*"  -o | uniq` )
+> diffs
+
+for folder in "${folders[@]}"
+do
+    mkdir -p "$out_dir/$folder"
+    files=( `curl "http://brouter.de/brouter/segments4/diff/$folder/" --silent | grep "[a-z0-9]*\.df5" -o | uniq` )
+    for file in "${files[@]}"
+    do
+        if [ ! -s "$out_dir/$folder/$file" ]; then # only download files that we don't already have
+            echo "$folder/$file" >> diffs
+        fi
+    done
+done
+
+<diffs xargs -I{} -P8 curl "http://brouter.de/brouter/segments4/diff/{}" --remote-time --output "$out_dir/{}" --silent
+
+rm diffs
+
+echo "All segments diff downloaded in ${SECONDS}s"