From 448bb11ad4f852c3b75da91c91ead9a13d672916 Mon Sep 17 00:00:00 2001
From: Arndt <Arndt.Brenschede@web.de>
Date: Sat, 10 Sep 2016 16:03:15 +0200
Subject: [PATCH] performance

---
 .../main/java/btools/codec/DataBuffers.java   |  3 +
 .../main/java/btools/codec/MicroCache.java    | 80 ++++++++++---------
 .../main/java/btools/codec/MicroCache2.java   |  4 +-
 .../main/java/btools/codec/TagValueCoder.java | 14 ++--
 .../src/main/java/btools/router/OsmPath.java  |  6 +-
 .../java/btools/router/RoutingContext.java    |  3 +
 .../expressions/BExpressionContext.java       | 17 ++--
 .../btools/mapaccess/GeometryDecoder.java     | 69 ++++++++++++++++
 .../java/btools/mapaccess/NodesCache.java     | 40 ++++------
 .../main/java/btools/mapaccess/OsmFile.java   | 28 +++++--
 .../main/java/btools/mapaccess/OsmLink.java   | 42 ----------
 .../java/btools/util/BitCoderContext.java     | 15 ++++
 .../main/java/btools/util/ByteDataReader.java |  8 ++
 13 files changed, 204 insertions(+), 125 deletions(-)
 create mode 100644 brouter-mapaccess/src/main/java/btools/mapaccess/GeometryDecoder.java

diff --git a/brouter-codec/src/main/java/btools/codec/DataBuffers.java b/brouter-codec/src/main/java/btools/codec/DataBuffers.java
index 4ee1038..ebe7f6e 100644
--- a/brouter-codec/src/main/java/btools/codec/DataBuffers.java
+++ b/brouter-codec/src/main/java/btools/codec/DataBuffers.java
@@ -1,5 +1,7 @@
 package btools.codec;
 
+import btools.util.BitCoderContext;
+
 /**
  * Container for some re-usable databuffers for the decoder
  */
@@ -7,6 +9,7 @@ public final class DataBuffers
 {
   public byte[] iobuffer;
   public byte[] tagbuf1 = new byte[256];
+  public BitCoderContext bctx1 = new BitCoderContext( tagbuf1 );
   public byte[] bbuf1 = new byte[65636];
   public int[] ibuf1 = new int[4096];
   public int[] ibuf2 = new int[2048];
diff --git a/brouter-codec/src/main/java/btools/codec/MicroCache.java b/brouter-codec/src/main/java/btools/codec/MicroCache.java
index 3dde2ae..b95f772 100644
--- a/brouter-codec/src/main/java/btools/codec/MicroCache.java
+++ b/brouter-codec/src/main/java/btools/codec/MicroCache.java
@@ -135,47 +135,51 @@ public class MicroCache extends ByteDataWriter
     return n > 0 ? fapos[n - 1] & 0x7fffffff : 0;
   }
 
-  public final void collect( int threshold )
+  public final int collect( int threshold )
   {
-    if ( delcount > threshold )
+    if ( delcount <= threshold )
     {
-      virgin = false;
-
-      int nsize = size - delcount;
-      if ( nsize == 0 )
-      {
-        faid = null;
-        fapos = null;
-      }
-      else
-      {
-        int[] nfaid = new int[nsize];
-        int[] nfapos = new int[nsize];
-        int idx = 0;
-
-        byte[] nab = new byte[ab.length - delbytes];
-        int nab_off = 0;
-        for ( int i = 0; i < size; i++ )
-        {
-          int pos = fapos[i];
-          if ( ( pos & 0x80000000 ) == 0 )
-          {
-            int start = startPos( i );
-            int end = fapos[i];
-            int len = end - start;
-            System.arraycopy( ab, start, nab, nab_off, len );
-            nfaid[idx] = faid[i];
-            nab_off += len;
-            nfapos[idx] = nab_off;
-            idx++;
-          }
-        }
-        faid = nfaid;
-        fapos = nfapos;
-        ab = nab;
-      }
-      init( nsize );
+      return 0;
     }
+
+    virgin = false;
+
+    int nsize = size - delcount;
+    if ( nsize == 0 )
+    {
+      faid = null;
+      fapos = null;
+    }
+    else
+    {
+      int[] nfaid = new int[nsize];
+      int[] nfapos = new int[nsize];
+      int idx = 0;
+
+      byte[] nab = new byte[ab.length - delbytes];
+      int nab_off = 0;
+      for ( int i = 0; i < size; i++ )
+      {
+        int pos = fapos[i];
+        if ( ( pos & 0x80000000 ) == 0 )
+        {
+          int start = startPos( i );
+          int end = fapos[i];
+          int len = end - start;
+          System.arraycopy( ab, start, nab, nab_off, len );
+          nfaid[idx] = faid[i];
+          nab_off += len;
+          nfapos[idx] = nab_off;
+          idx++;
+        }
+      }
+      faid = nfaid;
+      fapos = nfapos;
+      ab = nab;
+    }
+    int deleted = delbytes;
+    init( nsize );
+    return deleted;
   }
 
   public final void unGhost()
diff --git a/brouter-codec/src/main/java/btools/codec/MicroCache2.java b/brouter-codec/src/main/java/btools/codec/MicroCache2.java
index ca5093d..01675d0 100644
--- a/brouter-codec/src/main/java/btools/codec/MicroCache2.java
+++ b/brouter-codec/src/main/java/btools/codec/MicroCache2.java
@@ -44,8 +44,8 @@ public final class MicroCache2 extends MicroCache
 
     StatCoderContext bc = new StatCoderContext( dataBuffers.iobuffer );
 
-    TagValueCoder wayTagCoder = new TagValueCoder( bc, dataBuffers.tagbuf1, wayValidator );
-    TagValueCoder nodeTagCoder = new TagValueCoder( bc, dataBuffers.tagbuf1, null );
+    TagValueCoder wayTagCoder = new TagValueCoder( bc, dataBuffers, wayValidator );
+    TagValueCoder nodeTagCoder = new TagValueCoder( bc, dataBuffers, null );
     NoisyDiffCoder nodeIdxDiff = new NoisyDiffCoder( bc );
     NoisyDiffCoder nodeEleDiff = new NoisyDiffCoder( bc );
     NoisyDiffCoder extLonDiff = new NoisyDiffCoder(bc);
diff --git a/brouter-codec/src/main/java/btools/codec/TagValueCoder.java b/brouter-codec/src/main/java/btools/codec/TagValueCoder.java
index 5cddfb1..580c816 100644
--- a/brouter-codec/src/main/java/btools/codec/TagValueCoder.java
+++ b/brouter-codec/src/main/java/btools/codec/TagValueCoder.java
@@ -78,9 +78,9 @@ public final class TagValueCoder
     this.bc = bc;
   }
 
-  public TagValueCoder( BitCoderContext bc, byte[] buffer, TagValueValidator validator )
+  public TagValueCoder( BitCoderContext bc, DataBuffers buffers, TagValueValidator validator )
   {
-    tree = decodeTree( bc, buffer, validator );
+    tree = decodeTree( bc, buffers, validator );
     this.bc = bc;
   }
 
@@ -89,18 +89,20 @@ public final class TagValueCoder
     identityMap = new HashMap<TagValueSet, TagValueSet>();
   }
 
-  private Object decodeTree( BitCoderContext bc, byte[] buffer, TagValueValidator validator )
+  private Object decodeTree( BitCoderContext bc, DataBuffers buffers, TagValueValidator validator )
   {
     boolean isNode = bc.decodeBit();
     if ( isNode )
     {
       TreeNode node = new TreeNode();
-      node.child1 = decodeTree( bc, buffer, validator );
-      node.child2 = decodeTree( bc, buffer, validator );
+      node.child1 = decodeTree( bc, buffers, validator );
+      node.child2 = decodeTree( bc, buffers, validator );
       return node;
     }
 
-	  BitCoderContext ctx = new BitCoderContext( buffer );
+    byte[] buffer = buffers.tagbuf1;
+	  BitCoderContext ctx = buffers.bctx1;
+	  ctx.reset( buffer );
 
     int inum = 0;
     int lastEncodedInum = 0;
diff --git a/brouter-core/src/main/java/btools/router/OsmPath.java b/brouter-core/src/main/java/btools/router/OsmPath.java
index 90a6f8e..3e853cc 100644
--- a/brouter-core/src/main/java/btools/router/OsmPath.java
+++ b/brouter-core/src/main/java/btools/router/OsmPath.java
@@ -160,7 +160,11 @@ final class OsmPath implements OsmLinkHolder
       cost += iicost;
     }
 
-    OsmTransferNode transferNode = link.decodeFirsttransfer( p1 );
+//    OsmTransferNode transferNode = link.decodeGeometry( p1, rc.byteDataReaderGeometry, rc.transferNodeCache  );
+
+    OsmTransferNode transferNode = link.geometry == null ? null
+                  : rc.geometryDecoder.decodeGeometry( link.geometry, p1, link.targetNode, link.counterLinkWritten );
+
     OsmNode targetNode = link.targetNode;
     for(;;)
     {
diff --git a/brouter-core/src/main/java/btools/router/RoutingContext.java b/brouter-core/src/main/java/btools/router/RoutingContext.java
index bfd6ef8..053b954 100644
--- a/brouter-core/src/main/java/btools/router/RoutingContext.java
+++ b/brouter-core/src/main/java/btools/router/RoutingContext.java
@@ -12,6 +12,7 @@ import java.util.List;
 import btools.expressions.BExpressionContext;
 import btools.expressions.BExpressionContextNode;
 import btools.expressions.BExpressionContextWay;
+import btools.mapaccess.GeometryDecoder;
 
 public final class RoutingContext
 {
@@ -40,6 +41,8 @@ public final class RoutingContext
 
   public BExpressionContextWay expctxWay;
   public BExpressionContextNode expctxNode;
+  
+  public GeometryDecoder geometryDecoder = new GeometryDecoder();
 
   public int memoryclass = 64;
   
diff --git a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java
index c1122bd..03e537c 100644
--- a/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java
+++ b/brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java
@@ -44,6 +44,8 @@ public abstract class BExpressionContext implements IByteArrayUnifier
   private int[] lookupData = new int[0];
 
   private byte[] abBuf = new byte[256];
+  private BitCoderContext ctxEndode  = new BitCoderContext( abBuf );
+  private BitCoderContext ctxDecode = new BitCoderContext( new byte[0] );
   
   private Map<String,Integer> variableNumbers = new HashMap<String,Integer>();
 
@@ -121,11 +123,13 @@ public abstract class BExpressionContext implements IByteArrayUnifier
 
   public byte[] encode( int[] ld )
   {
-	// start with first bit hardwired ("reversedirection")
-	BitCoderContext ctx = new BitCoderContext( abBuf );
+    BitCoderContext ctx = ctxEndode;
+    ctx.reset();
 	
-	int skippedTags = 0;
-	int nonNullTags= 0;
+    int skippedTags = 0;
+    int nonNullTags= 0;
+
+    // (skip first bit ("reversedirection") )
   	
     // all others are generic
     for( int inum = 1; inum < lookupValues.size(); inum++ ) // loop over lookup names
@@ -174,13 +178,16 @@ public abstract class BExpressionContext implements IByteArrayUnifier
     decode( lookupData, false, ab );
     lookupDataValid = true;
   }
+  
+
 
   /**
    * decode a byte-array into a lookup data array
    */
   private void decode( int[] ld, boolean inverseDirection, byte[] ab )
   {
-    BitCoderContext ctx = new BitCoderContext(ab);
+    BitCoderContext ctx = ctxDecode;
+    ctx.reset( ab );
 	  
     // start with first bit hardwired ("reversedirection")
     ld[0] = inverseDirection ? 2 : 0;
diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/GeometryDecoder.java b/brouter-mapaccess/src/main/java/btools/mapaccess/GeometryDecoder.java
new file mode 100644
index 0000000..6533d5a
--- /dev/null
+++ b/brouter-mapaccess/src/main/java/btools/mapaccess/GeometryDecoder.java
@@ -0,0 +1,69 @@
+/**
+ * Container for link between two Osm nodes
+ *
+ * @author ab
+ */
+package btools.mapaccess;
+
+import btools.util.ByteDataReader;
+
+
+public final class GeometryDecoder
+{
+  private ByteDataReader r = new ByteDataReader( null );
+  private OsmTransferNode[] cachedNodes;
+  private int nCachedNodes = 128;
+
+  public GeometryDecoder()
+  {
+    // create some caches
+    cachedNodes = new OsmTransferNode[nCachedNodes];
+    for( int i=0; i<nCachedNodes; i++ )
+    {
+      cachedNodes[i] = new OsmTransferNode();
+    }
+  }
+
+
+
+  public OsmTransferNode decodeGeometry( byte[] geometry, OsmNode sourceNode, OsmNode targetNode, boolean reverseLink )
+  {
+      OsmTransferNode firstTransferNode = null;
+      OsmTransferNode lastTransferNode = null;
+      OsmNode startnode =  reverseLink ? targetNode : sourceNode;
+      r.reset( geometry );
+      int olon = startnode.ilon;
+      int olat = startnode.ilat;
+      int oselev = startnode.selev;
+      int idx = 0;
+      while ( r.hasMoreData() )
+      {
+        OsmTransferNode trans = idx < nCachedNodes ? cachedNodes[idx++] : new OsmTransferNode();
+        trans.ilon = olon + r.readVarLengthSigned();
+        trans.ilat = olat + r.readVarLengthSigned();
+        trans.selev = (short)(oselev + r.readVarLengthSigned());
+        olon = trans.ilon;
+        olat = trans.ilat;
+        oselev = trans.selev;
+        if ( reverseLink ) // reverse chaining
+        {
+          trans.next = firstTransferNode;
+          firstTransferNode = trans;
+        }
+        else
+        {
+          trans.next = null;
+          if ( lastTransferNode == null )
+          {
+            firstTransferNode = trans;
+          }
+          else
+          {
+            lastTransferNode.next = trans;
+          }
+          lastTransferNode = trans;
+        }
+      }
+      return firstTransferNode;
+  }
+}
diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java b/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java
index 8ccbc26..b36cb2c 100644
--- a/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java
+++ b/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java
@@ -7,9 +7,7 @@ package btools.mapaccess;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.List;
 
 import btools.codec.DataBuffers;
 import btools.codec.MicroCache;
@@ -43,10 +41,15 @@ public final class NodesCache
   
   private boolean garbageCollectionEnabled = false;
   private boolean ghostCleaningDone = false;
+
+
+  private long cacheSumClean = 0;
+  private long ghostSum = 0;
+  private long ghostWakeup = 0;
   
   public String formatStatus()
   {
-    return "collecting=" + garbageCollectionEnabled + " noGhosts=" + ghostCleaningDone + " cacheSum=" + cacheSum;
+    return "collecting=" + garbageCollectionEnabled + " noGhosts=" + ghostCleaningDone + " cacheSum=" + cacheSum + " cacheSumClean=" + cacheSumClean + " ghostSum=" + ghostSum + " ghostWakeup=" + ghostWakeup ;
   }
 
   public NodesCache( String segmentDir, OsmNodesMap nodesMap, BExpressionContextWay ctxWay, boolean forceSecondaryData, long maxmem, NodesCache oldCache )
@@ -90,6 +93,7 @@ public final class NodesCache
       dataBuffers = new DataBuffers();
       secondarySegmentsDir = StorageConfigHelper.getSecondarySegmentDir( segmentDir );
     }
+    ghostSum = cacheSum;
   }
   
   public void cleanNonVirgin()
@@ -110,39 +114,28 @@ public final class NodesCache
   private void checkEnableCacheCleaning()
   {
     if ( cacheSum < maxmem || ghostCleaningDone )
+    {
       return;
+    }
 
     for ( int i = 0; i < fileRows.length; i++ )
     {
       OsmFile[] fileRow = fileRows[i];
       if ( fileRow == null )
+      {
         continue;
-      int nghosts = 0;
+      }
       for ( OsmFile osmf : fileRow )
       {
         if ( garbageCollectionEnabled )
         {
-          if ( osmf.ghost )
-          {
-            nghosts++;
-          }
+          cacheSum -= osmf.cleanGhosts();
         }
         else
         {
-          osmf.cleanAll();
+          cacheSum -= osmf.collectAll();
         }
       }
-      if ( nghosts == 0 )
-        continue;
-      int j = 0;
-      OsmFile[] frow = new OsmFile[fileRow.length - nghosts];
-      for ( OsmFile osmf : fileRow )
-      {
-        if ( osmf.ghost )
-          continue;
-        frow[j++] = osmf;
-      }
-      fileRows[i] = frow;
     }
     
     if ( garbageCollectionEnabled )
@@ -150,7 +143,8 @@ public final class NodesCache
       ghostCleaningDone = true;
     }
     else
-    {    
+    {
+      cacheSumClean = cacheSum;
       garbageCollectionEnabled = true;
     }
   }
@@ -189,7 +183,6 @@ public final class NodesCache
         newFileRow[ndegrees] = osmf;
         fileRows[latDegree] = newFileRow;
       }
-      osmf.ghost = false;
       currentFileName = osmf.filename;
 
       if ( !osmf.hasData() )
@@ -208,6 +201,7 @@ public final class NodesCache
       else if ( segment.ghost )
       {
         segment.unGhost();
+        ghostWakeup += segment.getDataSize();
       }
       return segment;
     }
@@ -240,7 +234,7 @@ public final class NodesCache
 
     if ( garbageCollectionEnabled ) // garbage collection
     {
-      segment.collect( segment.getSize() >> 2 ); // threshold = 1/4 of size is deleted
+      cacheSum -= segment.collect( segment.getSize() >> 1 ); // threshold = 1/2 of size is deleted
     }
 
     return !node.isHollow();
diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java
index 6100fa6..1ad7e66 100644
--- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java
+++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java
@@ -29,8 +29,6 @@ final class OsmFile
 
   public String filename;
 
-  public boolean ghost = false;
-
   private int divisor;
   private int cellsize;
   private int ncaches;
@@ -162,7 +160,6 @@ final class OsmFile
   long setGhostState()
   {
     long sum = 0;
-    ghost = true;
     int nc = microCaches == null ? 0 : microCaches.length;
     for ( int i = 0; i < nc; i++ )
     {
@@ -182,8 +179,26 @@ final class OsmFile
     return sum;
   }
 
-  void cleanAll()
+  long collectAll()
   {
+    long deleted = 0;
+    int nc = microCaches == null ? 0 : microCaches.length;
+    for ( int i = 0; i < nc; i++ )
+    {
+      MicroCache mc = microCaches[i];
+      if ( mc == null )
+        continue;
+      if ( !mc.ghost )
+      {
+        deleted += mc.collect( 0 );
+      }
+    }
+    return deleted;
+  }
+
+  long cleanGhosts()
+  {
+    long deleted = 0;
     int nc = microCaches == null ? 0 : microCaches.length;
     for ( int i = 0; i < nc; i++ )
     {
@@ -194,11 +209,8 @@ final class OsmFile
       {
         microCaches[i] = null;
       }
-      else
-      {
-        mc.collect( 0 );
-      }
     }
+    return deleted;
   }
 
   void cleanNonVirgin()
diff --git a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmLink.java b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmLink.java
index 63d4fd2..11bfba6 100644
--- a/brouter-mapaccess/src/main/java/btools/mapaccess/OsmLink.java
+++ b/brouter-mapaccess/src/main/java/btools/mapaccess/OsmLink.java
@@ -5,7 +5,6 @@
  */
 package btools.mapaccess;
 
-import btools.util.ByteDataReader;
 
 
 public class OsmLink
@@ -33,47 +32,6 @@ public class OsmLink
     this.geometry = geometry;
   }
 
-  final public OsmTransferNode decodeFirsttransfer( OsmNode sourceNode )
-  {
-      if ( geometry == null ) return null;
-
-      OsmTransferNode firstTransferNode = null;
-      OsmTransferNode lastTransferNode = null;
-      OsmNode startnode =  counterLinkWritten ? targetNode : sourceNode;
-      ByteDataReader r = new ByteDataReader( geometry );
-      int olon = startnode.ilon;
-      int olat = startnode.ilat;
-      int oselev = startnode.selev;
-      while ( r.hasMoreData() )
-      {
-        OsmTransferNode trans = new OsmTransferNode();
-        trans.ilon = olon + r.readVarLengthSigned();
-        trans.ilat = olat + r.readVarLengthSigned();
-        trans.selev = (short)(oselev + r.readVarLengthSigned());
-        olon = trans.ilon;
-        olat = trans.ilat;
-        oselev = trans.selev;
-        if ( counterLinkWritten ) // reverse chaining
-        {
-          trans.next = firstTransferNode;
-          firstTransferNode = trans;
-        }
-        else
-        {
-          if ( lastTransferNode == null )
-          {
-            firstTransferNode = trans;
-          }
-          else
-          {
-            lastTransferNode.next = trans;
-          }
-          lastTransferNode = trans;
-        }
-      }
-      return firstTransferNode;
-  }
-
    final public void addLinkHolder( OsmLinkHolder holder )
    {
      if ( firstlinkholder != null ) { holder.setNextForLink( firstlinkholder ); }
diff --git a/brouter-util/src/main/java/btools/util/BitCoderContext.java b/brouter-util/src/main/java/btools/util/BitCoderContext.java
index c6d0dc7..825eec9 100644
--- a/brouter-util/src/main/java/btools/util/BitCoderContext.java
+++ b/brouter-util/src/main/java/btools/util/BitCoderContext.java
@@ -15,6 +15,21 @@ public class BitCoderContext
     this.ab = ab;
     idxMax = ab.length-1;
   }
+  
+  public final void reset( byte[] ab )
+  {
+    this.ab = ab;
+    idxMax = ab.length-1;
+    reset();
+  }    
+
+  public final void reset()
+  {
+    idx = -1;
+    bm = 0x100;
+    bits = 0;
+    b = 0;
+  }    
 
   /**
    * encode a distance with a variable bit length
diff --git a/brouter-util/src/main/java/btools/util/ByteDataReader.java b/brouter-util/src/main/java/btools/util/ByteDataReader.java
index 8209c75..489ca69 100644
--- a/brouter-util/src/main/java/btools/util/ByteDataReader.java
+++ b/brouter-util/src/main/java/btools/util/ByteDataReader.java
@@ -24,6 +24,14 @@ public class ByteDataReader
 	 aboffset = offset;
 	 aboffsetEnd = ab == null ? 0 : ab.length;
   }
+  
+  public final void reset( byte[] byteArray )
+  {
+    ab = byteArray;
+    aboffset = 0;
+    aboffsetEnd = ab == null ? 0 : ab.length;
+  }
+  
 
   public final int readInt()
   {