brouter/brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java

/**
 * Efficient cache or osmnodes
 *
 * @author ab
 */
package btools.mapaccess;

import java.io.File;
import java.io.IOException;
import java.util.HashMap;

import btools.codec.DataBuffers;
import btools.codec.MicroCache;
import btools.codec.WaypointMatcher;
import btools.expressions.BExpressionContextWay;

public final class NodesCache
{
  private File segmentDir;
  private File secondarySegmentsDir = null;

  private OsmNodesMap nodesMap;
  private BExpressionContextWay expCtxWay;
  private int lookupVersion;
  private int lookupMinorVersion;
  private boolean forceSecondaryData;
  private String currentFileName;

  private HashMap<String, PhysicalFile> fileCache;
  private DataBuffers dataBuffers;

  private OsmFile[][] fileRows;

  public WaypointMatcher waypointMatcher;

  public boolean first_file_access_failed = false;
  public String first_file_access_name;

  private long cacheSum = 0;
  private long maxmem;
  private boolean detailed;
  
  private boolean garbageCollectionEnabled = false;
  private boolean ghostCleaningDone = false;


  private long cacheSumClean = 0;
  private long ghostSum = 0;
  private long ghostWakeup = 0;
  
  public String formatStatus()
  {
    return "collecting=" + garbageCollectionEnabled + " noGhosts=" + ghostCleaningDone + " cacheSum=" + cacheSum + " cacheSumClean=" + cacheSumClean + " ghostSum=" + ghostSum + " ghostWakeup=" + ghostWakeup ;
  }

  public NodesCache( String segmentDir, OsmNodesMap nodesMap, BExpressionContextWay ctxWay, boolean forceSecondaryData, long maxmem, NodesCache oldCache, boolean detailed )
  {
    this.segmentDir = new File( segmentDir );
    this.nodesMap = nodesMap;
    this.expCtxWay = ctxWay;
    this.lookupVersion = ctxWay.meta.lookupVersion;
    this.lookupMinorVersion = ctxWay.meta.lookupMinorVersion;
    this.forceSecondaryData = forceSecondaryData;
    this.maxmem = maxmem;
    this.detailed = detailed;
    
    if ( ctxWay != null )
    {
      ctxWay.setDecodeForbidden( detailed );
    }

    first_file_access_failed = false;
    first_file_access_name = null;

    if ( !this.segmentDir.isDirectory() )
      throw new RuntimeException( "segment directory " + segmentDir + " does not exist" );

    if ( oldCache != null )
    {
      fileCache = oldCache.fileCache;
      dataBuffers = oldCache.dataBuffers;
      secondarySegmentsDir = oldCache.secondarySegmentsDir;

      // re-use old, virgin caches (if same detail-mode)
      if ( oldCache.detailed == detailed)
      {
        fileRows = oldCache.fileRows;
        for ( OsmFile[] fileRow : fileRows )
        {
          if ( fileRow == null )
            continue;
          for ( OsmFile osmf : fileRow )
          {
            cacheSum += osmf.setGhostState();
          }
        }
      }
      else
      {
        fileRows = new OsmFile[180][];
      }
    }
    else
    {
      fileCache = new HashMap<String, PhysicalFile>( 4 );
      fileRows = new OsmFile[180][];
      dataBuffers = new DataBuffers();
      secondarySegmentsDir = StorageConfigHelper.getSecondarySegmentDir( segmentDir );
    }
    ghostSum = cacheSum;
  }
  
  public void clean( boolean all )
  {
      for ( OsmFile[] fileRow : fileRows )
      {
        if ( fileRow == null )
          continue;
        for ( OsmFile osmf : fileRow )
        {
          osmf.clean( all);
        }
      }
  }

  // if the cache sum exceeded a threshold,
  // clean all ghosts and enable garbage collection
  private void checkEnableCacheCleaning()
  {
    if ( cacheSum < maxmem )
    {
      return;
    }

    for ( int i = 0; i < fileRows.length; i++ )
    {
      OsmFile[] fileRow = fileRows[i];
      if ( fileRow == null )
      {
        continue;
      }
      for ( OsmFile osmf : fileRow )
      {
        if ( garbageCollectionEnabled && !ghostCleaningDone )
        {
          cacheSum -= osmf.cleanGhosts();
        }
        else
        {
          cacheSum -= osmf.collectAll();
        }
      }
    }
    
    if ( garbageCollectionEnabled )
    {
      ghostCleaningDone = true;
      maxmem *= 2;
    }
    else
    {
      cacheSumClean = cacheSum;
      garbageCollectionEnabled = true;
    }
  }

  public int loadSegmentFor( int ilon, int ilat )
  {
    MicroCache mc = getSegmentFor( ilon, ilat );
    return mc == null ? 0 : mc.getSize();
  }

  public MicroCache getSegmentFor( int ilon, int ilat )
  {
    try
    {
      int lonDegree = ilon / 1000000;
      int latDegree = ilat / 1000000;
      OsmFile osmf = null;
      OsmFile[] fileRow = fileRows[latDegree];
      int ndegrees = fileRow == null ? 0 : fileRow.length;
      for ( int i = 0; i < ndegrees; i++ )
      {
        if ( fileRow[i].lonDegree == lonDegree )
        {
          osmf = fileRow[i];
          break;
        }
      }
      if ( osmf == null )
      {
        osmf = fileForSegment( lonDegree, latDegree );
        OsmFile[] newFileRow = new OsmFile[ndegrees + 1];
        for ( int i = 0; i < ndegrees; i++ )
        {
          newFileRow[i] = fileRow[i];
        }
        newFileRow[ndegrees] = osmf;
        fileRows[latDegree] = newFileRow;
      }
      currentFileName = osmf.filename;

      if ( !osmf.hasData() )
      {
        return null;
      }

      MicroCache segment = osmf.getMicroCache( ilon, ilat );
      if ( segment == null )
      {
        checkEnableCacheCleaning();
        segment = osmf.createMicroCache( ilon, ilat, dataBuffers, expCtxWay, waypointMatcher );

        cacheSum += segment.getDataSize();
      }
      else if ( segment.ghost )
      {
        segment.unGhost();
        ghostWakeup += segment.getDataSize();
      }
      return segment;
    }
    catch (RuntimeException re)
    {
      throw re;
    }
    catch (Exception e)
    {
      throw new RuntimeException( "error reading datafile " + currentFileName + ": " + e, e );
    }
  }

  public boolean obtainNonHollowNode( OsmNode node )
  {
    if ( !node.isHollow() )
      return true;

    MicroCache segment = getSegmentFor( node.ilon, node.ilat );
    if ( segment == null )
    {
      return false;
    }

    long id = node.getIdFromPos();
    if ( segment.getAndClear( id ) )
    {
      node.parseNodeBody( segment, nodesMap, expCtxWay );
    }

    if ( garbageCollectionEnabled ) // garbage collection
    {
      cacheSum -= segment.collect( segment.getSize() >> 1 ); // threshold = 1/2 of size is deleted
    }

    return !node.isHollow();
  }

  private OsmFile fileForSegment( int lonDegree, int latDegree ) throws Exception
  {
    int lonMod5 = lonDegree % 5;
    int latMod5 = latDegree % 5;

    int lon = lonDegree - 180 - lonMod5;
    String slon = lon < 0 ? "W" + ( -lon ) : "E" + lon;
    int lat = latDegree - 90 - latMod5;

    String slat = lat < 0 ? "S" + ( -lat ) : "N" + lat;
    String filenameBase = slon + "_" + slat;

    currentFileName = filenameBase + ".rd5";

    PhysicalFile ra = null;
    if ( !fileCache.containsKey( filenameBase ) )
    {
      File f = null;
      if ( !forceSecondaryData )
      {
        File primary = new File( segmentDir, filenameBase + ".rd5" );
        if ( primary .exists() )
        {
          f = primary;
        }
      }
      if ( f == null )
      {
        File secondary = new File( secondarySegmentsDir, filenameBase + ".rd5" );
        if ( secondary.exists() )
        {
          f = secondary;
        }
      }
      if ( f != null )
      {
        currentFileName = f.getName();
        ra = new PhysicalFile( f, dataBuffers, lookupVersion, lookupMinorVersion );
      }
      fileCache.put( filenameBase, ra );
    }
    ra = fileCache.get( filenameBase );
    OsmFile osmf = new OsmFile( ra, lonDegree, latDegree, dataBuffers );

    if ( first_file_access_name == null )
    {
      first_file_access_name = currentFileName;
      first_file_access_failed = osmf.filename == null;
    }

    return osmf;
  }

  public void close()
  {
    for ( PhysicalFile f : fileCache.values() )
    {
      try
      {
        if ( f != null )
          f.ra.close();
      }
      catch (IOException ioe)
      {
        // ignore
      }
    }
  }
}