// context for simple expression
// context means:
// - the local variables
// - the local variable names
// - the lookup-input variables

package btools.expressions;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.StringTokenizer;
import java.util.TreeMap;

import btools.util.BitCoderContext;
import btools.util.Crc32;
import btools.util.IByteArrayUnifier;
import btools.util.LruMap;


public abstract class BExpressionContext implements IByteArrayUnifier
{
  private  static final String CONTEXT_TAG = "---context:";
	
  private String context;
  private boolean _inOurContext = false;
  private BufferedReader _br = null;
  private boolean _readerDone = false;

  private Map<String,Integer> lookupNumbers = new HashMap<String,Integer>();
  private ArrayList<BExpressionLookupValue[]> lookupValues = new ArrayList<BExpressionLookupValue[]>();
  private ArrayList<String> lookupNames = new ArrayList<String>();
  private ArrayList<int[]> lookupHistograms = new ArrayList<int[]>();
  private boolean[] lookupIdxUsed;

  private boolean lookupDataFrozen = false;

  private int[] lookupData = new int[0];

  private byte[] abBuf = new byte[256];
  private BitCoderContext ctxEndode  = new BitCoderContext( abBuf );
  private BitCoderContext ctxDecode = new BitCoderContext( new byte[0] );
  
  private Map<String,Integer> variableNumbers = new HashMap<String,Integer>();

  private float[] variableData;


  // hash-cache for function results
  private CacheNode probeCacheNode = new CacheNode();
  private LruMap cache;

  private VarWrapper probeVarSet = new VarWrapper();
  private LruMap resultVarCache;

  private List<BExpression> expressionList;

  private int minWriteIdx;

  // build-in variable indexes for fast access
  private int[] buildInVariableIdx;
  private int nBuildInVars;
  
  private float[] currentVars;
  private int currentVarOffset;
  
  protected void setInverseVars()
  {
    currentVarOffset = nBuildInVars;
  }

  abstract String[] getBuildInVariableNames();

  protected final float getBuildInVariable( int idx ) 
  {
    return currentVars[idx+currentVarOffset];
  }

  private int linenr;

  public BExpressionMetaData meta;
  private boolean lookupDataValid = false;

  protected BExpressionContext( String context, BExpressionMetaData meta )
  {
    this( context, 4096, meta );
  }

  /**
   * Create an Expression-Context for the given node
   *
   * @param context  global, way or node - context of that instance
   * @param hashSize  size of hashmap for result caching
   */
  protected BExpressionContext( String context, int hashSize, BExpressionMetaData meta )
  {
     this.context = context;
     this.meta = meta;
     
     if ( meta != null ) meta.registerListener(context, this );

     if ( Boolean.getBoolean( "disableExpressionCache" ) ) hashSize = 1;
      
     // create the expression cache
     cache = new LruMap( 4*hashSize, hashSize );
     resultVarCache = new LruMap( 4096, 4096 );
  }

  /**
   * encode internal lookup data to a byte array
   */
  public byte[] encode()
  {
	if ( !lookupDataValid ) throw new IllegalArgumentException( "internal error: encoding undefined data?" );
    return encode( lookupData );
  }

  public byte[] encode( int[] ld )
  {
    BitCoderContext ctx = ctxEndode;
    ctx.reset();
	
    int skippedTags = 0;
    int nonNullTags= 0;

    // (skip first bit ("reversedirection") )
  	
    // all others are generic
    for( int inum = 1; inum < lookupValues.size(); inum++ ) // loop over lookup names
    {
      int d = ld[inum];
      if ( d == 0 )
      {
        skippedTags++;
        continue;
      }
      ctx.encodeVarBits( skippedTags+1 );
      nonNullTags++;
      skippedTags = 0;
      
      // 0 excluded already, 1 (=unknown) we rotate up to 8
      // to have the good code space for the popular values
      int dd = d < 2 ? 7 : ( d < 9 ? d - 2 : d - 1);
      ctx.encodeVarBits(  dd );
    }
    ctx.encodeVarBits( 0 );
    
    if ( nonNullTags == 0) return null;
    
    int len = ctx.getEncodedLength();
    byte[] ab = new byte[len];
    System.arraycopy( abBuf, 0, ab, 0, len );
    
    
    // crosscheck: decode and compare
    int[] ld2 = new int[lookupValues.size()];
    decode( ld2, false, ab );
    for( int inum = 1; inum < lookupValues.size(); inum++ ) // loop over lookup names (except reverse dir)
    {
      if ( ld2[inum] != ld[inum] ) throw new RuntimeException( "assertion failed encoding inum=" + inum + " val=" + ld[inum] + " " + getKeyValueDescription(false, ab) );
    }    
    
    return ab;
  }


  /**
   * decode byte array to internal lookup data
   */
  public void decode( byte[] ab )
  {
    decode( lookupData, false, ab );
    lookupDataValid = true;
  }
  


  /**
   * decode a byte-array into a lookup data array
   */
  private void decode( int[] ld, boolean inverseDirection, byte[] ab )
  {
    BitCoderContext ctx = ctxDecode;
    ctx.reset( ab );
	  
    // start with first bit hardwired ("reversedirection")
    ld[0] = inverseDirection ? 2 : 0;
  	
    // all others are generic
  	int inum = 1;
    for(;;)
    {
      int delta = ctx.decodeVarBits();
      if ( delta == 0) break;
      if ( inum + delta > ld.length ) break; // higher minor version is o.k.
      
      while ( delta-- > 1 ) ld[inum++] = 0;

      // see encoder for value rotation
      int dd = ctx.decodeVarBits();
      int d = dd == 7 ? 1 : ( dd < 7 ? dd + 2 : dd + 1);
      if ( d >= lookupValues.get(inum).length ) d = 1; // map out-of-range to unknown
      ld[inum++] = d;
    }
    while( inum < ld.length ) ld[inum++] = 0;
  }

  public String getKeyValueDescription( boolean inverseDirection, byte[] ab )
  {
	StringBuilder sb = new StringBuilder( 200 );
    decode( lookupData, inverseDirection, ab );
    for( int inum = 0; inum < lookupValues.size(); inum++ ) // loop over lookup names
    {
      BExpressionLookupValue[] va = lookupValues.get(inum);
      String value = va[lookupData[inum]].toString();
      if ( value != null && value.length() > 0 )
      {
        if ( sb.length() > 0 ) sb.append( ' ' );
        sb.append(lookupNames.get( inum ) + "=" + value );
      }
    }
    return sb.toString();
  }

  private int parsedLines = 0;
  private boolean fixTagsWritten = false;
  
  public void parseMetaLine( String line )
  {
      parsedLines++;
      StringTokenizer tk = new StringTokenizer( line, " " );
      String name = tk.nextToken();
      String value = tk.nextToken();
      int idx = name.indexOf( ';' );
      if ( idx >= 0 ) name = name.substring( 0, idx );

      if ( !fixTagsWritten )
      {
        fixTagsWritten = true;
        if ( "way".equals( context ) ) addLookupValue( "reversedirection", "yes", null );
        else if ( "node".equals( context ) ) addLookupValue( "nodeaccessgranted", "yes", null );
      }
      if ( "reversedirection".equals( name ) ) return; // this is hardcoded
      if ( "nodeaccessgranted".equals( name ) ) return; // this is hardcoded
      BExpressionLookupValue newValue = addLookupValue( name, value, null );

      // add aliases
      while( newValue != null && tk.hasMoreTokens() ) newValue.addAlias( tk.nextToken() );
  }
  
  public void finishMetaParsing()
  {
    if ( parsedLines == 0 && !"global".equals(context) )
    {
      throw new IllegalArgumentException( "lookup table does not contain data for context " + context + " (old version?)" );
    }

    // post-process metadata:
    lookupDataFrozen = true;
    
    lookupIdxUsed = new boolean[lookupValues.size()];
  }

  public final void evaluate( int[] lookupData2 )
  {
    lookupData = lookupData2;
    evaluate();
  }

  private void evaluate()
  {
    int n = expressionList.size();
    for( int expidx = 0; expidx < n; expidx++ )
    {
      expressionList.get( expidx ).evaluate( this );
    }
  }

  private long requests;
  private long requests2;
  private long cachemisses;
  
  public String cacheStats()
  {
    return "requests=" + requests + " requests2=" + requests2 + " cachemisses=" + cachemisses;
  }
  
  private CacheNode lastCacheNode = new CacheNode();

  // @Override
  public final byte[] unify( byte[] ab, int offset, int len )
  {
    probeCacheNode.ab = null; // crc based cache lookup only
    probeCacheNode.crc = Crc32.crc( ab, offset, len );

    CacheNode cn = (CacheNode)cache.get( probeCacheNode );
    if ( cn != null )
    {
      byte[] cab = cn.ab;
      if ( cab.length == len )
      {
        for( int i=0; i<len; i++ )
        {
          if ( cab[i] != ab[i+offset] )
          {
            cn = null;
            break;
          }
        }
        if ( cn != null )
        {
          lastCacheNode = cn;
          return cn.ab;
        }
      }
    }
    byte[] nab = new byte[len];
    System.arraycopy( ab, offset, nab, 0, len );
    return nab;
  }


  public final void evaluate( boolean inverseDirection, byte[] ab )
  {
    requests++;
    lookupDataValid = false; // this is an assertion for a nasty pifall

    CacheNode cn;
    if ( lastCacheNode.ab == ab )
    {
      cn = lastCacheNode;
    }
    else
    {
      probeCacheNode.ab = ab;
      probeCacheNode.crc = Crc32.crc( ab, 0, ab.length );
      cn = (CacheNode)cache.get( probeCacheNode );
    }

    if ( cn == null )
    {
      cachemisses++;
    
      cn = (CacheNode)cache.removeLru();
      if ( cn == null )
      {
        cn = new CacheNode();
      }
      cn.crc = probeCacheNode.crc;
      cn.ab = ab;
      cache.put( cn );

      if ( probeVarSet.vars == null )
      {
        probeVarSet.vars = new float[2*nBuildInVars];
      }

      // forward direction
      decode( lookupData, false, ab );
      evaluateInto( probeVarSet.vars, 0 );

      // inverse direction
      lookupData[0] = 2; // inverse shortcut: reuse decoding
      evaluateInto( probeVarSet.vars, nBuildInVars );
      
      probeVarSet.hash = Arrays.hashCode( probeVarSet.vars );
      
      // unify the result variable set
      VarWrapper vw = (VarWrapper)resultVarCache.get( probeVarSet );
      if ( vw == null )
      {
        vw = (VarWrapper)resultVarCache.removeLru();
        if ( vw == null )
        {
          vw = new VarWrapper();
        }
        vw.hash = probeVarSet.hash;
        vw.vars = probeVarSet.vars;
        probeVarSet.vars = null;
        resultVarCache.put( vw );
      }
      cn.vars = vw.vars;
    }
    else
    {
      if ( ab == cn.ab ) requests2++;
    
      cache.touch( cn );
    }

    currentVars = cn.vars;
    currentVarOffset = inverseDirection ? nBuildInVars : 0;
  }

  private void evaluateInto( float[] vars, int offset )
  {
      evaluate();
      for ( int vi = 0; vi < nBuildInVars; vi++ )
      {
        int idx = buildInVariableIdx[vi];
        vars[vi+offset] = idx == -1 ? 0.f : variableData[idx];
      }
  }


  public void dumpStatistics()
  {
    TreeMap<String,String> counts = new TreeMap<String,String>();
    // first count
    for( String name: lookupNumbers.keySet() )
    {
      int cnt = 0;
      int inum = lookupNumbers.get(name).intValue();
      int[] histo = lookupHistograms.get(inum);
//    if ( histo.length == 500 ) continue;
      for( int i=2; i<histo.length; i++ )
      {
        cnt += histo[i];
      }
      counts.put( "" + ( 1000000000 + cnt) + "_" + name, name );
    }

    while( counts.size() > 0 )
    {
      String key = counts.lastEntry().getKey();
      String name = counts.get(key);
      counts.remove( key );
      int inum = lookupNumbers.get(name).intValue();
      BExpressionLookupValue[] values = lookupValues.get(inum);
      int[] histo = lookupHistograms.get(inum);
      if ( values.length == 1000 ) continue;
      String[] svalues = new String[values.length];
      for( int i=0; i<values.length; i++ )
      {
        String scnt = "0000000000" + histo[i];
        scnt = scnt.substring( scnt.length() - 10 );
        svalues[i] =  scnt + " " + values[i].toString();
      }
      Arrays.sort( svalues );
      for( int i=svalues.length-1; i>=0; i-- )
      {
        System.out.println( name + ";" + svalues[i] );
      }
    }
  }

  /**
   * @return a new lookupData array, or null if no metadata defined
   */
  public int[] createNewLookupData()
  {
    if ( lookupDataFrozen )
    {
      return new int[lookupValues.size()];
    }
    return null;
  }

  /**
   * generate random values for regression testing
   */
  public int[] generateRandomValues( Random rnd )
  {
    int[] data = createNewLookupData();
    data[0] = 2*rnd.nextInt( 2 ); // reverse-direction = 0 or 2
    for( int inum = 1; inum < data.length; inum++ )
    {
      int nvalues = lookupValues.get( inum ).length;
      data[inum] = 0;
      if ( inum > 1 && rnd.nextInt( 10 ) > 0 ) continue; // tags other than highway only 10%
      data[inum] = rnd.nextInt( nvalues );
    }
    lookupDataValid = true;
    return data;
  }

  public void assertAllVariablesEqual( BExpressionContext other )
  {
    int nv = variableData.length;
    int nv2 = other.variableData.length;
    if ( nv != nv2 ) throw new RuntimeException( "mismatch in variable-count: " + nv + "<->" + nv2 );
    for( int i=0; i<nv; i++ )
    {
      if ( variableData[i] != other.variableData[i] )
      {
        throw new RuntimeException( "mismatch in variable " + variableName(i) + " " + variableData[i] + "<->" + other.variableData[i]
           + "\ntags = " + getKeyValueDescription( false, encode()  ) );
      }
    }
  }

  private String variableName( int idx )
  {
    for( Map.Entry<String,Integer> e : variableNumbers.entrySet() )
    {
      if ( e.getValue().intValue() == idx )
      {
        return e.getKey();
      }
    }
    throw new RuntimeException( "no variable for index" + idx );
  }

  /**
   * add a new lookup-value for the given name to the given lookupData array.
   * If no array is given (null value passed), the value is added to
   * the context-binded array. In that case, unknown names and values are
   * created dynamically.
   *
   * @return a newly created value element, if any, to optionally add aliases
   */
  public BExpressionLookupValue addLookupValue( String name, String value, int[] lookupData2 )
  {
    BExpressionLookupValue newValue = null;
    Integer num = lookupNumbers.get( name );
    if ( num == null )
    {
      if ( lookupData2 != null )
      {
        // do not create unknown name for external data array
        return newValue;
      }

      // unknown name, create
      num = new Integer( lookupValues.size() );
      lookupNumbers.put( name, num );
      lookupNames.add( name );
      lookupValues.add( new BExpressionLookupValue[]{ new BExpressionLookupValue( "" )
                                                    , new BExpressionLookupValue( "unknown" ) } );
      lookupHistograms.add( new int[2] );
      int[] ndata = new int[lookupData.length+1];
      System.arraycopy( lookupData, 0, ndata, 0, lookupData.length );
      lookupData = ndata;
    }

    // look for that value
    int inum = num.intValue();
    BExpressionLookupValue[] values = lookupValues.get( inum );
    int[] histo = lookupHistograms.get( inum );
    int i=0;
    for( ; i<values.length; i++ )
    {
      BExpressionLookupValue v = values[i];
      if ( v.matches( value ) ) break;
    }
    if ( i == values.length )
    {
      if ( lookupData2 != null )
      {
        // do not create unknown value for external data array,
        // record as 'other' instead
        lookupData2[inum] = 1;
        return newValue;
      }

      if ( i == 499 )
      {
        // System.out.println( "value limit reached for: " + name );
      }
      if ( i == 500 )
      {
        return newValue;
      }
      // unknown value, create
      BExpressionLookupValue[] nvalues = new BExpressionLookupValue[values.length+1];
      int[] nhisto = new int[values.length+1];
      System.arraycopy( values, 0, nvalues, 0, values.length );
      System.arraycopy( histo, 0, nhisto, 0, histo.length );
      values = nvalues;
      histo = nhisto;
      newValue = new BExpressionLookupValue( value );
      values[i] = newValue;
      lookupHistograms.set(inum, histo);
      lookupValues.set(inum, values);
    }

    histo[i]++;

    // finally remember the actual data
    if ( lookupData2 != null ) lookupData2[inum] = i;
    else lookupData[inum] = i;
    return newValue;
  }

  /**
   * add a value-index to to internal array
   * value-index means 0=unknown, 1=other, 2=value-x, ...
   */
  public void addLookupValue( String name, int valueIndex )
  {
    Integer num = lookupNumbers.get( name );
    if ( num == null )
    {
      return;
    }

    // look for that value
    int inum = num.intValue();
    int nvalues = lookupValues.get( inum ).length;
    if ( valueIndex < 0 || valueIndex >= nvalues ) throw new IllegalArgumentException( "value index out of range for name " + name + ": " + valueIndex );
    lookupData[inum] = valueIndex;
  }


  /**
   * special hack for yes/proposed relations:
   * add a lookup value if not yet a smaller, >1 value was added
   * add a 2=yes if the provided value is out of range
   * value-index means here 0=unknown, 1=other, 2=yes, 3=proposed
   */
  public void addSmallestLookupValue( String name, int valueIndex )
  {
    Integer num = lookupNumbers.get( name );
    if ( num == null )
    {
      return;
    }

    // look for that value
    int inum = num.intValue();
    int nvalues = lookupValues.get( inum ).length;
    int oldValueIndex = lookupData[inum];
    if ( oldValueIndex > 1 && oldValueIndex < valueIndex )
    {
      return;
    }
    if ( valueIndex >= nvalues )
    {
      valueIndex = nvalues-1;
    }
    if ( valueIndex < 0 ) throw new IllegalArgumentException( "value index out of range for name " + name + ": " + valueIndex );
    lookupData[inum] = valueIndex;
  }

  public boolean getBooleanLookupValue( String name )
  {
    Integer num = lookupNumbers.get( name );
    return num != null && lookupData[num.intValue()] == 2;
  }

  public void parseFile( File file, String readOnlyContext )
  {
    if ( !file.exists() )
    {
      throw new IllegalArgumentException( "profile "  + file + " does not exist" );
    }
    try
    {
      if ( readOnlyContext != null )
      {
        linenr = 1;
        String realContext = context;
        context = readOnlyContext;
        expressionList = _parseFile( file );
        variableData = new float[variableNumbers.size()];
        evaluate( lookupData ); // lookupData is dummy here - evaluate just to create the variables
        context = realContext;
      }
      linenr = 1;
      minWriteIdx = variableData == null ? 0 : variableData.length;

      expressionList = _parseFile( file );

      // determine the build-in variable indices
      String[] varNames = getBuildInVariableNames();
      nBuildInVars = varNames.length;
      buildInVariableIdx = new int[nBuildInVars];
      for( int vi=0; vi<varNames.length; vi++ )
      {
        buildInVariableIdx[vi] = getVariableIdx( varNames[vi], false );
      }

      float[] readOnlyData = variableData;
      variableData = new float[variableNumbers.size()];
      for( int i=0; i<minWriteIdx; i++ )
      {
        variableData[i] = readOnlyData[i];
      }
    }
    catch( Exception e )
    {
      if ( e instanceof IllegalArgumentException )
      {
        throw new IllegalArgumentException( "ParseException at line " + linenr + ": " + e.getMessage() );
      }
      throw new RuntimeException( e );
    }
    if ( expressionList.size() == 0 )
    {
        throw new IllegalArgumentException( file.getAbsolutePath()
             + " does not contain expressions for context " + context + " (old version?)" );
    }
  }

  private List<BExpression> _parseFile( File file ) throws Exception
  {
    _br = new BufferedReader( new FileReader( file ) );
    _readerDone = false;
    List<BExpression> result = new ArrayList<BExpression>();
    for(;;)
    {
      BExpression exp = BExpression.parse( this, 0 );
      if ( exp == null ) break;
      result.add( exp );
    }
    _br.close();
    _br = null;
    return result;
  }


  public float getVariableValue( String name, float defaultValue )
  {
    Integer num = variableNumbers.get( name );
    return num == null ? defaultValue : getVariableValue( num.intValue() );
  }

  float getVariableValue( int variableIdx )
  {
    return variableData[variableIdx];
  }

  int getVariableIdx( String name, boolean create )
  {
    Integer num = variableNumbers.get( name );
    if ( num == null )
    {
      if ( create )
      {
        num = new Integer( variableNumbers.size() );
        variableNumbers.put( name, num );
      }
      else
      {
        return -1;
      }
    }
    return num.intValue();
  }

  int getMinWriteIdx()
  {
    return minWriteIdx;
  }

  float getLookupMatch( int nameIdx, int[] valueIdxArray )
  {
    for( int i=0; i<valueIdxArray.length; i++ )
    {
      if ( lookupData[nameIdx] == valueIdxArray[i] )
      {
        return 1.0f;
      }
    }
    return 0.0f;
  }

  public int getLookupNameIdx( String name )
  {
    Integer num = lookupNumbers.get( name );
    return num == null ? -1 : num.intValue();
  }

  public final void markLookupIdxUsed( int idx )
  {
    lookupIdxUsed[ idx ] = true;
  }
  
  public final boolean isLookupIdxUsed( int idx )
  {
    return idx < lookupIdxUsed.length ? lookupIdxUsed[idx] : false;
  }

  public final void setAllTagsUsed()
  {
    for( int i=0; i<lookupIdxUsed.length; i++ )
    {
      lookupIdxUsed[i] = true;
    }
  }

  int getLookupValueIdx( int nameIdx, String value )
  {
    BExpressionLookupValue[] values = lookupValues.get( nameIdx );
    for( int i=0; i< values.length; i++ )
    {
      if ( values[i].equals( value ) ) return i;
    }
    return -1;
  }


  String parseToken() throws Exception
  {
    for(;;)
    {
      String token = _parseToken();
      if ( token == null ) return null;
      if ( token.startsWith( CONTEXT_TAG ) )
      {
        _inOurContext = token.substring( CONTEXT_TAG.length() ).equals( context );
      }
      else if ( _inOurContext )
      {
        return token;
      }
    }
  }


  private String _parseToken() throws Exception
  {
    StringBuilder sb = new StringBuilder(32);
    boolean inComment = false;
    for(;;)
    {
      int ic = _readerDone ? -1 : _br.read();
      if ( ic < 0 )
      {
        if ( sb.length() == 0 ) return null;
        _readerDone = true;
         return sb.toString();
      }
      char c = (char)ic;
      if ( c == '\n' ) linenr++;

      if ( inComment )
      {
        if ( c == '\r' || c == '\n' ) inComment = false;
        continue;
      }
      if ( Character.isWhitespace( c ) )
      {
        if ( sb.length() > 0 ) return sb.toString();
        else continue;
      }
      if ( c == '#' && sb.length() == 0 ) inComment = true;
      else sb.append( c );
    }
  }

  float assign( int variableIdx, float value )
  {
    variableData[variableIdx] = value;
    return value;
  }

}