Merge pull request #573 from abrensch/constant_exp_opti

optimizing constant expressions in profile parsing

brouter-core/src/main/java/btools/router/ProfileCache.java

@@ -91,8 +91,8 @@ public final class ProfileCache {
 
     meta.readMetaData(new File(profileDir, "lookups.dat"));
 
-    rc.expctxWay.parseFile(profileFile, "global");
+    rc.expctxWay.parseFile(profileFile, "global", rc.keyValues);
-    rc.expctxNode.parseFile(profileFile, "global");
+    rc.expctxNode.parseFile(profileFile, "global", rc.keyValues);
 
     rc.readGlobalConfig();
 

brouter-core/src/main/java/btools/router/RoutingContext.java

@@ -108,16 +108,6 @@ public final class RoutingContext {
 
   public void readGlobalConfig() {
     BExpressionContext expctxGlobal = expctxWay; // just one of them...
-
-    if (keyValues != null) {
-      // add parameter to context
-      for (Map.Entry<String, String> e : keyValues.entrySet()) {
-        float f = Float.parseFloat(e.getValue());
-        expctxWay.setVariableValue(e.getKey(), f, true);
-        expctxNode.setVariableValue(e.getKey(), f, true);
-      }
-    }
-
     setModel(expctxGlobal._modelClass);
 
     carMode = 0.f != expctxGlobal.getVariableValue("validForCars", 0.f);

brouter-expressions/src/main/java/btools/expressions/BExpression.java

@@ -32,8 +32,9 @@ final class BExpression {
   private BExpression op3;
   private float numberValue;
   private int variableIdx;
-  private int lookupNameIdx;
+  private int lookupNameIdx = -1;
   private int[] lookupValueIdxArray;
+  private boolean doNotChange;
 
   // Parse the expression and all subexpression
   public static BExpression parse(BExpressionContext ctx, int level) throws Exception {
@@ -41,6 +42,68 @@ final class BExpression {
   }
 
   private static BExpression parse(BExpressionContext ctx, int level, String optionalToken) throws Exception {
+    BExpression e = parseRaw(ctx, level, optionalToken);
+    if (e == null) {
+      return null;
+    }
+
+    if (ASSIGN_EXP == e.typ) {
+      // manage assined an injected values
+      BExpression assignedBefore = ctx.lastAssignedExpression.get(e.variableIdx);
+      if (assignedBefore != null && assignedBefore.doNotChange) {
+        e.op1 = assignedBefore; // was injected as key-value
+        e.op1.doNotChange = false; // protect just once, can be changed in second assignement
+      }
+      ctx.lastAssignedExpression.set(e.variableIdx, e.op1);
+    }
+    else if (!ctx.skipConstantExpressionOptimizations) {
+      // try to simplify the expression
+      if (VARIABLE_EXP == e.typ) {
+        BExpression ae = ctx.lastAssignedExpression.get(e.variableIdx);
+        if (ae != null && ae.typ == NUMBER_EXP) {
+          e = ae;
+        }
+      } else {
+        BExpression eCollapsed = e.tryCollapse();
+        if (e != eCollapsed) {
+          e = eCollapsed; // allow breakspoint..
+        }
+        BExpression eEvaluated = e.tryEvaluateConstant();
+        if (e != eEvaluated) {
+          e = eEvaluated; // allow breakspoint..
+        }
+      }
+    }
+    if (level == 0) {
+      // mark the used lookups after the
+      // expression is collapsed to not mark
+      // lookups as used that appear in the profile
+      // but are de-activated by constant expressions
+      int nodeCount = e.markLookupIdxUsed(ctx);
+      ctx.expressionNodeCount += nodeCount;
+    }
+    return e;
+  }
+
+  private int markLookupIdxUsed(BExpressionContext ctx) {
+    int nodeCount = 1;
+    if (lookupNameIdx >= 0) {
+      ctx.markLookupIdxUsed(lookupNameIdx);
+    }
+    if (op1 != null) {
+      nodeCount += op1.markLookupIdxUsed(ctx);
+    }
+    if (op2 != null) {
+      nodeCount += op2.markLookupIdxUsed(ctx);
+    }
+    if (op3 != null) {
+      nodeCount += op3.markLookupIdxUsed(ctx);
+    }
+    return nodeCount;
+  }
+
+  private static BExpression parseRaw(BExpressionContext ctx, int level, String optionalToken) throws Exception {
+
     boolean brackets = false;
     String operator = ctx.parseToken();
     if (optionalToken != null && optionalToken.equals(operator)) {
@@ -64,6 +127,7 @@ final class BExpression {
 
     BExpression exp = new BExpression();
     int nops = 3;
+
     boolean ifThenElse = false;
 
     if ("switch".equals(operator)) {
@@ -124,7 +188,6 @@ final class BExpression {
             if (exp.lookupNameIdx < 0) {
               throw new IllegalArgumentException("unknown lookup name: " + name);
             }
-            ctx.markLookupIdxUsed(exp.lookupNameIdx);
             StringTokenizer tk = new StringTokenizer(values, "|");
             int nt = tk.countTokens();
             int nt2 = nt == 0 ? 1 : nt;
@@ -245,6 +308,50 @@ final class BExpression {
     }
   }
 
+  // Try to collapse the expression
+  // if logically possible
+  private BExpression tryCollapse() {
+    switch (typ) {
+      case OR_EXP:
+        return NUMBER_EXP == op1.typ ?
+          (op1.numberValue != 0.f ? op1 : op2)
+          : (NUMBER_EXP == op2.typ ?
+            (op2.numberValue != 0.f ? op2 : op1)
+            : this);
+      case AND_EXP:
+        return NUMBER_EXP == op1.typ ?
+          (op1.numberValue == 0.f ? op1 : op2)
+          : (NUMBER_EXP == op2.typ ?
+            (op2.numberValue == 0.f ? op2 : op1)
+            : this);
+      case ADD_EXP:
+        return NUMBER_EXP == op1.typ ?
+          (op1.numberValue == 0.f ? op2 : this)
+          : (NUMBER_EXP == op2.typ ?
+            (op2.numberValue == 0.f ? op1 : this)
+            : this);
+      case SWITCH_EXP:
+        return NUMBER_EXP == op1.typ ?
+          (op1.numberValue == 0.f ? op3 : op2) : this;
+      default:
+        return this;
+    }
+  }
+
+  // Try to evaluate the expression
+  // if all operands are constant
+  private BExpression tryEvaluateConstant() {
+    if (op1 != null && NUMBER_EXP == op1.typ
+      && (op2 == null || NUMBER_EXP == op2.typ)
+      && (op3 == null || NUMBER_EXP == op3.typ)) {
+      BExpression exp = new BExpression();
+      exp.typ = NUMBER_EXP;
+      exp.numberValue = evaluate(null);
+      return exp;
+    }
+    return this;
+  }
+
   private float max(float v1, float v2) {
     return v1 > v2 ? v1 : v2;
   }
@@ -252,4 +359,38 @@ final class BExpression {
   private float min(float v1, float v2) {
     return v1 < v2 ? v1 : v2;
   }
+
+  @Override
+  public String toString() {
+    if (typ == NUMBER_EXP) {
+      return "" + numberValue;
+    }
+    if (typ == VARIABLE_EXP) {
+      return "vidx=" + variableIdx;
+    }
+    StringBuilder sb = new StringBuilder("typ=" + typ + " ops=(");
+    addOp(sb, op1);
+    addOp(sb, op2);
+    addOp(sb, op3);
+    sb.append(')');
+    return sb.toString();
+  }
+
+  private void addOp(StringBuilder sb, BExpression e) {
+    if (e != null) {
+      sb.append('[').append(e.toString()).append(']');
+    }
+  }
+
+  static BExpression createAssignExpressionFromKeyValue(BExpressionContext ctx, String key, String value) {
+    BExpression e = new BExpression();
+    e.typ = ASSIGN_EXP;
+    e.variableIdx = ctx.getVariableIdx(key, true);
+    e.op1 = new BExpression();
+    e.op1.typ = NUMBER_EXP;
+    e.op1.numberValue = Float.parseFloat(value);
+    e.op1.doNotChange = true;
+    ctx.lastAssignedExpression.set(e.variableIdx, e.op1);
+    return e;
+  }
 }

brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java

@@ -52,6 +52,10 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
 
   private Map<String, Integer> variableNumbers = new HashMap<>();
 
+  List<BExpression> lastAssignedExpression = new ArrayList<>();
+  boolean skipConstantExpressionOptimizations = false;
+  int expressionNodeCount;
+
   private float[] variableData;
 
 
@@ -770,6 +774,10 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
   }
 
   public void parseFile(File file, String readOnlyContext) {
+    parseFile(file, readOnlyContext, null);
+  }
+
+  public void parseFile(File file, String readOnlyContext, Map<String, String> keyValues) {
     if (!file.exists()) {
       throw new IllegalArgumentException("profile " + file + " does not exist");
     }
@@ -778,15 +786,15 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
         linenr = 1;
         String realContext = context;
         context = readOnlyContext;
-        expressionList = _parseFile(file);
+        expressionList = _parseFile(file, keyValues);
         variableData = new float[variableNumbers.size()];
         evaluate(lookupData); // lookupData is dummy here - evaluate just to create the variables
         context = realContext;
       }
       linenr = 1;
       minWriteIdx = variableData == null ? 0 : variableData.length;
-
+      expressionList = _parseFile(file, null);
-      expressionList = _parseFile(file);
+      lastAssignedExpression = null;
 
       // determine the build-in variable indices
       String[] varNames = getBuildInVariableNames();
@@ -812,10 +820,19 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
     }
   }
 
-  private List<BExpression> _parseFile(File file) throws Exception {
+  private List<BExpression> _parseFile(File file, Map<String, String> keyValues) throws Exception {
     _br = new BufferedReader(new FileReader(file));
     _readerDone = false;
     List<BExpression> result = new ArrayList<>();
+
+    // if injected keyValues are present, create assign expressions for them
+    if (keyValues != null) {
+      for (String key : keyValues.keySet()) {
+        String value = keyValues.get(key);
+        result.add(BExpression.createAssignExpressionFromKeyValue(this, key, value));
+      }
+    }
+
     for (; ; ) {
       BExpression exp = BExpression.parse(this, 0);
       if (exp == null) break;
@@ -857,6 +874,7 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
       if (create) {
         num = variableNumbers.size();
         variableNumbers.put(name, num);
+        lastAssignedExpression.add(null);
       } else {
         return -1;
       }
@@ -896,6 +914,19 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
     }
   }
 
+  public String usedTagList() {
+    StringBuilder sb = new StringBuilder();
+    for (int inum = 0; inum < lookupValues.size(); inum++) {
+      if (lookupIdxUsed[inum]) {
+        if (sb.length() > 0) {
+          sb.append(',');
+        }
+        sb.append(lookupNames.get(inum));
+      }
+    }
+    return sb.toString();
+  }
+
   int getLookupValueIdx(int nameIdx, String value) {
     BExpressionLookupValue[] values = lookupValues.get(nameIdx);
     for (int i = 0; i < values.length; i++) {

brouter-expressions/src/main/java/btools/expressions/ProfileComparator.java

@@ -25,10 +25,20 @@ public final class ProfileComparator {
     BExpressionMetaData meta2 = new BExpressionMetaData();
     BExpressionContext expctx1 = nodeContext ? new BExpressionContextNode(meta1) : new BExpressionContextWay(meta1);
     BExpressionContext expctx2 = nodeContext ? new BExpressionContextNode(meta2) : new BExpressionContextWay(meta2);
+
+    // if same profiles, compare different optimization levels
+    if (profile1File.getName().equals(profile2File.getName())) {
+      expctx2.skipConstantExpressionOptimizations = true;
+    }
+
     meta1.readMetaData(lookupFile);
     meta2.readMetaData(lookupFile);
     expctx1.parseFile(profile1File, "global");
+    System.out.println("usedTags1=" +  expctx1.usedTagList());
     expctx2.parseFile(profile2File, "global");
+    System.out.println("usedTags2=" +  expctx2.usedTagList());
+
+    System.out.println("nodeContext=" + nodeContext + " nodeCount1=" + expctx1.expressionNodeCount + " nodeCount2=" + expctx2.expressionNodeCount);
 
     Random rnd = new Random();
     for (int i = 0; i < nsamples; i++) {

brouter-expressions/src/test/java/btools/expressions/ConstantOptimizerTest.java

@@ -0,0 +1,50 @@
+package btools.expressions;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.Random;
+import java.util.Map;
+import java.util.HashMap;
+
+public class ConstantOptimizerTest {
+  @Test
+  public void compareOptimizerModesTest() {
+
+    File lookupFile = new File(getClass().getResource("/lookups_test.dat").getPath());
+    File profileFile = new File(getClass().getResource("/profile_test.brf").getPath());
+
+    BExpressionMetaData meta1 = new BExpressionMetaData();
+    BExpressionMetaData meta2 = new BExpressionMetaData();
+    BExpressionContext expctx1 = new BExpressionContextWay(meta1);
+    BExpressionContext expctx2 = new BExpressionContextWay(meta2);
+    expctx2.skipConstantExpressionOptimizations = true;
+
+    Map<String, String> keyValue = new HashMap<>();
+    keyValue.put("global_inject1", "5");
+    keyValue.put("global_inject2", "6");
+    keyValue.put("global_inject3", "7");
+
+    meta1.readMetaData(lookupFile);
+    meta2.readMetaData(lookupFile);
+    expctx1.parseFile(profileFile, "global", keyValue);
+    expctx2.parseFile(profileFile, "global", keyValue);
+
+    float d = 0.0001f;
+    Assert.assertEquals(5f, expctx1.getVariableValue("global_inject1", 0f), d);
+    Assert.assertEquals(9f, expctx1.getVariableValue("global_inject2", 0f), d); // should be modified in 2. assign!
+    Assert.assertEquals(7f, expctx1.getVariableValue("global_inject3", 0f), d);
+    Assert.assertEquals(3f, expctx1.getVariableValue("global_inject4", 3f), d); // un-assigned
+
+    Assert.assertTrue("expected far less exporessions nodes if optimized",  expctx2.expressionNodeCount - expctx1.expressionNodeCount >= 311-144);
+
+    Random rnd = new Random(17464); // fixed seed for unit test...
+    for (int i = 0; i < 10000; i++) {
+      int[] data = expctx1.generateRandomValues(rnd);
+      expctx1.evaluate(data);
+      expctx2.evaluate(data);
+      expctx1.assertAllVariablesEqual(expctx2);
+    }
+  }
+}

brouter-expressions/src/test/resources/profile_test.brf

@@ -0,0 +1,88 @@
+---context:global   # following code refers to global config
+
+assign   global_false = false
+assign   global_true = true
+assign   global_and = and false global_true
+
+assign   global_inject1 = 5
+assign   global_inject2 = 13
+assign   global_inject2 = add global_inject2 3
+
+assign global_or = or ( or global_true global_false ) ( or global_false global_true )
+assign global_and = and ( and global_true true ) false
+
+---context:way   # following code refers to way-tags
+
+assign v = highway=primary
+assign w = surface=asphalt
+
+# test constant or/and
+assign costfactor =
+  add multiply 1 or 1 1
+  add multiply 2 or 1 0
+  add multiply 4 or 0 1
+  add multiply 8 or 0 0
+  add multiply 16 and 1 1
+  add multiply 32 and 1 1
+  add multiply 64 and 1 1
+      multiply 128 and 1 1
+  
+# test variable or
+assign turncost =
+  add multiply 1 or v 1
+  add multiply 2 or v 0
+  add multiply 4 or 0 v
+  add multiply 8 or 1 v
+      multiply 16 or v w
+
+# test variable and
+assign uphillcostfactor =
+  add multiply 1 and v 1
+  add multiply 2 and v 0
+  add multiply 4 and 0 v
+  add multiply 8 and 1 v
+      multiply 16 and v w
+
+# test add
+assign downhillcostfactor =
+  add multiply 1 add 1 1
+  add multiply 2 add 1 0
+  add multiply 4 add 0 1
+  add multiply 8 add 0 0
+  add multiply 16 add v 1
+  add multiply 32 add v 0
+  add multiply 64 add 1 v
+      multiply 128 add 0 v
+
+# test max
+assign initialcost =
+  add multiply 1 max 1 2
+  add multiply 2 max multiply 2 v 1
+  add multiply 4 max 1 multiply 2 v
+      multiply 8 max multiply 2 v v
+
+# test switch
+assign initialclassifier =
+  add multiply 1 switch 1 1 0
+  add multiply 2 switch 0 1 0
+  add multiply 4 switch 1 0 1
+  add multiply 8 switch 0 0 1
+  add multiply 16 switch v 1 1
+  add multiply 32 switch v 0 1
+  add multiply 64 switch v 1 0
+  add multiply 128 switch v 0 1
+      multiply 256 switch 1 v w
+
+# test global calcs
+assign priorityclassifier =
+  add multiply 1 global_false
+  add multiply 2 global_true
+  add multiply 4 global_and
+  add multiply 8 global_inject1
+  add multiply 16 global_inject2
+  add multiply 32 global_or
+      multiply 64 global_and
+
+---context:node  # following code refers to node tags
+
+assign initialcost = 1