constant expressions: reworks keyvalue-injection, unit-test

brouter-expressions/src/main/java/btools/expressions/BExpression.java

@@ -34,6 +34,7 @@ final class BExpression {
   private int variableIdx;
   private int lookupNameIdx = -1;
   private int[] lookupValueIdxArray;
+  private boolean doNotChange;
 
   // Parse the expression and all subexpression
   public static BExpression parse(BExpressionContext ctx, int level) throws Exception {
@@ -46,41 +47,59 @@ final class BExpression {
       return null;
     }
 
-    // try to simplify the expression
     if (ASSIGN_EXP == e.typ) {
-      ctx.lastAssignedExpression.set(e.variableIdx, e.op1);
-    } else if (VARIABLE_EXP == e.typ) {
-      BExpression ae = ctx.lastAssignedExpression.get(e.variableIdx);
-      if (ae != null && ae.typ == NUMBER_EXP) {
-        e = ae;
+      // manage assined an injected values
+      BExpression assignedBefore = ctx.lastAssignedExpression.get(e.variableIdx);
+      if (assignedBefore != null && assignedBefore.doNotChange) {
+        e.op1 = assignedBefore; // was injected as key-value
+        e.op1.doNotChange = false; // protect just once, can be changed in second assignement
+      }
+      ctx.lastAssignedExpression.set(e.variableIdx, e.op1);
+    }
+    else if (!ctx.skipConstantExpressionOptimizations) {
+      // try to simplify the expression
+      if (VARIABLE_EXP == e.typ) {
+        BExpression ae = ctx.lastAssignedExpression.get(e.variableIdx);
+        if (ae != null && ae.typ == NUMBER_EXP) {
+          e = ae;
+        }
+      } else {
+        BExpression eCollapsed = e.tryCollapse();
+        if (e != eCollapsed) {
+          e = eCollapsed; // allow breakspoint..
+        }
+        BExpression eEvaluated = e.tryEvaluateConstant();
+        if (e != eEvaluated) {
+          e = eEvaluated; // allow breakspoint..
+        }
       }
-    } else {
-      e = e.tryCollapse();
-      e = e.tryEvaluateConstant();
     }
     if (level == 0) {
       // mark the used lookups after the
       // expression is collapsed to not mark
       // lookups as used that appear in the profile
       // but are de-activated by constant expressions
-      e.markLookupIdxUsed(ctx);
+      int nodeCount = e.markLookupIdxUsed(ctx);
+      ctx.expressionNodeCount += nodeCount;
     }
     return e;
   }
 
-  private void markLookupIdxUsed(BExpressionContext ctx) {
+  private int markLookupIdxUsed(BExpressionContext ctx) {
+    int nodeCount = 1;
     if (lookupNameIdx >= 0) {
       ctx.markLookupIdxUsed(lookupNameIdx);
     }
     if (op1 != null) {
-      op1.markLookupIdxUsed(ctx);
+      nodeCount += op1.markLookupIdxUsed(ctx);
     }
     if (op2 != null) {
-      op2.markLookupIdxUsed(ctx);
+      nodeCount += op2.markLookupIdxUsed(ctx);
     }
     if (op3 != null) {
-      op3.markLookupIdxUsed(ctx);
+      nodeCount += op3.markLookupIdxUsed(ctx);
     }
+    return nodeCount;
   }
 
   private static BExpression parseRaw(BExpressionContext ctx, int level, String optionalToken) throws Exception {
@@ -108,7 +127,6 @@ final class BExpression {
 
     BExpression exp = new BExpression();
     int nops = 3;
-    BExpression op1Replacement = null;
 
     boolean ifThenElse = false;
 
@@ -156,16 +174,6 @@ final class BExpression {
           exp.variableIdx = ctx.getVariableIdx(variable, true);
           if (exp.variableIdx < ctx.getMinWriteIdx())
             throw new IllegalArgumentException("cannot assign to readonly variable " + variable);
-
-          // possibly replace the value to assign by an injected value
-          if (ctx.keyValues != null) {
-            String v = ctx.keyValues.get(variable);
-            if (v != null) {
-              op1Replacement = new BExpression();
-              op1Replacement.typ = NUMBER_EXP;
-              op1Replacement.numberValue = Float.parseFloat(v);
-            }
-          }
         } else if ("not".equals(operator)) {
           exp.typ = NOT_EXP;
         } else {
@@ -232,9 +240,6 @@ final class BExpression {
     // parse operands
     if (nops > 0) {
       exp.op1 = BExpression.parse(ctx, level + 1, exp.typ == ASSIGN_EXP ? "=" : null);
-      if (op1Replacement != null) {
-        exp.op1 = op1Replacement;
-      }
     }
     if (nops > 1) {
       if (ifThenElse) checkExpectedToken(ctx, "then");
@@ -305,7 +310,7 @@ final class BExpression {
 
   // Try to collapse the expression
   // if logically possible
-  public BExpression tryCollapse() {
+  private BExpression tryCollapse() {
     switch (typ) {
       case OR_EXP:
         return NUMBER_EXP == op1.typ ?
@@ -335,7 +340,7 @@ final class BExpression {
 
   // Try to evaluate the expression
   // if all operands are constant
-  public BExpression tryEvaluateConstant() {
+  private BExpression tryEvaluateConstant() {
     if (op1 != null && NUMBER_EXP == op1.typ
       && (op2 == null || NUMBER_EXP == op2.typ)
       && (op3 == null || NUMBER_EXP == op3.typ)) {
@@ -354,4 +359,38 @@ final class BExpression {
   private float min(float v1, float v2) {
     return v1 < v2 ? v1 : v2;
   }
+
+  @Override
+  public String toString() {
+    if (typ == NUMBER_EXP) {
+      return "" + numberValue;
+    }
+    if (typ == VARIABLE_EXP) {
+      return "vidx=" + variableIdx;
+    }
+    StringBuilder sb = new StringBuilder("typ=" + typ + " ops=(");
+    addOp(sb, op1);
+    addOp(sb, op2);
+    addOp(sb, op3);
+    sb.append(')');
+    return sb.toString();
+  }
+
+  private void addOp(StringBuilder sb, BExpression e) {
+    if (e != null) {
+      sb.append('[').append(e.toString()).append(']');
+    }
+  }
+
+  static BExpression createAssignExpressionFromKeyValue(BExpressionContext ctx, String key, String value) {
+    BExpression e = new BExpression();
+    e.typ = ASSIGN_EXP;
+    e.variableIdx = ctx.getVariableIdx(key, true);
+    e.op1 = new BExpression();
+    e.op1.typ = NUMBER_EXP;
+    e.op1.numberValue = Float.parseFloat(value);
+    e.op1.doNotChange = true;
+    ctx.lastAssignedExpression.set(e.variableIdx, e.op1);
+    return e;
+  }
 }

brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java

@@ -53,7 +53,8 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
   private Map<String, Integer> variableNumbers = new HashMap<>();
 
   List<BExpression> lastAssignedExpression = new ArrayList<>();
-  Map<String, String> keyValues;
+  boolean skipConstantExpressionOptimizations = false;
+  int expressionNodeCount;
 
   private float[] variableData;
 
@@ -771,14 +772,12 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
     }
     return foreignContext.getOutputVariableIndex(name, true);
   }
-  
-  public void parseFile(File file, String readOnlyContext, Map<String, String> keyValues) {
-    this.keyValues = keyValues;
-    parseFile(file, readOnlyContext);
-    this.keyValues = null;
-  }
 
   public void parseFile(File file, String readOnlyContext) {
+    parseFile(file, readOnlyContext, null);
+  }
+
+  public void parseFile(File file, String readOnlyContext, Map<String, String> keyValues) {
     if (!file.exists()) {
       throw new IllegalArgumentException("profile " + file + " does not exist");
     }
@@ -787,14 +786,14 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
         linenr = 1;
         String realContext = context;
         context = readOnlyContext;
-        expressionList = _parseFile(file);
+        expressionList = _parseFile(file, keyValues);
         variableData = new float[variableNumbers.size()];
         evaluate(lookupData); // lookupData is dummy here - evaluate just to create the variables
         context = realContext;
       }
       linenr = 1;
       minWriteIdx = variableData == null ? 0 : variableData.length;
-      expressionList = _parseFile(file);
+      expressionList = _parseFile(file, null);
       lastAssignedExpression = null;
 
       // determine the build-in variable indices
@@ -821,10 +820,19 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
     }
   }
 
-  private List<BExpression> _parseFile(File file) throws Exception {
+  private List<BExpression> _parseFile(File file, Map<String, String> keyValues) throws Exception {
     _br = new BufferedReader(new FileReader(file));
     _readerDone = false;
     List<BExpression> result = new ArrayList<>();
+
+    // if injected keyValues are present, create assign expressions for them
+    if (keyValues != null) {
+      for (String key : keyValues.keySet()) {
+        String value = keyValues.get(key);
+        result.add(BExpression.createAssignExpressionFromKeyValue(this, key, value));
+      }
+    }
+
     for (; ; ) {
       BExpression exp = BExpression.parse(this, 0);
       if (exp == null) break;
@@ -906,6 +914,19 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
     }
   }
 
+  public String usedTagList() {
+    StringBuilder sb = new StringBuilder();
+    for (int inum = 0; inum < lookupValues.size(); inum++) {
+      if (lookupIdxUsed[inum]) {
+        if (sb.length() > 0) {
+          sb.append(',');
+        }
+        sb.append(lookupNames.get(inum));
+      }
+    }
+    return sb.toString();
+  }
+
   int getLookupValueIdx(int nameIdx, String value) {
     BExpressionLookupValue[] values = lookupValues.get(nameIdx);
     for (int i = 0; i < values.length; i++) {

brouter-expressions/src/main/java/btools/expressions/ProfileComparator.java

@@ -25,10 +25,20 @@ public final class ProfileComparator {
     BExpressionMetaData meta2 = new BExpressionMetaData();
     BExpressionContext expctx1 = nodeContext ? new BExpressionContextNode(meta1) : new BExpressionContextWay(meta1);
     BExpressionContext expctx2 = nodeContext ? new BExpressionContextNode(meta2) : new BExpressionContextWay(meta2);
+
+    // if same profiles, compare different optimization levels
+    if (profile1File.getName().equals(profile2File.getName())) {
+      expctx2.skipConstantExpressionOptimizations = true;
+    }
+
     meta1.readMetaData(lookupFile);
     meta2.readMetaData(lookupFile);
     expctx1.parseFile(profile1File, "global");
+    System.out.println("usedTags1=" +  expctx1.usedTagList());
     expctx2.parseFile(profile2File, "global");
+    System.out.println("usedTags2=" +  expctx2.usedTagList());
+
+    System.out.println("nodeContext=" + nodeContext + " nodeCount1=" + expctx1.expressionNodeCount + " nodeCount2=" + expctx2.expressionNodeCount);
 
     Random rnd = new Random();
     for (int i = 0; i < nsamples; i++) {

brouter-expressions/src/test/java/btools/expressions/ConstantOptimizerTest.java

@@ -0,0 +1,50 @@
+package btools.expressions;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.Random;
+import java.util.Map;
+import java.util.HashMap;
+
+public class ConstantOptimizerTest {
+  @Test
+  public void compareOptimizerModesTest() {
+
+    File lookupFile = new File(getClass().getResource("/lookups_test.dat").getPath());
+    File profileFile = new File(getClass().getResource("/profile_test.brf").getPath());
+
+    BExpressionMetaData meta1 = new BExpressionMetaData();
+    BExpressionMetaData meta2 = new BExpressionMetaData();
+    BExpressionContext expctx1 = new BExpressionContextWay(meta1);
+    BExpressionContext expctx2 = new BExpressionContextWay(meta2);
+    expctx2.skipConstantExpressionOptimizations = true;
+
+    Map<String, String> keyValue = new HashMap<>();
+    keyValue.put("global_inject1", "5");
+    keyValue.put("global_inject2", "6");
+    keyValue.put("global_inject3", "7");
+
+    meta1.readMetaData(lookupFile);
+    meta2.readMetaData(lookupFile);
+    expctx1.parseFile(profileFile, "global", keyValue);
+    expctx2.parseFile(profileFile, "global", keyValue);
+
+    float d = 0.0001f;
+    Assert.assertEquals(5f, expctx1.getVariableValue("global_inject1", 0f), d);
+    Assert.assertEquals(9f, expctx1.getVariableValue("global_inject2", 0f), d); // should be modified in 2. assign!
+    Assert.assertEquals(7f, expctx1.getVariableValue("global_inject3", 0f), d);
+    Assert.assertEquals(3f, expctx1.getVariableValue("global_inject4", 3f), d); // un-assigned
+
+    Assert.assertTrue("expected far less exporessions nodes if optimized",  expctx2.expressionNodeCount - expctx1.expressionNodeCount >= 311-144);
+
+    Random rnd = new Random(17464); // fixed seed for unit test...
+    for (int i = 0; i < 10000; i++) {
+      int[] data = expctx1.generateRandomValues(rnd);
+      expctx1.evaluate(data);
+      expctx2.evaluate(data);
+      expctx1.assertAllVariablesEqual(expctx2);
+    }
+  }
+}

brouter-expressions/src/test/resources/profile_test.brf

@@ -0,0 +1,88 @@
+---context:global   # following code refers to global config
+
+assign   global_false = false
+assign   global_true = true
+assign   global_and = and false global_true
+
+assign   global_inject1 = 5
+assign   global_inject2 = 13
+assign   global_inject2 = add global_inject2 3
+
+assign global_or = or ( or global_true global_false ) ( or global_false global_true )
+assign global_and = and ( and global_true true ) false
+
+---context:way   # following code refers to way-tags
+
+assign v = highway=primary
+assign w = surface=asphalt
+
+# test constant or/and
+assign costfactor =
+  add multiply 1 or 1 1
+  add multiply 2 or 1 0
+  add multiply 4 or 0 1
+  add multiply 8 or 0 0
+  add multiply 16 and 1 1
+  add multiply 32 and 1 1
+  add multiply 64 and 1 1
+      multiply 128 and 1 1
+  
+# test variable or
+assign turncost =
+  add multiply 1 or v 1
+  add multiply 2 or v 0
+  add multiply 4 or 0 v
+  add multiply 8 or 1 v
+      multiply 16 or v w
+
+# test variable and
+assign uphillcostfactor =
+  add multiply 1 and v 1
+  add multiply 2 and v 0
+  add multiply 4 and 0 v
+  add multiply 8 and 1 v
+      multiply 16 and v w
+
+# test add
+assign downhillcostfactor =
+  add multiply 1 add 1 1
+  add multiply 2 add 1 0
+  add multiply 4 add 0 1
+  add multiply 8 add 0 0
+  add multiply 16 add v 1
+  add multiply 32 add v 0
+  add multiply 64 add 1 v
+      multiply 128 add 0 v
+
+# test max
+assign initialcost =
+  add multiply 1 max 1 2
+  add multiply 2 max multiply 2 v 1
+  add multiply 4 max 1 multiply 2 v
+      multiply 8 max multiply 2 v v
+
+# test switch
+assign initialclassifier =
+  add multiply 1 switch 1 1 0
+  add multiply 2 switch 0 1 0
+  add multiply 4 switch 1 0 1
+  add multiply 8 switch 0 0 1
+  add multiply 16 switch v 1 1
+  add multiply 32 switch v 0 1
+  add multiply 64 switch v 1 0
+  add multiply 128 switch v 0 1
+      multiply 256 switch 1 v w
+
+# test global calcs
+assign priorityclassifier =
+  add multiply 1 global_false
+  add multiply 2 global_true
+  add multiply 4 global_and
+  add multiply 8 global_inject1
+  add multiply 16 global_inject2
+  add multiply 32 global_or
+      multiply 64 global_and
+
+---context:node  # following code refers to node tags
+
+assign initialcost = 1