Fabio/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request #573 from abrensch/constant_exp_opti

Browse source 
optimizing constant expressions in profile parsing
This commit is contained in:
abrensch 2023-07-06 13:05:13 +02:00 committed by GitHub
commit 79aa07ae84
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 328 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
brouter-core/src/main/java/btools/router/ProfileCache.java
View file

@ -91,8 +91,8 @@ public final class ProfileCache {
meta.readMetaData(new File(profileDir, "lookups.dat"));
rc.expctxWay.parseFile(profileFile, "global");
rc.expctxNode.parseFile(profileFile, "global");
rc.expctxWay.parseFile(profileFile, "global", rc.keyValues);
rc.expctxNode.parseFile(profileFile, "global", rc.keyValues);
rc.readGlobalConfig();

10
brouter-core/src/main/java/btools/router/RoutingContext.java
View file

@ -108,16 +108,6 @@ public final class RoutingContext {
public void readGlobalConfig() {
BExpressionContext expctxGlobal = expctxWay; // just one of them...
if (keyValues != null) {
// add parameter to context
for (Map.Entry<String, String> e : keyValues.entrySet()) {
float f = Float.parseFloat(e.getValue());
expctxWay.setVariableValue(e.getKey(), f, true);
expctxNode.setVariableValue(e.getKey(), f, true);
}
}
setModel(expctxGlobal._modelClass);
carMode = 0.f != expctxGlobal.getVariableValue("validForCars", 0.f);

145
brouter-expressions/src/main/java/btools/expressions/BExpression.java
View file

@ -32,8 +32,9 @@ final class BExpression {
private BExpression op3;
private float numberValue;
private int variableIdx;
private int lookupNameIdx;
private int lookupNameIdx = -1;
private int[] lookupValueIdxArray;
private boolean doNotChange;
// Parse the expression and all subexpression
public static BExpression parse(BExpressionContext ctx, int level) throws Exception {
@ -41,6 +42,68 @@ final class BExpression {
}
private static BExpression parse(BExpressionContext ctx, int level, String optionalToken) throws Exception {
BExpression e = parseRaw(ctx, level, optionalToken);
if (e == null) {
return null;
}
if (ASSIGN_EXP == e.typ) {
// manage assined an injected values
BExpression assignedBefore = ctx.lastAssignedExpression.get(e.variableIdx);
if (assignedBefore != null && assignedBefore.doNotChange) {
e.op1 = assignedBefore; // was injected as key-value
e.op1.doNotChange = false; // protect just once, can be changed in second assignement
}
ctx.lastAssignedExpression.set(e.variableIdx, e.op1);
}
else if (!ctx.skipConstantExpressionOptimizations) {
// try to simplify the expression
if (VARIABLE_EXP == e.typ) {
BExpression ae = ctx.lastAssignedExpression.get(e.variableIdx);
if (ae != null && ae.typ == NUMBER_EXP) {
e = ae;
}
} else {
BExpression eCollapsed = e.tryCollapse();
if (e != eCollapsed) {
e = eCollapsed; // allow breakspoint..
}
BExpression eEvaluated = e.tryEvaluateConstant();
if (e != eEvaluated) {
e = eEvaluated; // allow breakspoint..
}
}
}
if (level == 0) {
// mark the used lookups after the
// expression is collapsed to not mark
// lookups as used that appear in the profile
// but are de-activated by constant expressions
int nodeCount = e.markLookupIdxUsed(ctx);
ctx.expressionNodeCount += nodeCount;
}
return e;
}
private int markLookupIdxUsed(BExpressionContext ctx) {
int nodeCount = 1;
if (lookupNameIdx >= 0) {
ctx.markLookupIdxUsed(lookupNameIdx);
}
if (op1 != null) {
nodeCount += op1.markLookupIdxUsed(ctx);
}
if (op2 != null) {
nodeCount += op2.markLookupIdxUsed(ctx);
}
if (op3 != null) {
nodeCount += op3.markLookupIdxUsed(ctx);
}
return nodeCount;
}
private static BExpression parseRaw(BExpressionContext ctx, int level, String optionalToken) throws Exception {
boolean brackets = false;
String operator = ctx.parseToken();
if (optionalToken != null && optionalToken.equals(operator)) {
@ -64,6 +127,7 @@ final class BExpression {
BExpression exp = new BExpression();
int nops = 3;
boolean ifThenElse = false;
if ("switch".equals(operator)) {
@ -124,7 +188,6 @@ final class BExpression {
if (exp.lookupNameIdx < 0) {
throw new IllegalArgumentException("unknown lookup name: " + name);
}
ctx.markLookupIdxUsed(exp.lookupNameIdx);
StringTokenizer tk = new StringTokenizer(values, "|");
int nt = tk.countTokens();
int nt2 = nt == 0 ? 1 : nt;
@ -245,6 +308,50 @@ final class BExpression {
}
}
// Try to collapse the expression
// if logically possible
private BExpression tryCollapse() {
switch (typ) {
case OR_EXP:
return NUMBER_EXP == op1.typ ?
(op1.numberValue != 0.f ? op1 : op2)
: (NUMBER_EXP == op2.typ ?
(op2.numberValue != 0.f ? op2 : op1)
: this);
case AND_EXP:
return NUMBER_EXP == op1.typ ?
(op1.numberValue == 0.f ? op1 : op2)
: (NUMBER_EXP == op2.typ ?
(op2.numberValue == 0.f ? op2 : op1)
: this);
case ADD_EXP:
return NUMBER_EXP == op1.typ ?
(op1.numberValue == 0.f ? op2 : this)
: (NUMBER_EXP == op2.typ ?
(op2.numberValue == 0.f ? op1 : this)
: this);
case SWITCH_EXP:
return NUMBER_EXP == op1.typ ?
(op1.numberValue == 0.f ? op3 : op2) : this;
default:
return this;
}
}
// Try to evaluate the expression
// if all operands are constant
private BExpression tryEvaluateConstant() {
if (op1 != null && NUMBER_EXP == op1.typ
&& (op2 == null || NUMBER_EXP == op2.typ)
&& (op3 == null || NUMBER_EXP == op3.typ)) {
BExpression exp = new BExpression();
exp.typ = NUMBER_EXP;
exp.numberValue = evaluate(null);
return exp;
}
return this;
}
private float max(float v1, float v2) {
return v1 > v2 ? v1 : v2;
}
@ -252,4 +359,38 @@ final class BExpression {
private float min(float v1, float v2) {
return v1 < v2 ? v1 : v2;
}
@Override
public String toString() {
if (typ == NUMBER_EXP) {
return "" + numberValue;
}
if (typ == VARIABLE_EXP) {
return "vidx=" + variableIdx;
}
StringBuilder sb = new StringBuilder("typ=" + typ + " ops=(");
addOp(sb, op1);
addOp(sb, op2);
addOp(sb, op3);
sb.append(')');
return sb.toString();
}
private void addOp(StringBuilder sb, BExpression e) {
if (e != null) {
sb.append('[').append(e.toString()).append(']');
}
}
static BExpression createAssignExpressionFromKeyValue(BExpressionContext ctx, String key, String value) {
BExpression e = new BExpression();
e.typ = ASSIGN_EXP;
e.variableIdx = ctx.getVariableIdx(key, true);
e.op1 = new BExpression();
e.op1.typ = NUMBER_EXP;
e.op1.numberValue = Float.parseFloat(value);
e.op1.doNotChange = true;
ctx.lastAssignedExpression.set(e.variableIdx, e.op1);
return e;
}
}

39
brouter-expressions/src/main/java/btools/expressions/BExpressionContext.java
View file

@ -52,6 +52,10 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
private Map<String, Integer> variableNumbers = new HashMap<>();
List<BExpression> lastAssignedExpression = new ArrayList<>();
boolean skipConstantExpressionOptimizations = false;
int expressionNodeCount;
private float[] variableData;
@ -770,6 +774,10 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
}
public void parseFile(File file, String readOnlyContext) {
parseFile(file, readOnlyContext, null);
}
public void parseFile(File file, String readOnlyContext, Map<String, String> keyValues) {
if (!file.exists()) {
throw new IllegalArgumentException("profile " + file + " does not exist");
}
@ -778,15 +786,15 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
linenr = 1;
String realContext = context;
context = readOnlyContext;
expressionList = _parseFile(file);
expressionList = _parseFile(file, keyValues);
variableData = new float[variableNumbers.size()];
evaluate(lookupData); // lookupData is dummy here - evaluate just to create the variables
context = realContext;
}
linenr = 1;
minWriteIdx = variableData == null ? 0 : variableData.length;
expressionList = _parseFile(file);
expressionList = _parseFile(file, null);
lastAssignedExpression = null;
// determine the build-in variable indices
String[] varNames = getBuildInVariableNames();
@ -812,10 +820,19 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
}
}
private List<BExpression> _parseFile(File file) throws Exception {
private List<BExpression> _parseFile(File file, Map<String, String> keyValues) throws Exception {
_br = new BufferedReader(new FileReader(file));
_readerDone = false;
List<BExpression> result = new ArrayList<>();
// if injected keyValues are present, create assign expressions for them
if (keyValues != null) {
for (String key : keyValues.keySet()) {
String value = keyValues.get(key);
result.add(BExpression.createAssignExpressionFromKeyValue(this, key, value));
}
}
for (; ; ) {
BExpression exp = BExpression.parse(this, 0);
if (exp == null) break;
@ -857,6 +874,7 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
if (create) {
num = variableNumbers.size();
variableNumbers.put(name, num);
lastAssignedExpression.add(null);
} else {
return -1;
}
@ -896,6 +914,19 @@ public abstract class BExpressionContext implements IByteArrayUnifier {
}
}
public String usedTagList() {
StringBuilder sb = new StringBuilder();
for (int inum = 0; inum < lookupValues.size(); inum++) {
if (lookupIdxUsed[inum]) {
if (sb.length() > 0) {
sb.append(',');
}
sb.append(lookupNames.get(inum));
}
}
return sb.toString();
}
int getLookupValueIdx(int nameIdx, String value) {
BExpressionLookupValue[] values = lookupValues.get(nameIdx);
for (int i = 0; i < values.length; i++) {

10
brouter-expressions/src/main/java/btools/expressions/ProfileComparator.java
View file

@ -25,10 +25,20 @@ public final class ProfileComparator {
BExpressionMetaData meta2 = new BExpressionMetaData();
BExpressionContext expctx1 = nodeContext ? new BExpressionContextNode(meta1) : new BExpressionContextWay(meta1);
BExpressionContext expctx2 = nodeContext ? new BExpressionContextNode(meta2) : new BExpressionContextWay(meta2);
// if same profiles, compare different optimization levels
if (profile1File.getName().equals(profile2File.getName())) {
expctx2.skipConstantExpressionOptimizations = true;
}
meta1.readMetaData(lookupFile);
meta2.readMetaData(lookupFile);
expctx1.parseFile(profile1File, "global");
System.out.println("usedTags1=" + expctx1.usedTagList());
expctx2.parseFile(profile2File, "global");
System.out.println("usedTags2=" + expctx2.usedTagList());
System.out.println("nodeContext=" + nodeContext + " nodeCount1=" + expctx1.expressionNodeCount + " nodeCount2=" + expctx2.expressionNodeCount);
Random rnd = new Random();
for (int i = 0; i < nsamples; i++) {

50
brouter-expressions/src/test/java/btools/expressions/ConstantOptimizerTest.java Normal file
View file

@ -0,0 +1,50 @@
package btools.expressions;
import org.junit.Assert;
import org.junit.Test;
import java.io.File;
import java.util.Random;
import java.util.Map;
import java.util.HashMap;
public class ConstantOptimizerTest {
@Test
public void compareOptimizerModesTest() {
File lookupFile = new File(getClass().getResource("/lookups_test.dat").getPath());
File profileFile = new File(getClass().getResource("/profile_test.brf").getPath());
BExpressionMetaData meta1 = new BExpressionMetaData();
BExpressionMetaData meta2 = new BExpressionMetaData();
BExpressionContext expctx1 = new BExpressionContextWay(meta1);
BExpressionContext expctx2 = new BExpressionContextWay(meta2);
expctx2.skipConstantExpressionOptimizations = true;
Map<String, String> keyValue = new HashMap<>();
keyValue.put("global_inject1", "5");
keyValue.put("global_inject2", "6");
keyValue.put("global_inject3", "7");
meta1.readMetaData(lookupFile);
meta2.readMetaData(lookupFile);
expctx1.parseFile(profileFile, "global", keyValue);
expctx2.parseFile(profileFile, "global", keyValue);
float d = 0.0001f;
Assert.assertEquals(5f, expctx1.getVariableValue("global_inject1", 0f), d);
Assert.assertEquals(9f, expctx1.getVariableValue("global_inject2", 0f), d); // should be modified in 2. assign!
Assert.assertEquals(7f, expctx1.getVariableValue("global_inject3", 0f), d);
Assert.assertEquals(3f, expctx1.getVariableValue("global_inject4", 3f), d); // un-assigned
Assert.assertTrue("expected far less exporessions nodes if optimized", expctx2.expressionNodeCount - expctx1.expressionNodeCount >= 311-144);
Random rnd = new Random(17464); // fixed seed for unit test...
for (int i = 0; i < 10000; i++) {
int[] data = expctx1.generateRandomValues(rnd);
expctx1.evaluate(data);
expctx2.evaluate(data);
expctx1.assertAllVariablesEqual(expctx2);
}
}
}

88
brouter-expressions/src/test/resources/profile_test.brf Normal file
View file

@ -0,0 +1,88 @@
---context:global # following code refers to global config
assign global_false = false
assign global_true = true
assign global_and = and false global_true
assign global_inject1 = 5
assign global_inject2 = 13
assign global_inject2 = add global_inject2 3
assign global_or = or ( or global_true global_false ) ( or global_false global_true )
assign global_and = and ( and global_true true ) false
---context:way # following code refers to way-tags
assign v = highway=primary
assign w = surface=asphalt
# test constant or/and
assign costfactor =
add multiply 1 or 1 1
add multiply 2 or 1 0
add multiply 4 or 0 1
add multiply 8 or 0 0
add multiply 16 and 1 1
add multiply 32 and 1 1
add multiply 64 and 1 1
multiply 128 and 1 1
# test variable or
assign turncost =
add multiply 1 or v 1
add multiply 2 or v 0
add multiply 4 or 0 v
add multiply 8 or 1 v
multiply 16 or v w
# test variable and
assign uphillcostfactor =
add multiply 1 and v 1
add multiply 2 and v 0
add multiply 4 and 0 v
add multiply 8 and 1 v
multiply 16 and v w
# test add
assign downhillcostfactor =
add multiply 1 add 1 1
add multiply 2 add 1 0
add multiply 4 add 0 1
add multiply 8 add 0 0
add multiply 16 add v 1
add multiply 32 add v 0
add multiply 64 add 1 v
multiply 128 add 0 v
# test max
assign initialcost =
add multiply 1 max 1 2
add multiply 2 max multiply 2 v 1
add multiply 4 max 1 multiply 2 v
multiply 8 max multiply 2 v v
# test switch
assign initialclassifier =
add multiply 1 switch 1 1 0
add multiply 2 switch 0 1 0
add multiply 4 switch 1 0 1
add multiply 8 switch 0 0 1
add multiply 16 switch v 1 1
add multiply 32 switch v 0 1
add multiply 64 switch v 1 0
add multiply 128 switch v 0 1
multiply 256 switch 1 v w
# test global calcs
assign priorityclassifier =
add multiply 1 global_false
add multiply 2 global_true
add multiply 4 global_and
add multiply 8 global_inject1
add multiply 16 global_inject2
add multiply 32 global_or
multiply 64 global_and
---context:node # following code refers to node tags
assign initialcost = 1