Not working under parallel
NanFengCheong opened this issue · comments
I am trying to speed up the execution for large number of dataset using parallel.
However, I encounter some list/dictionary concurrent add/update issue.
using System;
using System.Collections.Concurrent;
using System.Dynamic;
using System.Linq;
using System.Text.Json;
using net.adamec.lib.common.dmn.engine.engine.decisions;
using net.adamec.lib.common.dmn.engine.engine.runtime;
using net.adamec.lib.common.dmn.engine.parser;
public class Program
{
public static void Main()
{
string DmnXmlString = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><definitions xmlns=\"https://www.omg.org/spec/DMN/20191111/MODEL/\" xmlns:dmndi=\"https://www.omg.org/spec/DMN/20191111/DMNDI/\" xmlns:dc=\"http://www.omg.org/spec/DMN/20180521/DC/\" xmlns:di=\"http://www.omg.org/spec/DMN/20180521/DI/\" id=\"dish\" name=\"Dish\" namespace=\"http://camunda.org/schema/1.0/dmn\" exporter=\"dmn-js (https://demo.bpmn.io/dmn)\" exporterVersion=\"10.1.0\"> <inputData id=\"dayType_id\" name=\"TypeOfDay\"> <variable id=\"dayType_ii\" name=\"Type of day\" typeRef=\"string\" /> </inputData> <inputData id=\"temperature_id\" name=\"WeatherInCelsius\"> <variable id=\"temperature_ii\" name=\"Weather in Celsius\" typeRef=\"integer\" /> </inputData> <knowledgeSource id=\"host_ks\" name=\"Host\" /> <knowledgeSource id=\"guest_ks\" name=\"Guest Type\"> <authorityRequirement id=\"AuthorityRequirement_0vkhray\"> <requiredDecision href=\"#guestCount\" /> </authorityRequirement> </knowledgeSource> <businessKnowledgeModel id=\"elMenu\" name=\"El menú\" /> <decision id=\"dish-decision\" name=\"Dish Decision\"> <informationRequirement id=\"InformationRequirement_1d56kg6\"> <requiredDecision href=\"#guestCount\" /> </informationRequirement> <informationRequirement id=\"InformationRequirement_11onl5b\"> <requiredDecision href=\"#season\" /> </informationRequirement> <authorityRequirement id=\"AuthorityRequirement_142y75e\"> <requiredAuthority href=\"#host_ks\" /> </authorityRequirement> <decisionTable id=\"dishDecisionTable\"> <input id=\"seasonInput\" label=\"Season\"> <inputExpression id=\"seasonInputExpression\" typeRef=\"string\"> <text>season</text> </inputExpression> </input> <input id=\"guestCountInput\" label=\"GuestCount\"> <inputExpression id=\"guestCountInputExpression\" typeRef=\"integer\"> <text></text> </inputExpression> </input> <output id=\"output1\" label=\"Dish\" name=\"desiredDish\" typeRef=\"string\" /> <rule id=\"row-495762709-1\"> <inputEntry id=\"UnaryTests_1nxcsjr\"> <text>\"Winter\"</text> </inputEntry> <inputEntry id=\"UnaryTests_1r9yorj\"> <text><= 8</text> </inputEntry> <outputEntry id=\"LiteralExpression_1mtwzqz\"> <text>\"Spareribs\"</text> </outputEntry> </rule> <rule id=\"row-495762709-2\"> <inputEntry id=\"UnaryTests_1lxjbif\"> <text>\"Winter\"</text> </inputEntry> <inputEntry id=\"UnaryTests_0nhiedb\"> <text>> 8</text> </inputEntry> <outputEntry id=\"LiteralExpression_1h30r12\"> <text>\"Pasta\"</text> </outputEntry> </rule> <rule id=\"row-495762709-3\"> <inputEntry id=\"UnaryTests_0ifgmfm\"> <text>\"Summer\"</text> </inputEntry> <inputEntry id=\"UnaryTests_12cib9m\"> <text>> 10</text> </inputEntry> <outputEntry id=\"LiteralExpression_0wgaegy\"> <text>\"Light salad\"</text> </outputEntry> </rule> <rule id=\"row-495762709-7\"> <inputEntry id=\"UnaryTests_0ozm9s7\"> <text>\"Summer\"</text> </inputEntry> <inputEntry id=\"UnaryTests_0sesgov\"> <text><= 10</text> </inputEntry> <outputEntry id=\"LiteralExpression_1dvc5x3\"> <text>\"Beans salad\"</text> </outputEntry> </rule> <rule id=\"row-445981423-3\"> <inputEntry id=\"UnaryTests_1er0je1\"> <text>\"Spring\"</text> </inputEntry> <inputEntry id=\"UnaryTests_1uzqner\"> <text>< 10</text> </inputEntry> <outputEntry id=\"LiteralExpression_1pxy4g1\"> <text>\"Stew\"</text> </outputEntry> </rule> <rule id=\"row-445981423-4\"> <inputEntry id=\"UnaryTests_06or48g\"> <text>\"Spring\"</text> </inputEntry> <inputEntry id=\"UnaryTests_0wa71sy\"> <text>>= 10</text> </inputEntry> <outputEntry id=\"LiteralExpression_09ggol9\"> <text>\"Steak\"</text> </outputEntry> </rule> </decisionTable> </decision> <decision id=\"season\" name=\"Season decision\"> <informationRequirement id=\"InformationRequirement_1sdwefx\"> <requiredInput href=\"#temperature_id\" /> </informationRequirement> <decisionTable id=\"seasonDecisionTable\"> <input id=\"temperatureInput\" label=\"WeatherInCelsius\"> <inputExpression id=\"temperatureInputExpression\" typeRef=\"integer\"> <text></text> </inputExpression> </input> <output id=\"seasonOutput\" label=\"season\" name=\"season\" typeRef=\"string\" /> <rule id=\"row-495762709-5\"> <inputEntry id=\"UnaryTests_1fd0eqo\"> <text>>30</text> </inputEntry> <outputEntry id=\"LiteralExpression_0l98klb\"> <text>\"Summer\"</text> </outputEntry> </rule> <rule id=\"row-495762709-6\"> <inputEntry id=\"UnaryTests_1nz6at2\"> <text><10</text> </inputEntry> <outputEntry id=\"LiteralExpression_08moy1k\"> <text>\"Winter\"</text> </outputEntry> </rule> <rule id=\"row-445981423-2\"> <inputEntry id=\"UnaryTests_1a0imxy\"> <text>[10..30]</text> </inputEntry> <outputEntry id=\"LiteralExpression_1poftw4\"> <text>\"Spring\"</text> </outputEntry> </rule> </decisionTable> </decision> <decision id=\"guestCount\" name=\"Guest Count\"> <informationRequirement id=\"InformationRequirement_0j60f3j\"> <requiredInput href=\"#dayType_id\" /> </informationRequirement> <knowledgeRequirement id=\"KnowledgeRequirement_0n56cqb\"> <requiredKnowledge href=\"#elMenu\" /> </knowledgeRequirement> <decisionTable id=\"guestCountDecisionTable\"> <input id=\"typeOfDayInput\" label=\"TypeOfDay\"> <inputExpression id=\"typeOfDayInputExpression\" typeRef=\"string\"> <text></text> </inputExpression> </input> <output id=\"guestCountOutput\" label=\"GuestCount\" typeRef=\"integer\" /> <rule id=\"row-495762709-8\"> <inputEntry id=\"UnaryTests_0l72u8n\"> <text>\"Weekday\"</text> </inputEntry> <outputEntry id=\"LiteralExpression_0wuwqaz\"> <text>4</text> </outputEntry> </rule> <rule id=\"row-495762709-9\"> <inputEntry id=\"UnaryTests_03a73o9\"> <text>\"Holiday\"</text> </inputEntry> <outputEntry id=\"LiteralExpression_1whn119\"> <text>10</text> </outputEntry> </rule> <rule id=\"row-495762709-10\"> <inputEntry id=\"UnaryTests_12tygwt\"> <text>\"Weekend\"</text> </inputEntry> <outputEntry id=\"LiteralExpression_1b5k9t8\"> <text>15</text> </outputEntry> </rule> </decisionTable> </decision> <textAnnotation id=\"TextAnnotation_1\"> <text>Week day or week end</text> </textAnnotation> <association id=\"Association_18hoj4i\"> <sourceRef href=\"#dayType_id\" /> <targetRef href=\"#TextAnnotation_1\" /> </association> <dmndi:DMNDI> <dmndi:DMNDiagram id=\"DMNDiagram_05sfxgt\"> <dmndi:DMNShape id=\"DMNShape_1nkrqp5\" dmnElementRef=\"dayType_id\"> <dc:Bounds height=\"45\" width=\"125\" x=\"417\" y=\"377\" /> </dmndi:DMNShape> <dmndi:DMNShape id=\"DMNShape_0wgwr3t\" dmnElementRef=\"temperature_id\"> <dc:Bounds height=\"45\" width=\"125\" x=\"188\" y=\"377\" /> </dmndi:DMNShape> <dmndi:DMNShape id=\"DMNShape_17n98pm\" dmnElementRef=\"host_ks\"> <dc:Bounds height=\"63\" width=\"100\" x=\"646\" y=\"48\" /> </dmndi:DMNShape> <dmndi:DMNShape id=\"DMNShape_1i9incu\" dmnElementRef=\"guest_ks\"> <dc:Bounds height=\"63\" width=\"100\" x=\"660\" y=\"198\" /> </dmndi:DMNShape> <dmndi:DMNEdge id=\"DMNEdge_0tdfvdg\" dmnElementRef=\"AuthorityRequirement_0vkhray\"> <di:waypoint x=\"570\" y=\"245\" /> <di:waypoint x=\"660\" y=\"235\" /> </dmndi:DMNEdge> <dmndi:DMNShape id=\"DMNShape_1uo50vq\" dmnElementRef=\"elMenu\"> <dc:Bounds height=\"46\" width=\"135\" x=\"642\" y=\"307\" /> </dmndi:DMNShape> <dmndi:DMNShape id=\"DMNShape_0s7a8pk\" dmnElementRef=\"dish-decision\"> <dc:Bounds height=\"80\" width=\"180\" x=\"301\" y=\"48\" /> </dmndi:DMNShape> <dmndi:DMNEdge id=\"DMNEdge_1cvfntf\" dmnElementRef=\"InformationRequirement_1d56kg6\"> <di:waypoint x=\"480\" y=\"210\" /> <di:waypoint x=\"421\" y=\"148\" /> <di:waypoint x=\"421\" y=\"128\" /> </dmndi:DMNEdge> <dmndi:DMNEdge id=\"DMNEdge_0djoiii\" dmnElementRef=\"InformationRequirement_11onl5b\"> <di:waypoint x=\"251\" y=\"210\" /> <di:waypoint x=\"361\" y=\"148\" /> <di:waypoint x=\"361\" y=\"128\" /> </dmndi:DMNEdge> <dmndi:DMNEdge id=\"DMNEdge_0qqxexx\" dmnElementRef=\"AuthorityRequirement_142y75e\"> <di:waypoint x=\"646\" y=\"81\" /> <di:waypoint x=\"481\" y=\"86\" /> </dmndi:DMNEdge> <dmndi:DMNShape id=\"DMNShape_06z5z89\" dmnElementRef=\"season\"> <dc:Bounds height=\"80\" width=\"180\" x=\"161\" y=\"210\" /> </dmndi:DMNShape> <dmndi:DMNEdge id=\"DMNEdge_1383eyj\" dmnElementRef=\"InformationRequirement_1sdwefx\"> <di:waypoint x=\"251\" y=\"377\" /> <di:waypoint x=\"251\" y=\"310\" /> <di:waypoint x=\"251\" y=\"290\" /> </dmndi:DMNEdge> <dmndi:DMNShape id=\"DMNShape_0qbhe8q\" dmnElementRef=\"guestCount\"> <dc:Bounds height=\"80\" width=\"180\" x=\"390\" y=\"210\" /> </dmndi:DMNShape> <dmndi:DMNEdge id=\"DMNEdge_131oa1j\" dmnElementRef=\"KnowledgeRequirement_0n56cqb\"> <di:waypoint x=\"691\" y=\"307\" /> <di:waypoint x=\"570\" y=\"262\" /> </dmndi:DMNEdge> <dmndi:DMNEdge id=\"DMNEdge_1avtdb1\" dmnElementRef=\"InformationRequirement_0j60f3j\"> <di:waypoint x=\"480\" y=\"377\" /> <di:waypoint x=\"480\" y=\"310\" /> <di:waypoint x=\"480\" y=\"290\" /> </dmndi:DMNEdge> <dmndi:DMNShape id=\"DMNShape_0bblyhb\" dmnElementRef=\"TextAnnotation_1\"> <dc:Bounds height=\"45\" width=\"125\" x=\"328\" y=\"477\" /> </dmndi:DMNShape> <dmndi:DMNEdge id=\"DMNEdge_0aqnkob\" dmnElementRef=\"Association_18hoj4i\"> <di:waypoint x=\"480\" y=\"422\" /> <di:waypoint x=\"391\" y=\"477\" /> </dmndi:DMNEdge> </dmndi:DMNDiagram> </dmndi:DMNDI></definitions>";
string jsonString = "[{\"TypeOfDay\":\"Holiday\",\"WeatherInCelsius\":24},{\"TypeOfDay\":\"Weekend\",\"WeatherInCelsius\":4}]";
ConcurrentBag<DmnDecisionResult> results = new ConcurrentBag<DmnDecisionResult>();
System.Text.Json.JsonDocument.Parse(jsonString).RootElement
.EnumerateArray()
.AsParallel()
.ForAll(param =>
{
var def = DmnParser.ParseString13(DmnXmlString);
var ctx = DmnExecutionContextFactory.CreateExecutionContext(def);
foreach (var property in System.Text.Json.JsonDocument.Parse(param.GetRawText()).RootElement.EnumerateObject())
{
dynamic propertyvalue = getPropertyValue(property.Value);
ctx.WithInputParameter(property.Name.Trim().Replace(' ', '_'), propertyvalue);
}
var result = ctx.ExecuteDecision("Dish Decision");
results.Add(result);
});
Console.WriteLine(JsonSerializer.Serialize(results.ToList()));
}
private static dynamic getPropertyValue(JsonElement element)
{
switch (element.ValueKind)
{
case JsonValueKind.Null:
return null;
case JsonValueKind.Number:
if (element.GetRawText().Contains("."))
return element.GetDouble();
else
return element.GetInt32();
case JsonValueKind.False:
return false;
case JsonValueKind.True:
return true;
case JsonValueKind.Undefined:
return null;
case JsonValueKind.String:
return element.GetString();
case JsonValueKind.Object:
return Newtonsoft.Json.JsonConvert.DeserializeObject<ExpandoObject>(element.GetRawText(), new Newtonsoft.Json.Converters.ExpandoObjectConverter());
//case JsonValueKind.Array:
// result = srcData.EnumerateArray()
// .Select(o => new ReflectionDynamicObject { RealObject = o })
// .ToArray();
// break;
default:
return null;
}
}
}
System.InvalidOperationException: 'Operations that change non-concurrent collections must have exclusive access. A concurrent update was performed on this collection and corrupted its state. The collection's state is no longer correct.'
Hello @NanFengCheong ,
well, the DMN Engine was not designed for parallel processing, although it's interesting idea.
The issue is in DmnExecutionContext class. It uses static Dictionary as cache for pre processed expressions:
private static readonly Dictionary<(string, Type), Lambda> ParsedExpressionsCache = new Dictionary<(string, Type), Lambda>();
That's, I guess, where you're getting the concurrent access exception (while updating the cache during processing).
I did a quick test just by changing the Dictionary to ConcurrentDictionary and it seems to be working (and it also passes all tests):
private static readonly ConcurrentDictionary<(string, Type), Lambda> ParsedExpressionsCache = new ConcurrentDictionary<(string, Type), Lambda>();
So, you can try this "hack", however, as mentioned above, it was not designed this way, so there might be some other consequences (none came to my mind now, but...).
However, there is a better way how to solve (or workaround) your issue I think:
I built a bigger dataset in test code you provided to get a more demanding load:
string jsonString = "{\"TypeOfDay\":\"Holiday\",\"WeatherInCelsius\":24},{\"TypeOfDay\":\"Weekend\",\"WeatherInCelsius\":4}";
for (var i = 0; i < 15; i++)
{
jsonString = $"{jsonString},{jsonString}";
}
jsonString = $"[{jsonString}]";
XML parsing and creation of model definition (part of the DmnExecutionContextFactory.CreateExecutionContext overload you use) are quite compute heavy operations, that can be run only once and just the "pure" context is needed for each parallel item. So it will be better to do it this way:
var model = DmnParser.ParseString13(DmnXmlString);
var definition = DmnDefinitionFactory.CreateDmnDefinition(model);
System.Text.Json.JsonDocument.Parse(jsonString).RootElement
.EnumerateArray()
.AsParallel()
.ForAll(param =>
{
var ctx = DmnExecutionContextFactory.CreateExecutionContext(definition);
foreach (var property in System.Text.Json.JsonDocument.Parse(param.GetRawText()).RootElement.EnumerateObject())
{
dynamic propertyvalue = getPropertyValue(property.Value);
ctx.WithInputParameter(property.Name.Trim().Replace(' ', '_'), propertyvalue);
}
var result = ctx.ExecuteDecision("Dish Decision");
results.Add(result);
});
Even with your simple model and execution of 65536 decisions in parallel (the "bigger dataset" mentioned above), the processing time has been decreased from about 24s to 14s at my machine.
This will also bring the possibility to get rid of the issue with the parallel access to the expressions cache (so no need to modify the DmnExecutionContext class). Just build the cache before running the parallel block, so there will be no further (parallel) updates.
I believe, if your code will look like this,
var model = DmnParser.ParseString13(DmnXmlString);
var definition = DmnDefinitionFactory.CreateDmnDefinition(model);
var ctxTmpBuildCache = DmnExecutionContextFactory.CreateExecutionContext(definition);
var resultDummy = ctxTmpBuildCache.ExecuteDecision("Dish Decision");
System.Text.Json.JsonDocument.Parse(jsonString).RootElement
.EnumerateArray()
.AsParallel()
.ForAll(param =>
{
var ctx = DmnExecutionContextFactory.CreateExecutionContext(definition);
foreach (var property in System.Text.Json.JsonDocument.Parse(param.GetRawText()).RootElement.EnumerateObject())
{
dynamic propertyvalue = getPropertyValue(property.Value);
ctx.WithInputParameter(property.Name.Trim().Replace(' ', '_'), propertyvalue);
}
var result = ctx.ExecuteDecision("Dish Decision");
results.Add(result);
});
it will not only run faster, but it will also solve the original issue with concurrent update of cache
Hope this help
Radek
Hi @adamecr ,
Appreciate for the quick reply.
I managed to get it working.
Thanks.
I added another cache on top of the parallel.
Seems working fine 90% of the time.
It would fail intermittently.
public List<DmnDecisionResult> ExecuteDmnUsingArrayParamParallel(string dmnXmlString, string decisionName, string jsonArrayParam)
{
ConcurrentDictionary<string, DmnDecisionResult> cache = new();
ConcurrentBag<DmnDecisionResult> results = new();
var model = DmnParser.ParseString13(dmnXmlString);
var definition = DmnDefinitionFactory.CreateDmnDefinition(model);
_ = DmnExecutionContextFactory.CreateExecutionContext(definition).ExecuteDecision(decisionName);
JsonDocument.Parse(jsonArrayParam).RootElement
.EnumerateArray()
.AsParallel()
.ForAll(param =>
{
var hash = param.GetRawText();
if (cache.ContainsKey(hash))
{
results.Add(cache[hash]);
return;
}
var ctx = DmnExecutionContextFactory.CreateExecutionContext(definition);
foreach (var property in JsonDocument.Parse(param.GetRawText()).RootElement.EnumerateObject())
{
dynamic propertyvalue = getPropertyValue(property.Value);
ctx.WithInputParameter(property.Name.Trim().Replace(' ', '_'), propertyvalue);
}
var result = ctx.ExecuteDecision(decisionName);
cache.TryAdd(hash, result);
results.Add(result);
});
return results.ToList();
}