Skip to content

Commit

Permalink
Fix workflow instance restart failed due to duplicate key in varpool (#…
Browse files Browse the repository at this point in the history 
…16001)
  • Loading branch information
@ruanwenjun
ruanwenjun committed May 16, 2024
1 parent 68a1906 commit 61915a2
Show file tree
Hide file tree
Showing 14 changed files with 281 additions and 276 deletions.
2 changes: 2 additions & 0 deletions dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.dolphinscheduler.dao.DaoConfiguration;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.dao.entity.PluginDefine;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider;
import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
Expand Down Expand Up @@ -69,6 +70,7 @@ public void run(ApplicationReadyEvent readyEvent) {
log.info("Received spring application context ready event will load taskPlugin and write to DB");
// install task plugin
TaskPluginManager.loadPlugin();
DataSourceProcessorProvider.initialize();
for (Map.Entry<String, TaskChannelFactory> entry : TaskPluginManager.getTaskChannelFactoryMap().entrySet()) {
String taskPluginName = entry.getKey();
TaskChannelFactory taskChannelFactory = entry.getValue();
Expand Down
13 changes: 9 additions & 4 deletions ...i/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,10 +367,15 @@ public void deleteByWorkflowInstanceId(Integer workflowInstanceId) {
}
for (TaskInstance taskInstance : needToDeleteTaskInstances) {
if (StringUtils.isNotBlank(taskInstance.getLogPath())) {
ILogService iLogService =
SingletonJdkDynamicRpcClientProxyFactory.getProxyClient(taskInstance.getHost(),
ILogService.class);
iLogService.removeTaskInstanceLog(taskInstance.getLogPath());
try {
// Remove task instance log failed will not affect the deletion of task instance
ILogService iLogService =
SingletonJdkDynamicRpcClientProxyFactory.getProxyClient(taskInstance.getHost(),
ILogService.class);
iLogService.removeTaskInstanceLog(taskInstance.getLogPath());
} catch (Exception ex) {
log.error("Remove task instance log error", ex);
}
}
}

Expand Down
4 changes: 4 additions & 0 deletions ...org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ public class DataSourceProcessorProvider {
private DataSourceProcessorProvider() {
}

public static void initialize() {
log.info("Initialize DataSourceProcessorProvider");
}

public static DataSourceProcessor getDataSourceProcessor(@NonNull DbType dbType) {
return dataSourcePluginManager.getDataSourceProcessorMap().get(dbType.name());
}
Expand Down
2 changes: 2 additions & 0 deletions ...cheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.dolphinscheduler.dao.DaoConfiguration;
import org.apache.dolphinscheduler.meter.metrics.MetricsProvider;
import org.apache.dolphinscheduler.meter.metrics.SystemMetrics;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider;
import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.registry.api.RegistryConfiguration;
Expand Down Expand Up @@ -108,6 +109,7 @@ public void run() throws SchedulerException {

// install task plugin
TaskPluginManager.loadPlugin();
DataSourceProcessorProvider.initialize();

this.masterSlotManager.start();

Expand Down
200 changes: 30 additions & 170 deletions ...c/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,10 @@
import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostRequest;
import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostResponse;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.VarPoolUtils;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.event.StateEvent;
import org.apache.dolphinscheduler.server.master.event.StateEventHandleError;
Expand Down Expand Up @@ -97,6 +96,7 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
Expand Down Expand Up @@ -376,7 +376,8 @@ public void taskFinished(TaskInstance taskInstance) throws StateEventHandleExcep

if (taskInstance.getState().isSuccess()) {
completeTaskSet.add(taskInstance.getTaskCode());
mergeTaskInstanceVarPool(taskInstance);
workflowInstance.setVarPool(VarPoolUtils.mergeVarPoolJsonString(
Lists.newArrayList(workflowInstance.getVarPool(), taskInstance.getVarPool())));
processInstanceDao.upsertProcessInstance(workflowInstance);
ProjectUser projectUser =
processService.queryProjectWithUserByProcessInstanceId(workflowInstance.getId());
Expand Down Expand Up @@ -441,7 +442,6 @@ private void releaseTaskGroupIfNeeded(TaskInstance taskInstance) {

/**
* crate new task instance to retry, different objects from the original
*
*/
private void retryTaskInstance(TaskInstance taskInstance) throws StateEventHandleException {
ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance();
Expand Down Expand Up @@ -532,16 +532,6 @@ public void checkTaskInstanceByStateEvent(TaskStateEvent stateEvent) throws Stat
}
}

/**
* check if task instance exist by id
*/
public boolean checkTaskInstanceById(int taskInstanceId) {
if (taskInstanceMap.isEmpty()) {
return false;
}
return taskInstanceMap.containsKey(taskInstanceId);
}

/**
* get task instance from memory
*/
Expand Down Expand Up @@ -1070,7 +1060,7 @@ public TaskInstance cloneTolerantTaskInstance(TaskInstance taskInstance) {
* new a taskInstance
*
* @param processInstance process instance
* @param taskNode task node
* @param taskNode task node
* @return task instance
*/
public TaskInstance newTaskInstance(ProcessInstance processInstance, TaskNode taskNode) {
Expand Down Expand Up @@ -1161,80 +1151,32 @@ public TaskInstance newTaskInstance(ProcessInstance processInstance, TaskNode ta
return taskInstance;
}

public void getPreVarPool(TaskInstance taskInstance, Set<Long> preTask) {
void initializeTaskInstanceVarPool(TaskInstance taskInstance) {
// get pre task ,get all the task varPool to this task
// Do not use dag.getPreviousNodes because of the dag may be miss the upstream node
String preTasks =
workflowExecuteContext.getWorkflowGraph().getTaskNodeByCode(taskInstance.getTaskCode()).getPreTasks();
Set<Long> preTaskList = new HashSet<>(JSONUtils.toList(preTasks, Long.class));
ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance();
Map<String, Property> allProperty = new HashMap<>();
Map<String, TaskInstance> allTaskInstance = new HashMap<>();
if (CollectionUtils.isNotEmpty(preTask)) {
for (Long preTaskCode : preTask) {
Optional<TaskInstance> existTaskInstanceOptional = getTaskInstance(preTaskCode);
if (!existTaskInstanceOptional.isPresent()) {
continue;
}

Integer taskId = existTaskInstanceOptional.get().getId();
if (taskId == null) {
continue;
}
TaskInstance preTaskInstance = taskInstanceMap.get(taskId);
if (preTaskInstance == null) {
continue;
}
String preVarPool = preTaskInstance.getVarPool();
if (StringUtils.isNotEmpty(preVarPool)) {
List<Property> properties = JSONUtils.toList(preVarPool, Property.class);
for (Property info : properties) {
setVarPoolValue(allProperty, allTaskInstance, preTaskInstance, info);
}
}
}
if (allProperty.size() > 0) {
taskInstance.setVarPool(JSONUtils.toJsonString(allProperty.values()));
}
} else {
if (StringUtils.isNotEmpty(workflowInstance.getVarPool())) {
taskInstance.setVarPool(workflowInstance.getVarPool());
}
if (CollectionUtils.isEmpty(preTaskList)) {
taskInstance.setVarPool(workflowInstance.getVarPool());
return;
}
List<String> preTaskInstanceVarPools = preTaskList
.stream()
.map(taskCode -> getTaskInstance(taskCode).orElse(null))
.filter(Objects::nonNull)
.sorted(Comparator.comparing(TaskInstance::getEndTime))
.map(TaskInstance::getVarPool)
.collect(Collectors.toList());
taskInstance.setVarPool(VarPoolUtils.mergeVarPoolJsonString(preTaskInstanceVarPools));
}

public Collection<TaskInstance> getAllTaskInstances() {
return taskInstanceMap.values();
}

private void setVarPoolValue(Map<String, Property> allProperty,
Map<String, TaskInstance> allTaskInstance,
TaskInstance preTaskInstance, Property thisProperty) {
// for this taskInstance all the param in this part is IN.
thisProperty.setDirect(Direct.IN);
// get the pre taskInstance Property's name
String proName = thisProperty.getProp();
// if the Previous nodes have the Property of same name
if (allProperty.containsKey(proName)) {
// comparison the value of two Property
Property otherPro = allProperty.get(proName);
// if this property'value of loop is empty,use the other,whether the other's value is empty or not
if (StringUtils.isEmpty(thisProperty.getValue())) {
allProperty.put(proName, otherPro);
// if property'value of loop is not empty,and the other's value is not empty too, use the earlier value
} else if (StringUtils.isNotEmpty(otherPro.getValue())) {
TaskInstance otherTask = allTaskInstance.get(proName);
if (otherTask.getEndTime().getTime() > preTaskInstance.getEndTime().getTime()) {
allProperty.put(proName, thisProperty);
allTaskInstance.put(proName, preTaskInstance);
} else {
allProperty.put(proName, otherPro);
}
} else {
allProperty.put(proName, thisProperty);
allTaskInstance.put(proName, preTaskInstance);
}
} else {
allProperty.put(proName, thisProperty);
allTaskInstance.put(proName, preTaskInstance);
}
}

/**
* get complete task instance map, taskCode as key
*/
Expand Down Expand Up @@ -1311,45 +1253,10 @@ private void submitPostNode(Long parentNodeCode) throws StateEventHandleExceptio
}
// the end node of the branch of the dag
if (parentNodeCode != null && dag.getEndNode().contains(parentNodeCode)) {
Optional<TaskInstance> existTaskInstanceOptional = getTaskInstance(parentNodeCode);
if (existTaskInstanceOptional.isPresent()) {
TaskInstance endTaskInstance = taskInstanceMap.get(existTaskInstanceOptional.get().getId());
String taskInstanceVarPool = endTaskInstance.getVarPool();
if (StringUtils.isNotEmpty(taskInstanceVarPool)) {
Set<Property> taskProperties = new HashSet<>(JSONUtils.toList(taskInstanceVarPool, Property.class));
String processInstanceVarPool = workflowInstance.getVarPool();
List<Property> processGlobalParams =
new ArrayList<>(JSONUtils.toList(workflowInstance.getGlobalParams(), Property.class));
Map<String, Direct> oldProcessGlobalParamsMap = processGlobalParams.stream()
.collect(Collectors.toMap(Property::getProp, Property::getDirect));
Set<Property> processVarPoolOut = taskProperties.stream()
.filter(property -> property.getDirect().equals(Direct.OUT)
&& oldProcessGlobalParamsMap.containsKey(property.getProp())
&& oldProcessGlobalParamsMap.get(property.getProp()).equals(Direct.OUT))
.collect(Collectors.toSet());
Set<Property> taskVarPoolIn =
taskProperties.stream().filter(property -> property.getDirect().equals(Direct.IN))
.collect(Collectors.toSet());
if (StringUtils.isNotEmpty(processInstanceVarPool)) {
Set<Property> properties =
new HashSet<>(JSONUtils.toList(processInstanceVarPool, Property.class));
Set<String> newProcessVarPoolKeys =
taskProperties.stream().map(Property::getProp).collect(Collectors.toSet());
properties = properties.stream()
.filter(property -> !newProcessVarPoolKeys.contains(property.getProp()))
.collect(Collectors.toSet());
properties.addAll(processVarPoolOut);
properties.addAll(taskVarPoolIn);

workflowInstance.setVarPool(JSONUtils.toJsonString(properties));
} else {
Set<Property> varPool = new HashSet<>();
varPool.addAll(taskVarPoolIn);
varPool.addAll(processVarPoolOut);
workflowInstance.setVarPool(JSONUtils.toJsonString(varPool));
}
}
}
getTaskInstance(parentNodeCode)
.ifPresent(endTaskInstance -> workflowInstance.setVarPool(VarPoolUtils.mergeVarPoolJsonString(
Lists.newArrayList(workflowInstance.getVarPool(), endTaskInstance.getVarPool()))));

}

// if previous node success , post node submit
Expand Down Expand Up @@ -1907,14 +1814,8 @@ public void submitStandByTask() throws StateEventHandleException {
continue;
}
}
// init varPool only this task is the first time running
if (task.isFirstRun()) {
// get pre task ,get all the task varPool to this task
// Do not use dag.getPreviousNodes because of the dag may be miss the upstream node
String preTasks = workflowExecuteContext.getWorkflowGraph()
.getTaskNodeByCode(task.getTaskCode()).getPreTasks();
Set<Long> preTaskList = new HashSet<>(JSONUtils.toList(preTasks, Long.class));
getPreVarPool(task, preTaskList);
initializeTaskInstanceVarPool(task);
}
DependResult dependResult = getDependResultForTask(task);
if (DependResult.SUCCESS == dependResult) {
Expand Down Expand Up @@ -2095,29 +1996,9 @@ protected void clearDataIfExecuteTask() {
taskInstanceDao.updateById(taskInstance);
}

Set<String> removeSet = new HashSet<>();
for (TaskInstance taskInstance : removeTaskInstances) {
String taskVarPool = taskInstance.getVarPool();
if (StringUtils.isNotEmpty(taskVarPool)) {
List<Property> properties = JSONUtils.toList(taskVarPool, Property.class);
List<String> keys = properties.stream()
.filter(property -> property.getDirect().equals(Direct.OUT))
.map(property -> String.format("%s_%s", property.getProp(), property.getType()))
.collect(Collectors.toList());
removeSet.addAll(keys);
}
}

// remove varPool data and update process instance
// TODO: we can remove this snippet if : we get varPool from pre taskInstance instead of process instance when
// task can not get pre task from incomplete dag
List<Property> processProperties = JSONUtils.toList(workflowInstance.getVarPool(), Property.class);
processProperties = processProperties.stream()
.filter(property -> !(property.getDirect().equals(Direct.IN)
&& removeSet.contains(String.format("%s_%s", property.getProp(), property.getType()))))
.collect(Collectors.toList());

workflowInstance.setVarPool(JSONUtils.toJsonString(processProperties));
workflowInstance.setVarPool(
VarPoolUtils.subtractVarPoolJson(workflowInstance.getVarPool(),
removeTaskInstances.stream().map(TaskInstance::getVarPool).collect(Collectors.toList())));
processInstanceDao.updateById(workflowInstance);

// remove task instance from taskInstanceMap,taskCodeInstanceMap , completeTaskSet, validTaskMap, errorTaskMap
Expand Down Expand Up @@ -2154,25 +2035,4 @@ private void sendTaskLogOnMasterToRemoteIfNeeded(TaskInstance taskInstance) {
}
}

private void mergeTaskInstanceVarPool(TaskInstance taskInstance) {
String taskVarPoolJson = taskInstance.getVarPool();
if (StringUtils.isEmpty(taskVarPoolJson)) {
return;
}
ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance();
String processVarPoolJson = workflowInstance.getVarPool();
if (StringUtils.isEmpty(processVarPoolJson)) {
workflowInstance.setVarPool(taskVarPoolJson);
return;
}
List<Property> processVarPool = new ArrayList<>(JSONUtils.toList(processVarPoolJson, Property.class));
List<Property> taskVarPool = JSONUtils.toList(taskVarPoolJson, Property.class);
Set<String> newProcessVarPoolKeys = taskVarPool.stream().map(Property::getProp).collect(Collectors.toSet());
processVarPool = processVarPool.stream().filter(property -> !newProcessVarPoolKeys.contains(property.getProp()))
.collect(Collectors.toList());

processVarPool.addAll(taskVarPool);

workflowInstance.setVarPool(JSONUtils.toJsonString(processVarPool));
}
}

0 comments on commit 61915a2

Please sign in to comment.