From e8bcc6348cd36124f8551a853b343436aa60415a Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Tue, 2 Mar 2021 17:09:12 +0100 Subject: [PATCH 01/46] fix find all busy agents within job --- .../com/flowci/core/job/domain/JobAgent.java | 18 ++++++++++++++---- .../core/job/manager/JobActionManagerImpl.java | 17 +++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/JobAgent.java b/core/src/main/java/com/flowci/core/job/domain/JobAgent.java index f5f4cfd8c..ff6a8ca6f 100644 --- a/core/src/main/java/com/flowci/core/job/domain/JobAgent.java +++ b/core/src/main/java/com/flowci/core/job/domain/JobAgent.java @@ -39,11 +39,21 @@ public Collection all() { return agents.keySet(); } - public Collection allBusyAgents() { - List busy = new LinkedList<>(); + /** + * All busy agents, which are occupied by flow and assigned to step + */ + public Collection allBusyAgents(Collection ongoingSteps) { + Set busy = new HashSet<>(agents.size()); this.agents.forEach((k, v) -> { - if (!v.isEmpty()) { - busy.add(k); + if (v.isEmpty()) { + return; + } + + for(Step s : ongoingSteps) { + if (s.hasAgent() && s.getAgentId().equals(k)) { + busy.add(k); + return; + } } }); return busy; diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index 3e3545d4e..183a8ed88 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -504,7 +504,7 @@ public void accept(JobSmContext context) { @Override public void accept(JobSmContext context) { Job job = context.job; - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); } @Override @@ -522,7 +522,7 @@ public void accept(JobSmContext context) { Job job = context.job; Step step = context.step; stepService.toStatus(step, Step.Status.EXCEPTION, null, false); - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); logInfo(job, "finished with status {}", Failure); } @@ -538,7 +538,7 @@ public void onException(Throwable e, JobSmContext context) { public void accept(JobSmContext context) { Job job = context.job; setJobStatusAndSave(job, Job.Status.CANCELLING, null); - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); } @Override @@ -558,8 +558,9 @@ public void accept(JobSmContext context) { Job job = context.job; JobAgent jobAgent = getJobAgent(job.getId()); - if (jobAgent.allBusyAgents().isEmpty()) { - setOngingStepsToSkipped(job); + List ongoingSteps = stepService.list(job, Executed.OngoingStatus); + if (jobAgent.allBusyAgents(ongoingSteps).isEmpty()) { + setOngoingStepsToSkipped(job); return; } @@ -569,7 +570,7 @@ public void accept(JobSmContext context) { @Override public void onException(Throwable e, JobSmContext context) { Job job = context.job; - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); setJobStatusAndSave(job, Job.Status.CANCELLED, e.getMessage()); } @@ -590,7 +591,7 @@ public boolean canRun(JobSmContext context) { @Override public void accept(JobSmContext context) { Job job = context.job; - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); setJobStatusAndSave(job, Job.Status.CANCELLED, null); } @@ -775,7 +776,7 @@ private void setupJobYamlAndSteps(Job job, String yml) { job.getContext().merge(root.getEnvironments(), false); } - private void setOngingStepsToSkipped(Job job) { + private void setOngoingStepsToSkipped(Job job) { List steps = stepService.list(job, Executed.OngoingStatus); for (Step step : steps) { if (!step.hasAgent()) { From 689d27e3956c460234ae395c8e8f88f7cbcad01b Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Tue, 2 Mar 2021 17:09:12 +0100 Subject: [PATCH 02/46] fix find all busy agents within job --- .../com/flowci/core/job/domain/JobAgent.java | 18 ++++++++++++++---- .../core/job/manager/JobActionManagerImpl.java | 17 +++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/JobAgent.java b/core/src/main/java/com/flowci/core/job/domain/JobAgent.java index f5f4cfd8c..ff6a8ca6f 100644 --- a/core/src/main/java/com/flowci/core/job/domain/JobAgent.java +++ b/core/src/main/java/com/flowci/core/job/domain/JobAgent.java @@ -39,11 +39,21 @@ public Collection all() { return agents.keySet(); } - public Collection allBusyAgents() { - List busy = new LinkedList<>(); + /** + * All busy agents, which are occupied by flow and assigned to step + */ + public Collection allBusyAgents(Collection ongoingSteps) { + Set busy = new HashSet<>(agents.size()); this.agents.forEach((k, v) -> { - if (!v.isEmpty()) { - busy.add(k); + if (v.isEmpty()) { + return; + } + + for(Step s : ongoingSteps) { + if (s.hasAgent() && s.getAgentId().equals(k)) { + busy.add(k); + return; + } } }); return busy; diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index 3547bbe8d..893ba04da 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -504,7 +504,7 @@ public void accept(JobSmContext context) { @Override public void accept(JobSmContext context) { Job job = context.job; - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); } @Override @@ -522,7 +522,7 @@ public void accept(JobSmContext context) { Job job = context.job; Step step = context.step; stepService.toStatus(step, Step.Status.EXCEPTION, null, false); - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); logInfo(job, "finished with status {}", Failure); } @@ -538,7 +538,7 @@ public void onException(Throwable e, JobSmContext context) { public void accept(JobSmContext context) { Job job = context.job; setJobStatusAndSave(job, Job.Status.CANCELLING, null); - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); } @Override @@ -558,8 +558,9 @@ public void accept(JobSmContext context) { Job job = context.job; JobAgent jobAgent = getJobAgent(job.getId()); - if (jobAgent.allBusyAgents().isEmpty()) { - setOngingStepsToSkipped(job); + List ongoingSteps = stepService.list(job, Executed.OngoingStatus); + if (jobAgent.allBusyAgents(ongoingSteps).isEmpty()) { + setOngoingStepsToSkipped(job); return; } @@ -569,7 +570,7 @@ public void accept(JobSmContext context) { @Override public void onException(Throwable e, JobSmContext context) { Job job = context.job; - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); setJobStatusAndSave(job, Job.Status.CANCELLED, e.getMessage()); } @@ -590,7 +591,7 @@ public boolean canRun(JobSmContext context) { @Override public void accept(JobSmContext context) { Job job = context.job; - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); setJobStatusAndSave(job, Job.Status.CANCELLED, null); } @@ -775,7 +776,7 @@ private void setupJobYamlAndSteps(Job job, String yml) { job.getContext().merge(root.getEnvironments(), false); } - private void setOngingStepsToSkipped(Job job) { + private void setOngoingStepsToSkipped(Job job) { List steps = stepService.list(job, Executed.OngoingStatus); for (Step step : steps) { if (!step.hasAgent()) { From e2f78426300b1c064d50f1790864d12f855385ab Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Tue, 2 Mar 2021 17:09:12 +0100 Subject: [PATCH 03/46] fix find all busy agents within job --- .../com/flowci/core/job/domain/JobAgent.java | 18 ++++++++++++++---- .../core/job/manager/JobActionManagerImpl.java | 17 +++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/JobAgent.java b/core/src/main/java/com/flowci/core/job/domain/JobAgent.java index f5f4cfd8c..ff6a8ca6f 100644 --- a/core/src/main/java/com/flowci/core/job/domain/JobAgent.java +++ b/core/src/main/java/com/flowci/core/job/domain/JobAgent.java @@ -39,11 +39,21 @@ public Collection all() { return agents.keySet(); } - public Collection allBusyAgents() { - List busy = new LinkedList<>(); + /** + * All busy agents, which are occupied by flow and assigned to step + */ + public Collection allBusyAgents(Collection ongoingSteps) { + Set busy = new HashSet<>(agents.size()); this.agents.forEach((k, v) -> { - if (!v.isEmpty()) { - busy.add(k); + if (v.isEmpty()) { + return; + } + + for(Step s : ongoingSteps) { + if (s.hasAgent() && s.getAgentId().equals(k)) { + busy.add(k); + return; + } } }); return busy; diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index 3547bbe8d..893ba04da 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -504,7 +504,7 @@ public void accept(JobSmContext context) { @Override public void accept(JobSmContext context) { Job job = context.job; - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); } @Override @@ -522,7 +522,7 @@ public void accept(JobSmContext context) { Job job = context.job; Step step = context.step; stepService.toStatus(step, Step.Status.EXCEPTION, null, false); - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); logInfo(job, "finished with status {}", Failure); } @@ -538,7 +538,7 @@ public void onException(Throwable e, JobSmContext context) { public void accept(JobSmContext context) { Job job = context.job; setJobStatusAndSave(job, Job.Status.CANCELLING, null); - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); } @Override @@ -558,8 +558,9 @@ public void accept(JobSmContext context) { Job job = context.job; JobAgent jobAgent = getJobAgent(job.getId()); - if (jobAgent.allBusyAgents().isEmpty()) { - setOngingStepsToSkipped(job); + List ongoingSteps = stepService.list(job, Executed.OngoingStatus); + if (jobAgent.allBusyAgents(ongoingSteps).isEmpty()) { + setOngoingStepsToSkipped(job); return; } @@ -569,7 +570,7 @@ public void accept(JobSmContext context) { @Override public void onException(Throwable e, JobSmContext context) { Job job = context.job; - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); setJobStatusAndSave(job, Job.Status.CANCELLED, e.getMessage()); } @@ -590,7 +591,7 @@ public boolean canRun(JobSmContext context) { @Override public void accept(JobSmContext context) { Job job = context.job; - setOngingStepsToSkipped(job); + setOngoingStepsToSkipped(job); setJobStatusAndSave(job, Job.Status.CANCELLED, null); } @@ -775,7 +776,7 @@ private void setupJobYamlAndSteps(Job job, String yml) { job.getContext().merge(root.getEnvironments(), false); } - private void setOngingStepsToSkipped(Job job) { + private void setOngoingStepsToSkipped(Job job) { List steps = stepService.list(job, Executed.OngoingStatus); for (Step step : steps) { if (!step.hasAgent()) { From b7b2022cc6c8a0c40f071ccda9c7dce05ccdd058 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sat, 6 Mar 2021 18:07:40 +0100 Subject: [PATCH 04/46] save agent profile --- .../core/agent/dao/AgentProfileDao.java | 9 +++++++ .../com/flowci/core/agent/domain/Agent.java | 18 ------------- .../flowci/core/agent/domain/AgentInit.java | 2 -- .../core/agent/domain/AgentProfile.java | 27 +++++++++++++++++++ .../core/agent/event/OnAgentProfileEvent.java | 15 +++++++++++ .../core/agent/manager/AgentEventManager.java | 18 +++++++++++++ .../core/agent/service/AgentService.java | 5 ---- .../core/agent/service/AgentServiceImpl.java | 18 ++++++------- .../flowci/core/api/OpenRestController.java | 8 ------ 9 files changed, 78 insertions(+), 42 deletions(-) create mode 100644 core/src/main/java/com/flowci/core/agent/dao/AgentProfileDao.java create mode 100644 core/src/main/java/com/flowci/core/agent/domain/AgentProfile.java create mode 100644 core/src/main/java/com/flowci/core/agent/event/OnAgentProfileEvent.java diff --git a/core/src/main/java/com/flowci/core/agent/dao/AgentProfileDao.java b/core/src/main/java/com/flowci/core/agent/dao/AgentProfileDao.java new file mode 100644 index 000000000..93b23710b --- /dev/null +++ b/core/src/main/java/com/flowci/core/agent/dao/AgentProfileDao.java @@ -0,0 +1,9 @@ +package com.flowci.core.agent.dao; + +import com.flowci.core.agent.domain.AgentProfile; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface AgentProfileDao extends MongoRepository { +} diff --git a/core/src/main/java/com/flowci/core/agent/domain/Agent.java b/core/src/main/java/com/flowci/core/agent/domain/Agent.java index f83bc15fb..3205a38c6 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/Agent.java +++ b/core/src/main/java/com/flowci/core/agent/domain/Agent.java @@ -64,22 +64,6 @@ public static Status fromBytes(byte[] bytes) { } } - @Getter - @Setter - @Accessors(chain = true) - public static class Resource { - - private int cpu; - - private int totalMemory; // in MB - - private int freeMemory; // in MB - - private int totalDisk; // in MB - - private int freeDisk; // in MB - } - @Indexed(name = "index_agent_name", unique = true) private String name; @@ -97,8 +81,6 @@ public static class Resource { private OS os = OS.UNKNOWN; - private Resource resource = new Resource(); - private Set tags = Collections.emptySet(); private Status status = Status.OFFLINE; diff --git a/core/src/main/java/com/flowci/core/agent/domain/AgentInit.java b/core/src/main/java/com/flowci/core/agent/domain/AgentInit.java index a4fbafa31..2ded1fcb3 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/AgentInit.java +++ b/core/src/main/java/com/flowci/core/agent/domain/AgentInit.java @@ -38,6 +38,4 @@ public class AgentInit { private Common.OS os; private Agent.Status status; - - private Agent.Resource resource = new Agent.Resource(); } diff --git a/core/src/main/java/com/flowci/core/agent/domain/AgentProfile.java b/core/src/main/java/com/flowci/core/agent/domain/AgentProfile.java new file mode 100644 index 000000000..6edcee42a --- /dev/null +++ b/core/src/main/java/com/flowci/core/agent/domain/AgentProfile.java @@ -0,0 +1,27 @@ +package com.flowci.core.agent.domain; + +import com.flowci.core.common.domain.Mongoable; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + +/** + * Id is agent token + */ +@Getter +@Setter +@Accessors(chain = true) +public class AgentProfile extends Mongoable { + + private int cpuNum; + + private double cpuUsage; + + private int totalMemory; // in MB + + private int freeMemory; // in MB + + private int totalDisk; // in MB + + private int freeDisk; // in MB +} diff --git a/core/src/main/java/com/flowci/core/agent/event/OnAgentProfileEvent.java b/core/src/main/java/com/flowci/core/agent/event/OnAgentProfileEvent.java new file mode 100644 index 000000000..1618d7b50 --- /dev/null +++ b/core/src/main/java/com/flowci/core/agent/event/OnAgentProfileEvent.java @@ -0,0 +1,15 @@ +package com.flowci.core.agent.event; + +import com.flowci.core.agent.domain.AgentProfile; +import lombok.Getter; + +@Getter +public class OnAgentProfileEvent extends EventFromClient { + + private final AgentProfile profile; + + public OnAgentProfileEvent(Object source, AgentProfile profile) { + super(source, null, null); + this.profile = profile; + } +} diff --git a/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java b/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java index 2b137f1ee..18112f99c 100644 --- a/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java +++ b/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flowci.core.agent.domain.AgentInit; +import com.flowci.core.agent.domain.AgentProfile; import com.flowci.core.agent.domain.ShellLog; import com.flowci.core.agent.domain.TtyCmd; import com.flowci.core.agent.event.*; @@ -33,6 +34,8 @@ public class AgentEventManager extends BinaryWebSocketHandler { private final static String EventConnect = "connect___"; + private final static String EventProfile = "profile___"; + private final static String EventCmdOut = "cmd_out___"; private final static String EventShellLog = "slog______"; @@ -104,6 +107,11 @@ protected void handleBinaryMessage(WebSocketSession session, BinaryMessage messa if (EventTTYLog.equals(event)) { onTtyLog(body); + return; + } + + if (EventProfile.equals(event)) { + onProfile(token, body); } } @@ -183,6 +191,16 @@ private void onTtyLog(byte[] body) { } } + private void onProfile(String token, byte[] body) { + try { + AgentProfile profile = objectMapper.readValue(body, AgentProfile.class); + profile.setId(token); + eventManager.publish(new OnAgentProfileEvent(this, profile)); + } catch (IOException e) { + log.warn(e); + } + } + private static String getToken(WebSocketSession session) { return session.getHandshakeHeaders().get(HeaderToken).get(0); } diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentService.java b/core/src/main/java/com/flowci/core/agent/service/AgentService.java index e78796b70..8a55b3640 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentService.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentService.java @@ -95,11 +95,6 @@ public interface AgentService { */ Agent update(String token, String name, Set tags); - /** - * Update agent resource - */ - Agent update(String token, Agent.Resource resource); - /** * Update agent status */ diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java index 57d4aad43..2ddcce232 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flowci.core.agent.dao.AgentDao; +import com.flowci.core.agent.dao.AgentProfileDao; import com.flowci.core.agent.domain.Agent; import com.flowci.core.agent.domain.Agent.Status; import com.flowci.core.agent.domain.AgentInit; @@ -80,6 +81,9 @@ public class AgentServiceImpl implements AgentService { @Autowired private AgentDao agentDao; + @Autowired + private AgentProfileDao agentProfileDao; + @Autowired private SpringEventManager eventManager; @@ -314,14 +318,6 @@ public Agent update(String token, String name, Set tags) { } } - @Override - public Agent update(String token, Agent.Resource resource) { - Agent agent = getByToken(token); - agent.setResource(resource); - agentDao.save(agent); - return agent; - } - @Override public Agent update(Agent agent, Status status) { if (agent.getStatus() == status) { @@ -382,7 +378,6 @@ public void onConnected(OnConnectedEvent event) { target.setK8sCluster(init.getK8sCluster()); target.setUrl("http://" + init.getIp() + ":" + init.getPort()); target.setOs(init.getOs()); - target.setResource(init.getResource()); update(target, init.getStatus()); @@ -394,6 +389,11 @@ public void onConnected(OnConnectedEvent event) { } } + @EventListener + public void onProfileReceived(OnAgentProfileEvent event) { + agentProfileDao.save(event.getProfile()); + } + @EventListener public void onDisconnected(OnDisconnectedEvent event) { Optional lock = lock(); diff --git a/core/src/main/java/com/flowci/core/api/OpenRestController.java b/core/src/main/java/com/flowci/core/api/OpenRestController.java index dcde64886..0ebddbc7e 100644 --- a/core/src/main/java/com/flowci/core/api/OpenRestController.java +++ b/core/src/main/java/com/flowci/core/api/OpenRestController.java @@ -17,9 +17,7 @@ package com.flowci.core.api; -import com.flowci.core.agent.domain.Agent; import com.flowci.core.agent.service.AgentService; -import com.flowci.core.api.adviser.ApiAuth; import com.flowci.core.api.domain.AddStatsItem; import com.flowci.core.api.domain.CreateJobArtifact; import com.flowci.core.api.domain.CreateJobReport; @@ -133,12 +131,6 @@ public void uploadJobArtifact(@PathVariable String name, openRestService.saveJobArtifact(name, number, meta, file); } - @PostMapping("/profile") - public void profile(@RequestHeader(ApiAuth.HeaderAgentToken) String token, - @RequestBody Agent.Resource resource) { - agentService.update(token, resource); - } - @PostMapping("/logs/upload") public void upload(@RequestPart("file") MultipartFile file) { try (InputStream stream = file.getInputStream()) { From 98e032ca6348d625c5effbf5a7d8e33fda56b53e Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sat, 6 Mar 2021 18:17:42 +0100 Subject: [PATCH 05/46] set agent profile to job --- .../flowci/core/agent/domain/AgentProfile.java | 2 ++ .../core/agent/service/AgentService.java | 6 ++++++ .../core/agent/service/AgentServiceImpl.java | 11 +++++++---- .../java/com/flowci/core/job/domain/Job.java | 18 +++++++++++------- .../core/job/manager/JobActionManagerImpl.java | 8 +++++--- 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/com/flowci/core/agent/domain/AgentProfile.java b/core/src/main/java/com/flowci/core/agent/domain/AgentProfile.java index 6edcee42a..09655e883 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/AgentProfile.java +++ b/core/src/main/java/com/flowci/core/agent/domain/AgentProfile.java @@ -13,6 +13,8 @@ @Accessors(chain = true) public class AgentProfile extends Mongoable { + public static final AgentProfile EMPTY = new AgentProfile(); + private int cpuNum; private double cpuUsage; diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentService.java b/core/src/main/java/com/flowci/core/agent/service/AgentService.java index 8a55b3640..fe174b57b 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentService.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentService.java @@ -17,6 +17,7 @@ package com.flowci.core.agent.service; import com.flowci.core.agent.domain.Agent; +import com.flowci.core.agent.domain.AgentProfile; import com.flowci.core.agent.domain.CmdIn; import com.flowci.tree.Selector; @@ -35,6 +36,11 @@ public interface AgentService { */ Agent get(String id); + /** + * Get agent profile by token + */ + AgentProfile getProfile(String token); + /** * Get agent by name */ diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java index 2ddcce232..179f29a0b 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java @@ -19,11 +19,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flowci.core.agent.dao.AgentDao; import com.flowci.core.agent.dao.AgentProfileDao; -import com.flowci.core.agent.domain.Agent; +import com.flowci.core.agent.domain.*; import com.flowci.core.agent.domain.Agent.Status; -import com.flowci.core.agent.domain.AgentInit; -import com.flowci.core.agent.domain.CmdIn; -import com.flowci.core.agent.domain.Util; import com.flowci.core.agent.event.*; import com.flowci.core.agent.manager.AgentEventManager; import com.flowci.core.common.config.AppProperties; @@ -153,6 +150,12 @@ public Agent get(String id) { return optional.get(); } + @Override + public AgentProfile getProfile(String token) { + Optional optional = agentProfileDao.findById(token); + return optional.orElse(AgentProfile.EMPTY); + } + @Override public Agent getByName(String name) { Agent agent = agentDao.findByName(name); diff --git a/core/src/main/java/com/flowci/core/job/domain/Job.java b/core/src/main/java/com/flowci/core/job/domain/Job.java index cb0948042..2c8bff6c2 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Job.java +++ b/core/src/main/java/com/flowci/core/job/domain/Job.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.flowci.core.agent.domain.Agent; +import com.flowci.core.agent.domain.AgentProfile; import com.flowci.core.common.domain.Mongoable; import com.flowci.core.common.domain.Variables; import com.flowci.domain.StringVars; @@ -159,7 +160,9 @@ public static class AgentSnapshot { private String os; - private int cpu; + private int cpuNum; + + private double cpuUsage; private int totalMemory; @@ -323,15 +326,16 @@ public boolean isExpired() { return Instant.now().compareTo(expireAt) > 0; } - public void addAgentSnapshot(Agent agent) { + public void addAgentSnapshot(Agent agent, AgentProfile profile) { AgentSnapshot s = new AgentSnapshot(); s.name = agent.getName(); s.os = agent.getOs().name(); - s.cpu = agent.getResource().getCpu(); - s.totalMemory = agent.getResource().getTotalMemory(); - s.freeMemory = agent.getResource().getFreeMemory(); - s.totalDisk = agent.getResource().getTotalDisk(); - s.freeDisk = agent.getResource().getFreeDisk(); + s.cpuNum = profile.getCpuNum(); + s.cpuUsage = profile.getCpuUsage(); + s.totalMemory = profile.getTotalMemory(); + s.freeMemory = profile.getFreeMemory(); + s.totalDisk = profile.getTotalDisk(); + s.freeDisk = profile.getFreeDisk(); this.snapshots.put(agent.getId(), s); } diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index 893ba04da..a32912bf7 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -17,6 +17,7 @@ package com.flowci.core.job.manager; import com.flowci.core.agent.domain.Agent; +import com.flowci.core.agent.domain.AgentProfile; import com.flowci.core.agent.domain.CmdIn; import com.flowci.core.agent.domain.ShellIn; import com.flowci.core.agent.event.IdleAgentEvent; @@ -682,7 +683,8 @@ private Optional fetchAgentFromPool(Job job, Node node) { Optional optional = agentService.acquire(job.getId(), selector); if (optional.isPresent()) { Agent agent = optional.get(); - job.addAgentSnapshot(agent); + AgentProfile profile = agentService.getProfile(agent.getToken()); + job.addAgentSnapshot(agent, profile); jobAgentDao.addFlowToAgent(job.getId(), agent.getId(), flow.getPathAsString()); setJobStatusAndSave(job, job.getStatus(), null); return optional; @@ -709,8 +711,8 @@ private boolean assignAgentToWaitingStep(String agentId, Job job, NodeTree tree, Optional acquired = agentService.acquire(job.getId(), s, agentId, shouldIdle); if (acquired.isPresent()) { Agent agent = acquired.get(); - - job.addAgentSnapshot(agent); + AgentProfile profile = agentService.getProfile(agent.getToken()); + job.addAgentSnapshot(agent, profile); setJobStatusAndSave(job, job.getStatus(), null); jobAgentDao.addFlowToAgent(job.getId(), agent.getId(), f.getPathAsString()); From 8bdb7deb67b55abb39cd44b5f979761c733c5ba4 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sat, 6 Mar 2021 23:32:27 +0100 Subject: [PATCH 06/46] ws push agent profile --- .../com/flowci/core/agent/service/AgentServiceImpl.java | 9 +++++++++ .../com/flowci/core/common/config/WebSocketConfig.java | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java index 179f29a0b..ef7782b27 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java @@ -24,8 +24,10 @@ import com.flowci.core.agent.event.*; import com.flowci.core.agent.manager.AgentEventManager; import com.flowci.core.common.config.AppProperties; +import com.flowci.core.common.domain.PushEvent; import com.flowci.core.common.helper.CipherHelper; import com.flowci.core.common.helper.ThreadHelper; +import com.flowci.core.common.manager.SocketPushManager; import com.flowci.core.common.manager.SpringEventManager; import com.flowci.core.common.manager.SpringTaskManager; import com.flowci.core.common.rabbit.RabbitOperations; @@ -69,6 +71,9 @@ public class AgentServiceImpl implements AgentService { private static final int MaxIdleAgentPushBack = 10; // seconds + @Autowired + private String topicForAgentProfile; + @Autowired private AppProperties.Zookeeper zkProperties; @@ -99,6 +104,9 @@ public class AgentServiceImpl implements AgentService { @Autowired private RabbitOperations idleAgentQueueManager; + @Autowired + private SocketPushManager socketPushManager; + @EventListener(ContextRefreshedEvent.class) public void initAgentStatus() { taskManager.run("init-agent-status", true, () -> { @@ -395,6 +403,7 @@ public void onConnected(OnConnectedEvent event) { @EventListener public void onProfileReceived(OnAgentProfileEvent event) { agentProfileDao.save(event.getProfile()); + socketPushManager.push(topicForAgentProfile, PushEvent.STATUS_CHANGE, event.getProfile()); } @EventListener diff --git a/core/src/main/java/com/flowci/core/common/config/WebSocketConfig.java b/core/src/main/java/com/flowci/core/common/config/WebSocketConfig.java index 74396c8ec..b629acecb 100644 --- a/core/src/main/java/com/flowci/core/common/config/WebSocketConfig.java +++ b/core/src/main/java/com/flowci/core/common/config/WebSocketConfig.java @@ -108,6 +108,11 @@ public String topicForAgents() { return "/topic/agents"; } + @Bean("topicForAgentProfile") + public String topicForAgentProfile() { + return "/topic/agent_profile"; + } + /** * To subscribe agent host update */ From c397299f4f1bd0eba8b3e4db7a6c35e3460a41ba Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 7 Mar 2021 15:16:45 +0100 Subject: [PATCH 07/46] apply instant to agent status updated at --- .../com/flowci/core/agent/domain/Agent.java | 9 +++--- .../core/agent/service/AgentServiceImpl.java | 1 + .../flowci/core/common/helper/DateHelper.java | 32 ++++++++++++++++++- .../core/common/helper/JacksonHelper.java | 6 ++++ 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/flowci/core/agent/domain/Agent.java b/core/src/main/java/com/flowci/core/agent/domain/Agent.java index 3205a38c6..420b69fb8 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/Agent.java +++ b/core/src/main/java/com/flowci/core/agent/domain/Agent.java @@ -22,14 +22,15 @@ import com.flowci.domain.SimpleKeyPair; import com.flowci.tree.Selector; import com.google.common.base.Strings; -import lombok.*; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; import lombok.experimental.Accessors; import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; import java.time.Instant; import java.util.Collections; -import java.util.Date; import java.util.HashSet; import java.util.Set; @@ -85,7 +86,7 @@ public static Status fromBytes(byte[] bytes) { private Status status = Status.OFFLINE; - private Date statusUpdatedAt; + private Instant statusUpdatedAt; private String jobId; @@ -105,7 +106,7 @@ public Agent(String name, Set tags) { public void setStatus(Status status) { this.status = status; - this.statusUpdatedAt = new Date(); + this.statusUpdatedAt = Instant.now(); } public void setTags(Set tags) { diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java index ef7782b27..6a4eb46da 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java @@ -48,6 +48,7 @@ import org.springframework.stereotype.Service; import java.io.IOException; +import java.time.Instant; import java.util.*; import static com.flowci.core.agent.domain.Agent.Status.*; diff --git a/core/src/main/java/com/flowci/core/common/helper/DateHelper.java b/core/src/main/java/com/flowci/core/common/helper/DateHelper.java index 39948bedc..11079d3c0 100644 --- a/core/src/main/java/com/flowci/core/common/helper/DateHelper.java +++ b/core/src/main/java/com/flowci/core/common/helper/DateHelper.java @@ -16,10 +16,20 @@ package com.flowci.core.common.helper; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; import com.flowci.exception.ArgumentException; + +import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.Date; /** @@ -29,6 +39,26 @@ public abstract class DateHelper { private static final SimpleDateFormat intDayFormatter = new SimpleDateFormat("yyyyMMdd"); + private static final DateTimeFormatter utaDateFormat = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneOffset.UTC); + + public static class InstantUTCSerializer extends JsonSerializer { + + @Override + public void serialize(Instant instant, JsonGenerator gen, SerializerProvider provider) throws IOException { + String str = utaDateFormat.format(instant); + gen.writeString(str); + } + } + + public static class InstantUTCDeserializer extends JsonDeserializer { + + @Override + public Instant deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + return Instant.from(utaDateFormat.parse(p.getText())); + } + } + public static synchronized int toIntDay(Date date) { return Integer.parseInt(intDayFormatter.format(date)); } @@ -37,7 +67,7 @@ public static synchronized Instant toInstant(int day) { try { Date date = intDayFormatter.parse("" + day); return date.toInstant(); - } catch (ParseException e) { + } catch (ParseException e) { throw new ArgumentException("Invalid day format"); } } diff --git a/core/src/main/java/com/flowci/core/common/helper/JacksonHelper.java b/core/src/main/java/com/flowci/core/common/helper/JacksonHelper.java index d249aec19..4f875be16 100644 --- a/core/src/main/java/com/flowci/core/common/helper/JacksonHelper.java +++ b/core/src/main/java/com/flowci/core/common/helper/JacksonHelper.java @@ -23,6 +23,8 @@ import com.flowci.core.common.domain.JsonablePage; import org.springframework.data.domain.Pageable; +import java.time.Instant; + /** * @author yang */ @@ -35,6 +37,10 @@ public static ObjectMapper create() { SimpleModule module = new SimpleModule(); module.addDeserializer(Pageable.class, new JsonablePage.PageableDeserializer()); + + module.addSerializer(Instant.class, new DateHelper.InstantUTCSerializer()); + module.addDeserializer(Instant.class, new DateHelper.InstantUTCDeserializer()); + mapper.registerModule(module); return mapper; From fb35473c54ef0db6c50daf868ffc144584012921 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 7 Mar 2021 16:57:43 +0100 Subject: [PATCH 08/46] fix agent address with wrong port num --- .../java/com/flowci/core/agent/manager/AgentEventManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java b/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java index 18112f99c..127341edc 100644 --- a/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java +++ b/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java @@ -152,7 +152,7 @@ private void onConnected(WebSocketSession session, String token, byte[] body) { Objects.requireNonNull(init.getStatus(), "Agent status is missing"); init.setToken(token); - init.setIp(session.getRemoteAddress() == null ? null : session.getRemoteAddress().toString()); + init.setIp(session.getRemoteAddress() == null ? null : session.getRemoteAddress().getAddress().toString()); eventManager.publish(new OnConnectedEvent(this, token, session, init)); agentSessionStore.put(token, session); From baaf244781d2c0a838663a9dbdaf5615c690a190 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 14 Mar 2021 15:19:05 +0100 Subject: [PATCH 09/46] refactor parameter of create/update agent --- .../agent/controller/AgentController.java | 5 ++-- .../com/flowci/core/agent/domain/Agent.java | 7 ++++++ .../flowci/core/agent/domain/AgentConfig.java | 13 ++++++++++ .../agent/domain/CreateOrUpdateAgent.java | 11 +++++++-- .../agent/service/AgentHostServiceImpl.java | 6 ++++- .../core/agent/service/AgentService.java | 6 ++--- .../core/agent/service/AgentServiceImpl.java | 21 +++++++++------- .../core/test/agent/AgentServiceTest.java | 14 +++++++---- .../flowci/core/test/job/JobServiceTest.java | 24 ++++++++++++------- 9 files changed, 76 insertions(+), 31 deletions(-) create mode 100644 core/src/main/java/com/flowci/core/agent/domain/AgentConfig.java diff --git a/core/src/main/java/com/flowci/core/agent/controller/AgentController.java b/core/src/main/java/com/flowci/core/agent/controller/AgentController.java index 278777c59..9e496c1b4 100644 --- a/core/src/main/java/com/flowci/core/agent/controller/AgentController.java +++ b/core/src/main/java/com/flowci/core/agent/controller/AgentController.java @@ -28,7 +28,6 @@ import org.springframework.web.bind.annotation.*; import java.util.List; -import java.util.Optional; /** * @author yang @@ -57,10 +56,10 @@ public List list() { @Action(AgentAction.CREATE_UPDATE) public Agent createOrUpdate(@Validated @RequestBody CreateOrUpdateAgent body) { if (body.hasToken()) { - return agentService.update(body.getToken(), body.getName(), body.getTags()); + return agentService.update(body); } - return agentService.create(body.getName(), body.getTags(), Optional.empty()); + return agentService.create(body); } @DeleteMapping() diff --git a/core/src/main/java/com/flowci/core/agent/domain/Agent.java b/core/src/main/java/com/flowci/core/agent/domain/Agent.java index 420b69fb8..4a6312243 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/Agent.java +++ b/core/src/main/java/com/flowci/core/agent/domain/Agent.java @@ -84,6 +84,8 @@ public static Status fromBytes(byte[] bytes) { private Set tags = Collections.emptySet(); + private int exitOnIdle; + private Status status = Status.OFFLINE; private Instant statusUpdatedAt; @@ -104,6 +106,11 @@ public Agent(String name, Set tags) { this.setTags(tags); } + @JsonIgnore + public AgentConfig getConfig() { + return new AgentConfig().setExitOnIdle(exitOnIdle); + } + public void setStatus(Status status) { this.status = status; this.statusUpdatedAt = Instant.now(); diff --git a/core/src/main/java/com/flowci/core/agent/domain/AgentConfig.java b/core/src/main/java/com/flowci/core/agent/domain/AgentConfig.java new file mode 100644 index 000000000..2e30a9561 --- /dev/null +++ b/core/src/main/java/com/flowci/core/agent/domain/AgentConfig.java @@ -0,0 +1,13 @@ +package com.flowci.core.agent.domain; + +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + +@Getter +@Setter +@Accessors(chain = true) +public class AgentConfig { + + private int ExitOnIdle; +} diff --git a/core/src/main/java/com/flowci/core/agent/domain/CreateOrUpdateAgent.java b/core/src/main/java/com/flowci/core/agent/domain/CreateOrUpdateAgent.java index b9f609b32..617bd48aa 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/CreateOrUpdateAgent.java +++ b/core/src/main/java/com/flowci/core/agent/domain/CreateOrUpdateAgent.java @@ -17,14 +17,17 @@ package com.flowci.core.agent.domain; import com.google.common.base.Strings; -import java.util.Set; -import javax.validation.constraints.NotEmpty; import lombok.Data; +import lombok.experimental.Accessors; + +import javax.validation.constraints.NotEmpty; +import java.util.Set; /** * @author yang */ @Data +@Accessors(chain = true) public class CreateOrUpdateAgent { @NotEmpty @@ -34,6 +37,10 @@ public class CreateOrUpdateAgent { private String token; + private int exitOnIdle; + + private String hostId; + public boolean hasToken() { return !Strings.isNullOrEmpty(token); } diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java index 04f41d95d..7351b6d19 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java @@ -267,7 +267,11 @@ public boolean start(AgentHost host) { Agent agent = null; try { - agent = agentService.create(name, host.getTags(), Optional.of(host.getId())); + agent = agentService.create(new CreateOrUpdateAgent() + .setName(name) + .setTags(host.getTags()) + .setHostId(host.getId()) + ); StartOption startOption = mapping.get(host.getClass()).buildStartOption(host, agent); String cid = cm.start(startOption); diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentService.java b/core/src/main/java/com/flowci/core/agent/service/AgentService.java index fe174b57b..c19514b51 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentService.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentService.java @@ -19,12 +19,12 @@ import com.flowci.core.agent.domain.Agent; import com.flowci.core.agent.domain.AgentProfile; import com.flowci.core.agent.domain.CmdIn; +import com.flowci.core.agent.domain.CreateOrUpdateAgent; import com.flowci.tree.Selector; import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.Set; /** * @author yang @@ -94,12 +94,12 @@ public interface AgentService { /** * Create agent by name and tags */ - Agent create(String name, Set tags, Optional hostId); + Agent create(CreateOrUpdateAgent option); /** * Update agent name or and tags */ - Agent update(String token, String name, Set tags); + Agent update(CreateOrUpdateAgent option); /** * Update agent status diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java index 6a4eb46da..ed7a2c9ed 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java @@ -48,7 +48,6 @@ import org.springframework.stereotype.Service; import java.io.IOException; -import java.time.Instant; import java.util.*; import static com.flowci.core.agent.domain.Agent.Status.*; @@ -294,7 +293,9 @@ public void release(Collection ids) { } @Override - public Agent create(String name, Set tags, Optional hostId) { + public Agent create(CreateOrUpdateAgent option) { + String name = option.getName(); + Agent exist = agentDao.findByName(name); if (exist != null) { throw new DuplicateException("Agent name {0} is already defined", name); @@ -302,9 +303,10 @@ public Agent create(String name, Set tags, Optional hostId) { try { // create agent - Agent agent = new Agent(name, tags); + Agent agent = new Agent(name, option.getTags()); agent.setToken(UUID.randomUUID().toString()); - hostId.ifPresent(agent::setHostId); + agent.setHostId(option.getHostId()); + agent.setExitOnIdle(option.getExitOnIdle()); String dummyEmailForAgent = "agent." + name + "@flow.ci"; agent.setRsa(CipherHelper.RSA.gen(dummyEmailForAgent)); @@ -318,15 +320,16 @@ public Agent create(String name, Set tags, Optional hostId) { } @Override - public Agent update(String token, String name, Set tags) { - Agent agent = getByToken(token); - agent.setName(name); - agent.setTags(tags); + public Agent update(CreateOrUpdateAgent option) { + Agent agent = getByToken(option.getToken()); + agent.setName(option.getName()); + agent.setTags(option.getTags()); + agent.setExitOnIdle(option.getExitOnIdle()); try { return agentDao.save(agent); } catch (DuplicateKeyException e) { - throw new DuplicateException("Agent name {0} is already defined", name); + throw new DuplicateException("Agent name {0} is already defined", option.getName()); } } diff --git a/core/src/test/java/com/flowci/core/test/agent/AgentServiceTest.java b/core/src/test/java/com/flowci/core/test/agent/AgentServiceTest.java index f172c2e41..4255828e9 100644 --- a/core/src/test/java/com/flowci/core/test/agent/AgentServiceTest.java +++ b/core/src/test/java/com/flowci/core/test/agent/AgentServiceTest.java @@ -19,6 +19,7 @@ import com.flowci.core.agent.domain.Agent; import com.flowci.core.agent.domain.Agent.Status; import com.flowci.core.agent.domain.CmdIn; +import com.flowci.core.agent.domain.CreateOrUpdateAgent; import com.flowci.core.agent.domain.ShellIn; import com.flowci.core.agent.event.CmdSentEvent; import com.flowci.core.agent.service.AgentService; @@ -31,7 +32,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; -import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -56,7 +56,10 @@ public void should_init_root_node() { @Test public void should_create_agent_in_db() { - Agent agent = agentService.create("hello.test", ImmutableSet.of("local", "android"), Optional.empty()); + Agent agent = agentService.create(new CreateOrUpdateAgent() + .setName("hello.test") + .setTags(ImmutableSet.of("local", "android")) + ); Assert.assertNotNull(agent); Assert.assertEquals(agent, agentService.get(agent.getId())); } @@ -64,7 +67,10 @@ public void should_create_agent_in_db() { @Test public void should_make_agent_online() throws InterruptedException { // init: - Agent agent = agentService.create("hello.test", ImmutableSet.of("local", "android"), Optional.empty()); + Agent agent = agentService.create(new CreateOrUpdateAgent() + .setName("hello.test") + .setTags(ImmutableSet.of("local", "android")) + ); // when: Agent online = mockAgentOnline(agent.getToken()); @@ -78,7 +84,7 @@ public void should_make_agent_online() throws InterruptedException { public void should_dispatch_cmd_to_agent() throws InterruptedException { // init: CmdIn cmd = new ShellIn(); - Agent agent = agentService.create("hello.agent", null, Optional.empty()); + Agent agent = agentService.create(new CreateOrUpdateAgent().setName("hello.agent")); // when: CountDownLatch counter = new CountDownLatch(1); diff --git a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java index 049a48c36..ae6a49959 100644 --- a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java +++ b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java @@ -19,6 +19,7 @@ import com.flowci.core.agent.dao.AgentDao; import com.flowci.core.agent.domain.Agent; import com.flowci.core.agent.domain.CmdIn; +import com.flowci.core.agent.domain.CreateOrUpdateAgent; import com.flowci.core.agent.domain.ShellIn; import com.flowci.core.agent.event.AgentStatusEvent; import com.flowci.core.agent.event.CmdSentEvent; @@ -32,21 +33,27 @@ import com.flowci.core.job.dao.ExecutedCmdDao; import com.flowci.core.job.dao.JobAgentDao; import com.flowci.core.job.dao.JobDao; -import com.flowci.core.job.domain.Step; import com.flowci.core.job.domain.Job; import com.flowci.core.job.domain.Job.Status; import com.flowci.core.job.domain.Job.Trigger; +import com.flowci.core.job.domain.Step; import com.flowci.core.job.event.JobReceivedEvent; import com.flowci.core.job.event.JobStatusChangeEvent; import com.flowci.core.job.event.StartAsyncLocalTaskEvent; import com.flowci.core.job.manager.JobActionManager; import com.flowci.core.job.manager.YmlManager; -import com.flowci.core.job.service.*; +import com.flowci.core.job.service.JobEventService; +import com.flowci.core.job.service.JobService; +import com.flowci.core.job.service.LocalTaskService; +import com.flowci.core.job.service.StepService; import com.flowci.core.plugin.dao.PluginDao; import com.flowci.core.plugin.domain.Plugin; import com.flowci.core.test.ZookeeperScenario; import com.flowci.domain.*; -import com.flowci.tree.*; +import com.flowci.tree.FlowNode; +import com.flowci.tree.Node; +import com.flowci.tree.NodeTree; +import com.flowci.tree.YmlParser; import com.flowci.util.StringHelper; import lombok.extern.log4j.Log4j2; import org.junit.Assert; @@ -63,7 +70,6 @@ import java.io.IOException; import java.util.Date; import java.util.List; -import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -227,7 +233,7 @@ public void should_finish_whole_job() throws InterruptedException, IOException { String yaml = StringHelper.toString(load("flow-with-notify.yml")); yml = ymlService.saveYml(flow, Yml.DEFAULT_NAME, StringHelper.toBase64(yaml)); - Agent agent = agentService.create("hello.agent", null, Optional.empty()); + Agent agent = agentService.create(new CreateOrUpdateAgent().setName("hello.agent")); mockAgentOnline(agent.getToken()); Job job = jobService.create(flow, yml.getRaw(), Trigger.MANUAL, StringVars.EMPTY); @@ -344,7 +350,7 @@ public void should_finish_whole_job() throws InterruptedException, IOException { @Test public void should_handle_cmd_callback_for_success_status() { // init: agent and job - Agent agent = agentService.create("hello.agent", null, Optional.empty()); + Agent agent = agentService.create(new CreateOrUpdateAgent().setName("hello.agent")); Job job = prepareJobForRunningStatus(agent); NodeTree tree = ymlManager.getTree(job); @@ -389,7 +395,7 @@ public void should_handle_cmd_callback_for_failure_status() throws IOException { // init: agent and job String yaml = StringHelper.toString(load("flow-with-failure.yml")); yml = ymlService.saveYml(flow, Yml.DEFAULT_NAME, StringHelper.toBase64(yaml)); - Agent agent = agentService.create("hello.agent", null, Optional.empty()); + Agent agent = agentService.create(new CreateOrUpdateAgent().setName("hello.agent")); Job job = prepareJobForRunningStatus(agent); NodeTree tree = ymlManager.getTree(job); @@ -412,7 +418,7 @@ public void should_handle_cmd_callback_for_failure_status_but_allow_failure() th // init: agent and job String yaml = StringHelper.toString(load("flow-all-failure.yml")); yml = ymlService.saveYml(flow, Yml.DEFAULT_NAME, StringHelper.toBase64(yaml)); - Agent agent = agentService.create("hello.agent", null, Optional.empty()); + Agent agent = agentService.create(new CreateOrUpdateAgent().setName("hello.agent")); Job job = prepareJobForRunningStatus(agent); NodeTree tree = ymlManager.getTree(job); @@ -459,7 +465,7 @@ public void should_cancel_job_if_agent_offline() throws IOException, Interrupted yml = ymlService.saveYml(flow, Yml.DEFAULT_NAME, StringHelper.toBase64(yaml)); // mock agent online - Agent agent = agentService.create("hello.agent.2", null, Optional.empty()); + Agent agent = agentService.create(new CreateOrUpdateAgent().setName("hello.agent.2")); mockAgentOnline(agent.getToken()); // given: start job and wait for running From 107d8d985720256aeb707503acbed8f1062b28dd Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 14 Mar 2021 15:23:09 +0100 Subject: [PATCH 10/46] return agent config on connected --- .../flowci/core/agent/event/OnConnectedEvent.java | 5 +++++ .../flowci/core/agent/manager/AgentEventManager.java | 12 ++++++------ .../flowci/core/agent/service/AgentServiceImpl.java | 2 ++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/flowci/core/agent/event/OnConnectedEvent.java b/core/src/main/java/com/flowci/core/agent/event/OnConnectedEvent.java index bacd20758..6a6695e04 100644 --- a/core/src/main/java/com/flowci/core/agent/event/OnConnectedEvent.java +++ b/core/src/main/java/com/flowci/core/agent/event/OnConnectedEvent.java @@ -1,14 +1,19 @@ package com.flowci.core.agent.event; +import com.flowci.core.agent.domain.Agent; import com.flowci.core.agent.domain.AgentInit; import lombok.Getter; +import lombok.Setter; import org.springframework.web.socket.WebSocketSession; @Getter +@Setter public class OnConnectedEvent extends EventFromClient { private final AgentInit init; + private Agent agent; + public OnConnectedEvent(Object source, String token, WebSocketSession session, AgentInit init) { super(source, token, session); this.init = init; diff --git a/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java b/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java index 127341edc..0574d7bad 100644 --- a/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java +++ b/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java @@ -1,10 +1,7 @@ package com.flowci.core.agent.manager; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flowci.core.agent.domain.AgentInit; -import com.flowci.core.agent.domain.AgentProfile; -import com.flowci.core.agent.domain.ShellLog; -import com.flowci.core.agent.domain.TtyCmd; +import com.flowci.core.agent.domain.*; import com.flowci.core.agent.event.*; import com.flowci.core.common.domain.StatusCode; import com.flowci.core.common.domain.http.ResponseMessage; @@ -154,9 +151,12 @@ private void onConnected(WebSocketSession session, String token, byte[] body) { init.setToken(token); init.setIp(session.getRemoteAddress() == null ? null : session.getRemoteAddress().getAddress().toString()); - eventManager.publish(new OnConnectedEvent(this, token, session, init)); + OnConnectedEvent event = new OnConnectedEvent(this, token, session, init); + eventManager.publish(event); agentSessionStore.put(token, session); - writeMessage(token, new ResponseMessage(StatusCode.OK, null)); + + Agent agent = event.getAgent(); + writeMessage(token, new ResponseMessage<>(StatusCode.OK, agent.getConfig())); log.debug("Agent {} is connected with status {}", token, init.getStatus()); } catch (Exception e) { log.warn(e); diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java index ed7a2c9ed..3a2cd4150 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java @@ -399,6 +399,8 @@ public void onConnected(OnConnectedEvent event) { if (target.isIdle()) { idleAgentQueueManager.send(idleAgentQueue, target.getId().getBytes()); } + + event.setAgent(target); } finally { unlock(lock.get()); } From 99475b7e43faba6bd0b0ecc2d461c2ec1a05b289 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 14 Mar 2021 17:42:11 +0100 Subject: [PATCH 11/46] support edit on idle on host --- .../main/java/com/flowci/core/agent/domain/AgentHost.java | 5 +++++ .../java/com/flowci/core/agent/domain/SaveAgentHost.java | 5 +++++ .../com/flowci/core/agent/service/AgentHostServiceImpl.java | 1 + 3 files changed, 11 insertions(+) diff --git a/core/src/main/java/com/flowci/core/agent/domain/AgentHost.java b/core/src/main/java/com/flowci/core/agent/domain/AgentHost.java index e3cf81221..e08205019 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/AgentHost.java +++ b/core/src/main/java/com/flowci/core/agent/domain/AgentHost.java @@ -80,6 +80,11 @@ public enum Type { */ private Set tags = new HashSet<>(); + /** + * Ref to Agent.exitOnIdle + */ + private int exitOnIdle; + /** * Error message if connection fail */ diff --git a/core/src/main/java/com/flowci/core/agent/domain/SaveAgentHost.java b/core/src/main/java/com/flowci/core/agent/domain/SaveAgentHost.java index 46ec6fd98..ec3b16f09 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/SaveAgentHost.java +++ b/core/src/main/java/com/flowci/core/agent/domain/SaveAgentHost.java @@ -35,6 +35,8 @@ public class SaveAgentHost { private Set tags = new HashSet<>(); + private int exitOnIdle; + @NotNull private AgentHost.Type type; @@ -68,6 +70,7 @@ public AgentHost toObj() { host.setIp(ip); host.setMaxSize(maxSize); host.setPort(port); + host.setExitOnIdle(exitOnIdle); return host; } @@ -77,6 +80,7 @@ public AgentHost toObj() { host.setName(name); host.setTags(tags); host.setMaxSize(maxSize); + host.setExitOnIdle(exitOnIdle); return host; } @@ -88,6 +92,7 @@ public AgentHost toObj() { host.setSecret(secret); host.setNamespace(namespace); host.setMaxSize(maxSize); + host.setExitOnIdle(exitOnIdle); return host; } diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java index 7351b6d19..8e9c35518 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java @@ -270,6 +270,7 @@ public boolean start(AgentHost host) { agent = agentService.create(new CreateOrUpdateAgent() .setName(name) .setTags(host.getTags()) + .setExitOnIdle(host.getExitOnIdle()) .setHostId(host.getId()) ); From 51c801d8e1f5a817444d083fa54b03edb024787b Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 14 Mar 2021 22:50:48 +0100 Subject: [PATCH 12/46] rename agent parameter to option --- .../core/agent/controller/AgentController.java | 4 ++-- .../core/agent/controller/AgentHostController.java | 4 ++-- .../{SaveAgentHost.java => AgentHostOption.java} | 2 +- .../{CreateOrUpdateAgent.java => AgentOption.java} | 2 +- .../core/agent/service/AgentHostServiceImpl.java | 2 +- .../com/flowci/core/agent/service/AgentService.java | 6 +++--- .../flowci/core/agent/service/AgentServiceImpl.java | 4 ++-- .../flowci/core/test/agent/AgentControllerTest.java | 4 ++-- .../com/flowci/core/test/agent/AgentServiceTest.java | 8 ++++---- .../com/flowci/core/test/job/JobServiceTest.java | 12 ++++++------ 10 files changed, 24 insertions(+), 24 deletions(-) rename core/src/main/java/com/flowci/core/agent/domain/{SaveAgentHost.java => AgentHostOption.java} (98%) rename core/src/main/java/com/flowci/core/agent/domain/{CreateOrUpdateAgent.java => AgentOption.java} (96%) diff --git a/core/src/main/java/com/flowci/core/agent/controller/AgentController.java b/core/src/main/java/com/flowci/core/agent/controller/AgentController.java index 9e496c1b4..b0cb01b1c 100644 --- a/core/src/main/java/com/flowci/core/agent/controller/AgentController.java +++ b/core/src/main/java/com/flowci/core/agent/controller/AgentController.java @@ -18,7 +18,7 @@ import com.flowci.core.agent.domain.Agent; import com.flowci.core.agent.domain.AgentAction; -import com.flowci.core.agent.domain.CreateOrUpdateAgent; +import com.flowci.core.agent.domain.AgentOption; import com.flowci.core.agent.domain.DeleteAgent; import com.flowci.core.agent.service.AgentService; import com.flowci.core.auth.annotation.Action; @@ -54,7 +54,7 @@ public List list() { @PostMapping() @Action(AgentAction.CREATE_UPDATE) - public Agent createOrUpdate(@Validated @RequestBody CreateOrUpdateAgent body) { + public Agent createOrUpdate(@Validated @RequestBody AgentOption body) { if (body.hasToken()) { return agentService.update(body); } diff --git a/core/src/main/java/com/flowci/core/agent/controller/AgentHostController.java b/core/src/main/java/com/flowci/core/agent/controller/AgentHostController.java index 6b65ca406..781dda4d4 100644 --- a/core/src/main/java/com/flowci/core/agent/controller/AgentHostController.java +++ b/core/src/main/java/com/flowci/core/agent/controller/AgentHostController.java @@ -18,7 +18,7 @@ import com.flowci.core.agent.domain.AgentHost; import com.flowci.core.agent.domain.AgentHostAction; -import com.flowci.core.agent.domain.SaveAgentHost; +import com.flowci.core.agent.domain.AgentHostOption; import com.flowci.core.agent.service.AgentHostService; import com.flowci.core.auth.annotation.Action; import lombok.extern.log4j.Log4j2; @@ -56,7 +56,7 @@ public AgentHost deleteByName(@PathVariable String name) { @PostMapping @Action(AgentHostAction.CREATE_UPDATE) - public AgentHost createOrUpdate(@RequestBody @Validated SaveAgentHost body) { + public AgentHost createOrUpdate(@RequestBody @Validated AgentHostOption body) { AgentHost host = body.toObj(); return agentHostService.createOrUpdate(host); } diff --git a/core/src/main/java/com/flowci/core/agent/domain/SaveAgentHost.java b/core/src/main/java/com/flowci/core/agent/domain/AgentHostOption.java similarity index 98% rename from core/src/main/java/com/flowci/core/agent/domain/SaveAgentHost.java rename to core/src/main/java/com/flowci/core/agent/domain/AgentHostOption.java index ec3b16f09..b34465b31 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/SaveAgentHost.java +++ b/core/src/main/java/com/flowci/core/agent/domain/AgentHostOption.java @@ -29,7 +29,7 @@ @Getter @Setter -public class SaveAgentHost { +public class AgentHostOption { private String id; diff --git a/core/src/main/java/com/flowci/core/agent/domain/CreateOrUpdateAgent.java b/core/src/main/java/com/flowci/core/agent/domain/AgentOption.java similarity index 96% rename from core/src/main/java/com/flowci/core/agent/domain/CreateOrUpdateAgent.java rename to core/src/main/java/com/flowci/core/agent/domain/AgentOption.java index 617bd48aa..e42938848 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/CreateOrUpdateAgent.java +++ b/core/src/main/java/com/flowci/core/agent/domain/AgentOption.java @@ -28,7 +28,7 @@ */ @Data @Accessors(chain = true) -public class CreateOrUpdateAgent { +public class AgentOption { @NotEmpty private String name; diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java index 8e9c35518..488bfe9f7 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java @@ -267,7 +267,7 @@ public boolean start(AgentHost host) { Agent agent = null; try { - agent = agentService.create(new CreateOrUpdateAgent() + agent = agentService.create(new AgentOption() .setName(name) .setTags(host.getTags()) .setExitOnIdle(host.getExitOnIdle()) diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentService.java b/core/src/main/java/com/flowci/core/agent/service/AgentService.java index c19514b51..0d53bf0c2 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentService.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentService.java @@ -19,7 +19,7 @@ import com.flowci.core.agent.domain.Agent; import com.flowci.core.agent.domain.AgentProfile; import com.flowci.core.agent.domain.CmdIn; -import com.flowci.core.agent.domain.CreateOrUpdateAgent; +import com.flowci.core.agent.domain.AgentOption; import com.flowci.tree.Selector; import java.util.Collection; @@ -94,12 +94,12 @@ public interface AgentService { /** * Create agent by name and tags */ - Agent create(CreateOrUpdateAgent option); + Agent create(AgentOption option); /** * Update agent name or and tags */ - Agent update(CreateOrUpdateAgent option); + Agent update(AgentOption option); /** * Update agent status diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java index 3a2cd4150..5bb7ca45a 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java @@ -293,7 +293,7 @@ public void release(Collection ids) { } @Override - public Agent create(CreateOrUpdateAgent option) { + public Agent create(AgentOption option) { String name = option.getName(); Agent exist = agentDao.findByName(name); @@ -320,7 +320,7 @@ public Agent create(CreateOrUpdateAgent option) { } @Override - public Agent update(CreateOrUpdateAgent option) { + public Agent update(AgentOption option) { Agent agent = getByToken(option.getToken()); agent.setName(option.getName()); agent.setTags(option.getTags()); diff --git a/core/src/test/java/com/flowci/core/test/agent/AgentControllerTest.java b/core/src/test/java/com/flowci/core/test/agent/AgentControllerTest.java index 8c292cdf3..156726b58 100644 --- a/core/src/test/java/com/flowci/core/test/agent/AgentControllerTest.java +++ b/core/src/test/java/com/flowci/core/test/agent/AgentControllerTest.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.flowci.core.agent.domain.Agent; -import com.flowci.core.agent.domain.CreateOrUpdateAgent; +import com.flowci.core.agent.domain.AgentOption; import com.flowci.core.agent.domain.DeleteAgent; import com.flowci.core.common.config.AppProperties; import com.flowci.core.common.domain.StatusCode; @@ -110,7 +110,7 @@ public void should_delete_agent() throws Throwable { } private Agent createAgent(String name, Set tags, Integer code) throws Exception { - CreateOrUpdateAgent create = new CreateOrUpdateAgent(); + AgentOption create = new AgentOption(); create.setName(name); create.setTags(tags); diff --git a/core/src/test/java/com/flowci/core/test/agent/AgentServiceTest.java b/core/src/test/java/com/flowci/core/test/agent/AgentServiceTest.java index 4255828e9..0771a3995 100644 --- a/core/src/test/java/com/flowci/core/test/agent/AgentServiceTest.java +++ b/core/src/test/java/com/flowci/core/test/agent/AgentServiceTest.java @@ -19,7 +19,7 @@ import com.flowci.core.agent.domain.Agent; import com.flowci.core.agent.domain.Agent.Status; import com.flowci.core.agent.domain.CmdIn; -import com.flowci.core.agent.domain.CreateOrUpdateAgent; +import com.flowci.core.agent.domain.AgentOption; import com.flowci.core.agent.domain.ShellIn; import com.flowci.core.agent.event.CmdSentEvent; import com.flowci.core.agent.service.AgentService; @@ -56,7 +56,7 @@ public void should_init_root_node() { @Test public void should_create_agent_in_db() { - Agent agent = agentService.create(new CreateOrUpdateAgent() + Agent agent = agentService.create(new AgentOption() .setName("hello.test") .setTags(ImmutableSet.of("local", "android")) ); @@ -67,7 +67,7 @@ public void should_create_agent_in_db() { @Test public void should_make_agent_online() throws InterruptedException { // init: - Agent agent = agentService.create(new CreateOrUpdateAgent() + Agent agent = agentService.create(new AgentOption() .setName("hello.test") .setTags(ImmutableSet.of("local", "android")) ); @@ -84,7 +84,7 @@ public void should_make_agent_online() throws InterruptedException { public void should_dispatch_cmd_to_agent() throws InterruptedException { // init: CmdIn cmd = new ShellIn(); - Agent agent = agentService.create(new CreateOrUpdateAgent().setName("hello.agent")); + Agent agent = agentService.create(new AgentOption().setName("hello.agent")); // when: CountDownLatch counter = new CountDownLatch(1); diff --git a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java index ae6a49959..ec4d89127 100644 --- a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java +++ b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java @@ -19,7 +19,7 @@ import com.flowci.core.agent.dao.AgentDao; import com.flowci.core.agent.domain.Agent; import com.flowci.core.agent.domain.CmdIn; -import com.flowci.core.agent.domain.CreateOrUpdateAgent; +import com.flowci.core.agent.domain.AgentOption; import com.flowci.core.agent.domain.ShellIn; import com.flowci.core.agent.event.AgentStatusEvent; import com.flowci.core.agent.event.CmdSentEvent; @@ -233,7 +233,7 @@ public void should_finish_whole_job() throws InterruptedException, IOException { String yaml = StringHelper.toString(load("flow-with-notify.yml")); yml = ymlService.saveYml(flow, Yml.DEFAULT_NAME, StringHelper.toBase64(yaml)); - Agent agent = agentService.create(new CreateOrUpdateAgent().setName("hello.agent")); + Agent agent = agentService.create(new AgentOption().setName("hello.agent")); mockAgentOnline(agent.getToken()); Job job = jobService.create(flow, yml.getRaw(), Trigger.MANUAL, StringVars.EMPTY); @@ -350,7 +350,7 @@ public void should_finish_whole_job() throws InterruptedException, IOException { @Test public void should_handle_cmd_callback_for_success_status() { // init: agent and job - Agent agent = agentService.create(new CreateOrUpdateAgent().setName("hello.agent")); + Agent agent = agentService.create(new AgentOption().setName("hello.agent")); Job job = prepareJobForRunningStatus(agent); NodeTree tree = ymlManager.getTree(job); @@ -395,7 +395,7 @@ public void should_handle_cmd_callback_for_failure_status() throws IOException { // init: agent and job String yaml = StringHelper.toString(load("flow-with-failure.yml")); yml = ymlService.saveYml(flow, Yml.DEFAULT_NAME, StringHelper.toBase64(yaml)); - Agent agent = agentService.create(new CreateOrUpdateAgent().setName("hello.agent")); + Agent agent = agentService.create(new AgentOption().setName("hello.agent")); Job job = prepareJobForRunningStatus(agent); NodeTree tree = ymlManager.getTree(job); @@ -418,7 +418,7 @@ public void should_handle_cmd_callback_for_failure_status_but_allow_failure() th // init: agent and job String yaml = StringHelper.toString(load("flow-all-failure.yml")); yml = ymlService.saveYml(flow, Yml.DEFAULT_NAME, StringHelper.toBase64(yaml)); - Agent agent = agentService.create(new CreateOrUpdateAgent().setName("hello.agent")); + Agent agent = agentService.create(new AgentOption().setName("hello.agent")); Job job = prepareJobForRunningStatus(agent); NodeTree tree = ymlManager.getTree(job); @@ -465,7 +465,7 @@ public void should_cancel_job_if_agent_offline() throws IOException, Interrupted yml = ymlService.saveYml(flow, Yml.DEFAULT_NAME, StringHelper.toBase64(yaml)); // mock agent online - Agent agent = agentService.create(new CreateOrUpdateAgent().setName("hello.agent.2")); + Agent agent = agentService.create(new AgentOption().setName("hello.agent.2")); mockAgentOnline(agent.getToken()); // given: start job and wait for running From d36d74ae129222c0068cb74a4b633e55b29d3c25 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Wed, 17 Mar 2021 14:27:24 +0100 Subject: [PATCH 13/46] mk fun for duplicated code --- .../job/manager/JobActionManagerImpl.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index a32912bf7..19f4d247a 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -281,11 +281,7 @@ private void fromPending() { sm.add(PendingToCreated, new Action() { @Override public void accept(JobSmContext context) { - Job job = context.job; - String yml = context.yml; - - setupJobYamlAndSteps(job, yml); - setJobStatusAndSave(job, Job.Status.CREATED, StringHelper.EMPTY); + doFromXToCreated(context); } }); @@ -325,15 +321,19 @@ public void accept(JobSmContext context) { sm.add(LoadingToCreated, new Action() { @Override public void accept(JobSmContext context) { - Job job = context.job; - String yml = context.yml; - - setupJobYamlAndSteps(job, yml); - setJobStatusAndSave(job, Job.Status.CREATED, StringHelper.EMPTY); + doFromXToCreated(context); } }); } + private void doFromXToCreated(JobSmContext context) { + Job job = context.job; + String yml = context.yml; + + setupJobYamlAndSteps(job, yml); + setJobStatusAndSave(job, Job.Status.CREATED, StringHelper.EMPTY); + } + private void fromCreated() { sm.add(CreatedToTimeout, new Action() { @Override From cece71db6bbcebd01e4d9194628d1a507d93c756 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Wed, 17 Mar 2021 22:31:19 +0100 Subject: [PATCH 14/46] save job context stamp on step --- .../java/com/flowci/core/job/domain/Step.java | 6 ++++++ .../job/manager/JobActionManagerImpl.java | 2 +- .../flowci/core/job/service/StepService.java | 2 +- .../core/job/service/StepServiceImpl.java | 21 ++++++++++--------- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/Step.java b/core/src/main/java/com/flowci/core/job/domain/Step.java index a704a337f..24a0ed1c8 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Step.java +++ b/core/src/main/java/com/flowci/core/job/domain/Step.java @@ -115,6 +115,12 @@ public enum Type { */ private Vars output = new StringVars(); + /** + * Context of all previous steps context + */ + @JsonIgnore + private Vars context; + /** * Cmd start at timestamp */ diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index 19f4d247a..f0b6c3cf2 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -450,7 +450,7 @@ public void accept(JobSmContext context) throws Exception { Job job = context.job; Step step = context.step; - stepService.resultUpdate(step); + stepService.resultUpdate(job, step); updateJobContextAndLatestStatus(job, step); log.debug("Step {} been recorded", step.getNodePath()); diff --git a/core/src/main/java/com/flowci/core/job/service/StepService.java b/core/src/main/java/com/flowci/core/job/service/StepService.java index b07c8a07e..79caa64bf 100644 --- a/core/src/main/java/com/flowci/core/job/service/StepService.java +++ b/core/src/main/java/com/flowci/core/job/service/StepService.java @@ -74,7 +74,7 @@ public interface StepService { /** * To update properties are related with cmd executed result */ - void resultUpdate(Step result); + void resultUpdate(Job job, Step stepFromAgent); /** * Delete steps by flow id diff --git a/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java index f6dd29ed3..3dec5ead4 100644 --- a/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java @@ -175,20 +175,21 @@ public Step toStatus(Step entity, Executed.Status status, String err, boolean al } @Override - public void resultUpdate(Step cmd) { - checkNotNull(cmd.getId()); - Step entity = get(cmd.getId()); + public void resultUpdate(Job job, Step stepFromAgent) { + checkNotNull(stepFromAgent.getId()); + Step entity = get(stepFromAgent.getId()); // only update properties should from agent - entity.setProcessId(cmd.getProcessId()); - entity.setCode(cmd.getCode()); - entity.setStartAt(cmd.getStartAt()); - entity.setFinishAt(cmd.getFinishAt()); - entity.setLogSize(cmd.getLogSize()); - entity.setOutput(cmd.getOutput()); + entity.setProcessId(stepFromAgent.getProcessId()); + entity.setCode(stepFromAgent.getCode()); + entity.setStartAt(stepFromAgent.getStartAt()); + entity.setFinishAt(stepFromAgent.getFinishAt()); + entity.setLogSize(stepFromAgent.getLogSize()); + entity.setOutput(stepFromAgent.getOutput()); + entity.setContext(job.getContext()); // change status and save - toStatus(entity, cmd.getStatus(), cmd.getError(), false); + toStatus(entity, stepFromAgent.getStatus(), stepFromAgent.getError(), false); } @Override From 7cce5e6cac4e028ee8c90ac61e969b84f30a934e Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Wed, 17 Mar 2021 22:50:06 +0100 Subject: [PATCH 15/46] support restart failure step in controller layer --- .../core/job/controller/JobController.java | 6 +++++ .../java/com/flowci/core/job/domain/Job.java | 5 ++++ .../com/flowci/core/job/domain/RerunJob.java | 2 ++ .../java/com/flowci/core/job/domain/Step.java | 6 ----- .../job/manager/JobActionManagerImpl.java | 2 +- .../flowci/core/job/service/JobService.java | 10 ++++---- .../core/job/service/JobServiceImpl.java | 24 +++++++++++++++++++ .../flowci/core/job/service/StepService.java | 2 +- .../core/job/service/StepServiceImpl.java | 3 +-- 9 files changed, 46 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/controller/JobController.java b/core/src/main/java/com/flowci/core/job/controller/JobController.java index aa93a6ee4..d01af78a6 100644 --- a/core/src/main/java/com/flowci/core/job/controller/JobController.java +++ b/core/src/main/java/com/flowci/core/job/controller/JobController.java @@ -139,6 +139,12 @@ public void createAndStart(@Validated @RequestBody CreateJob body) { public void rerun(@Validated @RequestBody RerunJob body) { Job job = jobService.get(body.getJobId()); Flow flow = flowService.getById(job.getFlowId()); + + if (body.isFromFailureStep()) { + jobService.rerunFromFailureStep(flow, job); + return; + } + jobService.rerun(flow, job); } diff --git a/core/src/main/java/com/flowci/core/job/domain/Job.java b/core/src/main/java/com/flowci/core/job/domain/Job.java index 2c8bff6c2..029a6272b 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Job.java +++ b/core/src/main/java/com/flowci/core/job/domain/Job.java @@ -272,6 +272,11 @@ public boolean isDone() { return FINISH_STATUS.contains(status); } + @JsonIgnore + public boolean isFailure() { + return status == Status.FAILURE; + } + @JsonIgnore public String getQueueName() { return "flow.q." + flowId + ".job"; diff --git a/core/src/main/java/com/flowci/core/job/domain/RerunJob.java b/core/src/main/java/com/flowci/core/job/domain/RerunJob.java index 924090f51..9bbed481d 100644 --- a/core/src/main/java/com/flowci/core/job/domain/RerunJob.java +++ b/core/src/main/java/com/flowci/core/job/domain/RerunJob.java @@ -25,4 +25,6 @@ public class RerunJob { @NotEmpty private String jobId; + + private boolean fromFailureStep; } diff --git a/core/src/main/java/com/flowci/core/job/domain/Step.java b/core/src/main/java/com/flowci/core/job/domain/Step.java index 24a0ed1c8..a704a337f 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Step.java +++ b/core/src/main/java/com/flowci/core/job/domain/Step.java @@ -115,12 +115,6 @@ public enum Type { */ private Vars output = new StringVars(); - /** - * Context of all previous steps context - */ - @JsonIgnore - private Vars context; - /** * Cmd start at timestamp */ diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index f0b6c3cf2..19f4d247a 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -450,7 +450,7 @@ public void accept(JobSmContext context) throws Exception { Job job = context.job; Step step = context.step; - stepService.resultUpdate(job, step); + stepService.resultUpdate(step); updateJobContextAndLatestStatus(job, step); log.debug("Step {} been recorded", step.getNodePath()); diff --git a/core/src/main/java/com/flowci/core/job/service/JobService.java b/core/src/main/java/com/flowci/core/job/service/JobService.java index a0c22ceb0..367df7f88 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobService.java +++ b/core/src/main/java/com/flowci/core/job/service/JobService.java @@ -17,11 +17,8 @@ package com.flowci.core.job.service; import com.flowci.core.flow.domain.Flow; -import com.flowci.core.job.domain.Job; +import com.flowci.core.job.domain.*; import com.flowci.core.job.domain.Job.Trigger; -import com.flowci.core.job.domain.JobDesc; -import com.flowci.core.job.domain.JobItem; -import com.flowci.core.job.domain.JobYml; import com.flowci.domain.StringVars; import org.springframework.data.domain.Page; @@ -88,6 +85,11 @@ public interface JobService { */ Job rerun(Flow flow, Job job); + /** + * Restart job from failure step + */ + Job rerunFromFailureStep(Flow flow, Job job); + /** * Delete all jobs of the flow within an executor */ diff --git a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java index 5ada4c2af..622d9c780 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java @@ -243,6 +243,30 @@ public Job rerun(Flow flow, Job job) { return job; } + @Override + public Job rerunFromFailureStep(Flow flow, Job job) { + if (!job.isDone()) { + throw new StatusException("Job not finished, cannot re-start"); + } + + if (!job.isFailure()) { + throw new StatusException("Job is not failure status, cannot re-start from failure step"); + } + + // reset job properties + job.setFinishAt(null); + job.setStartAt(null); + job.setSnapshots(Maps.newHashMap()); + job.setStatus(Job.Status.PENDING); + job.setTrigger(Trigger.MANUAL); + job.setCreatedBy(sessionManager.getUserEmail()); + + // TODO: + + + return null; + } + @Override public void delete(Flow flow) { appTaskExecutor.execute(() -> { diff --git a/core/src/main/java/com/flowci/core/job/service/StepService.java b/core/src/main/java/com/flowci/core/job/service/StepService.java index 79caa64bf..e4df688bf 100644 --- a/core/src/main/java/com/flowci/core/job/service/StepService.java +++ b/core/src/main/java/com/flowci/core/job/service/StepService.java @@ -74,7 +74,7 @@ public interface StepService { /** * To update properties are related with cmd executed result */ - void resultUpdate(Job job, Step stepFromAgent); + void resultUpdate(Step stepFromAgent); /** * Delete steps by flow id diff --git a/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java index 3dec5ead4..e4c9f6a5d 100644 --- a/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java @@ -175,7 +175,7 @@ public Step toStatus(Step entity, Executed.Status status, String err, boolean al } @Override - public void resultUpdate(Job job, Step stepFromAgent) { + public void resultUpdate(Step stepFromAgent) { checkNotNull(stepFromAgent.getId()); Step entity = get(stepFromAgent.getId()); @@ -186,7 +186,6 @@ public void resultUpdate(Job job, Step stepFromAgent) { entity.setFinishAt(stepFromAgent.getFinishAt()); entity.setLogSize(stepFromAgent.getLogSize()); entity.setOutput(stepFromAgent.getOutput()); - entity.setContext(job.getContext()); // change status and save toStatus(entity, stepFromAgent.getStatus(), stepFromAgent.getError(), false); From 52aeeeb2bb7e1ba32f2a3a7e1b4746d3ac9f2b40 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Thu, 18 Mar 2021 14:07:41 +0100 Subject: [PATCH 16/46] start job from steps in the job --- .../core/job/manager/JobActionManagerImpl.java | 14 +++++++++++++- .../flowci/core/job/service/JobServiceImpl.java | 6 ++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index 19f4d247a..9f6df2ba5 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -409,7 +409,19 @@ public void accept(JobSmContext context) throws Exception { // start from root path, and block current thread since don't send ack back to queue NodeTree tree = ymlManager.getTree(job); - executeJob(job, Lists.newArrayList(tree.getRoot())); + + // start job from root or from current path + List stepsToStart = Lists.newLinkedList(); + for (String p : job.getCurrentPath()) { + stepsToStart.add(tree.get(p)); + } + + if (stepsToStart.isEmpty()) { + stepsToStart.add(tree.getRoot()); + } + + logInfo(job, "QueuedToRunning: start from nodes " + stepsToStart.toString()); + executeJob(job, stepsToStart); } @Override diff --git a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java index 622d9c780..92af74281 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java @@ -261,10 +261,8 @@ public Job rerunFromFailureStep(Flow flow, Job job) { job.setTrigger(Trigger.MANUAL); job.setCreatedBy(sessionManager.getUserEmail()); - // TODO: - - - return null; + jobActionManager.toStart(job); + return job; } @Override From 53ba9a5b4072740b49621812922b7917f9935a5f Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Thu, 18 Mar 2021 16:42:50 +0100 Subject: [PATCH 17/46] reset status and expire on rerun failure step --- .../com/flowci/core/job/manager/JobActionManagerImpl.java | 7 +------ .../java/com/flowci/core/job/service/JobServiceImpl.java | 3 ++- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index 9f6df2ba5..fc5f0d096 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -407,19 +407,14 @@ public void accept(JobSmContext context) throws Exception { job.setStartAt(new Date()); setJobStatusAndSave(job, Job.Status.RUNNING, null); - // start from root path, and block current thread since don't send ack back to queue NodeTree tree = ymlManager.getTree(job); - // start job from root or from current path + // start job from job's current path List stepsToStart = Lists.newLinkedList(); for (String p : job.getCurrentPath()) { stepsToStart.add(tree.get(p)); } - if (stepsToStart.isEmpty()) { - stepsToStart.add(tree.getRoot()); - } - logInfo(job, "QueuedToRunning: start from nodes " + stepsToStart.toString()); executeJob(job, stepsToStart); } diff --git a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java index 92af74281..5dfacf3e9 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java @@ -256,8 +256,9 @@ public Job rerunFromFailureStep(Flow flow, Job job) { // reset job properties job.setFinishAt(null); job.setStartAt(null); + job.setExpire(flow.getStepTimeout()); job.setSnapshots(Maps.newHashMap()); - job.setStatus(Job.Status.PENDING); + job.setStatus(Job.Status.CREATED); job.setTrigger(Trigger.MANUAL); job.setCreatedBy(sessionManager.getUserEmail()); From d8e1c68377c93d3962fa113717eb0ad0cc888e65 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Thu, 18 Mar 2021 22:42:16 +0100 Subject: [PATCH 18/46] reset job agent on rerun --- .../java/com/flowci/core/job/domain/Job.java | 25 +++++--------- .../job/manager/JobActionManagerImpl.java | 33 ++++++++++++++----- .../core/job/service/JobServiceImpl.java | 8 ++++- .../flowci/core/test/job/JobServiceTest.java | 2 +- 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/Job.java b/core/src/main/java/com/flowci/core/job/domain/Job.java index 029a6272b..c5ce0d0bf 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Job.java +++ b/core/src/main/java/com/flowci/core/job/domain/Job.java @@ -24,9 +24,9 @@ import com.flowci.domain.StringVars; import com.flowci.domain.Vars; import com.flowci.store.Pathable; -import com.flowci.tree.Node; import com.flowci.util.StringHelper; import com.google.common.collect.ImmutableSet; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.springframework.data.mongodb.core.index.CompoundIndex; @@ -44,6 +44,7 @@ @Getter @Setter @Document(collection = "job") +@EqualsAndHashCode(callSuper = true) @CompoundIndex( name = "index_job_flowid_and_buildnum", def = "{'flowId': 1, 'buildNumber': 1}", @@ -212,6 +213,7 @@ public static Pathable path(Long buildNumber) { // agent id : info private Map snapshots = new HashMap<>(); + // current running steps private Set currentPath = new HashSet<>(); private Vars context = new StringVars(); @@ -349,25 +351,16 @@ public void setErrorToContext(String err) { context.put(Variables.Job.Error, err); } - public void setCurrentPathFromNodes(List nodes) { + public Job resetCurrentPath() { this.currentPath.clear(); - for (Node n : nodes) { - this.currentPath.add(n.getPathAsString()); - } - } - - public void setCurrentPathFromNodes(Node node) { - this.currentPath.clear(); - this.currentPath.add(node.getPathAsString()); + return this; } - @Override - public boolean equals(Object o) { - return super.equals(o); + public void addToCurrentPath(Step step) { + this.currentPath.add(step.getNodePath()); } - @Override - public int hashCode() { - return super.hashCode(); + public void removeFromCurrentPath(Step step) { + this.currentPath.remove(step.getNodePath()); } } diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index fc5f0d096..d4eed4230 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -215,6 +215,12 @@ public void onIdleAgent(IdleAgentEvent event) { } try { + Agent agent = agentService.get(agentId); + if (agent.isBusy()) { + event.setFetched(false); + return; + } + NodeTree tree = ymlManager.getTree(job); boolean isAssigned = assignAgentToWaitingStep(agentId, job, tree, true); @@ -409,14 +415,18 @@ public void accept(JobSmContext context) throws Exception { NodeTree tree = ymlManager.getTree(job); - // start job from job's current path - List stepsToStart = Lists.newLinkedList(); + // start job from job's current step path + List nodesToStart = Lists.newLinkedList(); for (String p : job.getCurrentPath()) { - stepsToStart.add(tree.get(p)); + nodesToStart.add(tree.get(p)); } - logInfo(job, "QueuedToRunning: start from nodes " + stepsToStart.toString()); - executeJob(job, stepsToStart); + if (nodesToStart.isEmpty()) { + nodesToStart.add(tree.getRoot()); + } + + logInfo(job, "QueuedToRunning: start from nodes " + nodesToStart.toString()); + executeJob(job, nodesToStart); } @Override @@ -467,6 +477,7 @@ public void accept(JobSmContext context) throws Exception { } setJobStatusAndSave(job, Job.Status.RUNNING, null); + NodeTree tree = ymlManager.getTree(job); Node node = tree.get(step.getNodePath()); @@ -780,8 +791,6 @@ private void setupJobYamlAndSteps(Job job, String yml) { localTaskService.init(job); FlowNode root = ymlManager.parse(yml); - - job.setCurrentPathFromNodes(root); job.getContext().merge(root.getEnvironments(), false); } @@ -919,7 +928,6 @@ private Set getStepsStatus(Job job, Collection nodes) { } private void executeJob(Job job, List nodes) throws ScriptException { - job.setCurrentPathFromNodes(nodes); setJobStatusAndSave(job, job.getStatus(), null); NodeTree tree = ymlManager.getTree(job); @@ -943,6 +951,10 @@ private void executeJob(Job job, List nodes) throws ScriptException { continue; } + // add dispatchable step + job.addToCurrentPath(step); + jobDao.save(job); + Optional optionalFromJob = fetchAgentFromJob(job, node); if (optionalFromJob.isPresent()) { dispatch(job, node, step, optionalFromJob.get()); @@ -1028,6 +1040,11 @@ private void setJobStatusAndSave(Job job, Job.Status newStatus, String message) private void updateJobContextAndLatestStatus(Job job, Step step) { job.setFinishAt(step.getFinishAt()); + // remove step path if success, keep failure for rerun later + if (step.isSuccess()) { + job.removeFromCurrentPath(step); + } + // merge output to job context Vars context = job.getContext(); context.merge(step.getOutput()); diff --git a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java index 5dfacf3e9..997a3669f 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java @@ -220,7 +220,7 @@ public Job rerun(Flow flow, Job job) { job.setSnapshots(Maps.newHashMap()); job.setStatus(Job.Status.PENDING); job.setTrigger(Trigger.MANUAL); - job.setCurrentPathFromNodes(root); + job.resetCurrentPath(); job.setCreatedBy(sessionManager.getUserEmail()); // re-init job context @@ -233,6 +233,9 @@ public Job rerun(Flow flow, Job job) { context.put(Variables.Job.TriggerBy, sessionManager.get().getEmail()); context.merge(root.getEnvironments(), false); + // reset job agent + jobAgentDao.save(new JobAgent(job.getId(), flow.getId())); + // cleanup stepService.delete(job); localTaskService.delete(job); @@ -262,6 +265,9 @@ public Job rerunFromFailureStep(Flow flow, Job job) { job.setTrigger(Trigger.MANUAL); job.setCreatedBy(sessionManager.getUserEmail()); + // reset job agent + jobAgentDao.save(new JobAgent(job.getId(), flow.getId())); + jobActionManager.toStart(job); return job; } diff --git a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java index ec4d89127..ae9f6f7f1 100644 --- a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java +++ b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java @@ -538,7 +538,7 @@ private Job prepareJobForRunningStatus(Agent agent) { jobAgentDao.addFlowToAgent(job.getId(), agent.getId(), tree.getRoot().getPathAsString()); - job.setCurrentPathFromNodes(firstNode); + job.resetCurrentPath().getCurrentPath().add(firstNode.getPathAsString()); job.setStatus(Status.RUNNING); job.setStatusToContext(Status.RUNNING); From 53f84cbe3d4fe3c7a084703efa971ab5ba2ac561 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Fri, 19 Mar 2021 17:42:54 +0100 Subject: [PATCH 19/46] check agent status while recive from queue, and set max priority on rerun job --- .../com/flowci/core/agent/service/AgentServiceImpl.java | 6 ++++++ core/src/main/java/com/flowci/core/job/domain/Job.java | 4 ++-- .../com/flowci/core/job/manager/JobActionManagerImpl.java | 6 ------ .../java/com/flowci/core/job/service/JobServiceImpl.java | 2 ++ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java index 5bb7ca45a..a53401d86 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java @@ -127,6 +127,12 @@ public void subscribeIdleAgentQueue() throws IOException { String agentId = new String(body); log.debug("Got an idle agent {}", agentId); + Agent agent = get(agentId); + if (!agent.isIdle()) { + log.debug("Agent {} is not idle", agentId); + return true; + } + try { IdleAgentEvent event = new IdleAgentEvent(this, agentId); eventManager.publish(event); diff --git a/core/src/main/java/com/flowci/core/job/domain/Job.java b/core/src/main/java/com/flowci/core/job/domain/Job.java index c5ce0d0bf..b85b72b8b 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Job.java +++ b/core/src/main/java/com/flowci/core/job/domain/Job.java @@ -189,9 +189,9 @@ public static Pathable path(Long buildNumber) { private static final SimpleDateFormat DateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); - private static final Integer MinPriority = 1; + public static final Integer MinPriority = 1; - private static final Integer MaxPriority = 255; + public static final Integer MaxPriority = 255; /** * Job key is generated from {flow id}-{build number} diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index d4eed4230..02dde06b9 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -215,12 +215,6 @@ public void onIdleAgent(IdleAgentEvent event) { } try { - Agent agent = agentService.get(agentId); - if (agent.isBusy()) { - event.setFetched(false); - return; - } - NodeTree tree = ymlManager.getTree(job); boolean isAssigned = assignAgentToWaitingStep(agentId, job, tree, true); diff --git a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java index 997a3669f..a57b20a5c 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java @@ -221,6 +221,7 @@ public Job rerun(Flow flow, Job job) { job.setStatus(Job.Status.PENDING); job.setTrigger(Trigger.MANUAL); job.resetCurrentPath(); + job.setPriority(Job.MaxPriority); job.setCreatedBy(sessionManager.getUserEmail()); // re-init job context @@ -261,6 +262,7 @@ public Job rerunFromFailureStep(Flow flow, Job job) { job.setStartAt(null); job.setExpire(flow.getStepTimeout()); job.setSnapshots(Maps.newHashMap()); + job.setPriority(Job.MaxPriority); job.setStatus(Job.Status.CREATED); job.setTrigger(Trigger.MANUAL); job.setCreatedBy(sessionManager.getUserEmail()); From cafbdccfa88e6a32ffec3fe92c84ed56cd7faed6 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Fri, 19 Mar 2021 22:22:48 +0100 Subject: [PATCH 20/46] record agent connect time --- core/src/main/java/com/flowci/core/agent/domain/Agent.java | 2 ++ .../java/com/flowci/core/agent/service/AgentServiceImpl.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/core/src/main/java/com/flowci/core/agent/domain/Agent.java b/core/src/main/java/com/flowci/core/agent/domain/Agent.java index 4a6312243..6680c34ce 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/Agent.java +++ b/core/src/main/java/com/flowci/core/agent/domain/Agent.java @@ -90,6 +90,8 @@ public static Status fromBytes(byte[] bytes) { private Instant statusUpdatedAt; + private Instant connectedAt; + private String jobId; private String containerId; // for started from host diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java index a53401d86..5479fc4cf 100644 --- a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java +++ b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java @@ -48,6 +48,7 @@ import org.springframework.stereotype.Service; import java.io.IOException; +import java.time.Instant; import java.util.*; import static com.flowci.core.agent.domain.Agent.Status.*; @@ -399,6 +400,7 @@ public void onConnected(OnConnectedEvent event) { target.setK8sCluster(init.getK8sCluster()); target.setUrl("http://" + init.getIp() + ":" + init.getPort()); target.setOs(init.getOs()); + target.setConnectedAt(Instant.now()); update(target, init.getStatus()); From 44164026a3651185a0579e8e7bf0f3335a95079d Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sat, 20 Mar 2021 14:00:59 +0100 Subject: [PATCH 21/46] fix ut failure due to job current path --- .../test/java/com/flowci/core/test/job/JobServiceTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java index ae9f6f7f1..7f8065259 100644 --- a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java +++ b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java @@ -202,7 +202,7 @@ public void should_start_new_job() throws Throwable { NodeTree tree = ymlManager.getTree(job); Assert.assertEquals(Status.CREATED, job.getStatus()); - Assert.assertTrue(job.getCurrentPath().contains(tree.getRoot().getPathAsString())); + Assert.assertTrue(job.getCurrentPath().isEmpty()); jobActionManager.toStart(job); Assert.assertEquals(Status.QUEUED, job.getStatus()); @@ -518,7 +518,7 @@ public void should_rerun_job() { Assert.assertNotNull(context.get(Variables.Job.TriggerBy)); // then: verify job properties - Assert.assertTrue(job.getCurrentPath().contains(FlowNode.DEFAULT_ROOT_NAME)); + Assert.assertTrue(job.getCurrentPath().isEmpty()); Assert.assertFalse(job.isExpired()); Assert.assertNotNull(job.getCreatedAt()); Assert.assertNotNull(job.getCreatedBy()); From 296e387ba603fc430dee58262de0ffc57e2a94b2 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 21 Mar 2021 16:12:34 +0100 Subject: [PATCH 22/46] load post steps to node --- .../main/java/com/flowci/tree/NodeTree.java | 9 ++ .../java/com/flowci/tree/RegularStepNode.java | 5 + .../java/com/flowci/tree/yml/FlowYml.java | 7 +- .../java/com/flowci/tree/yml/StepYml.java | 3 +- .../java/com/flowci/tree/yml/YmlBase.java | 35 ++++--- tree/src/test/resources/flow-with-post.yml | 96 +++++++++++++++++++ 6 files changed, 134 insertions(+), 21 deletions(-) create mode 100644 tree/src/test/resources/flow-with-post.yml diff --git a/tree/src/main/java/com/flowci/tree/NodeTree.java b/tree/src/main/java/com/flowci/tree/NodeTree.java index d5da99a21..3cf656d24 100644 --- a/tree/src/main/java/com/flowci/tree/NodeTree.java +++ b/tree/src/main/java/com/flowci/tree/NodeTree.java @@ -67,6 +67,7 @@ public int numOfNode() { /** * Get all last steps + * * @return */ public Collection ends() { @@ -212,4 +213,12 @@ private void buildEndNodes() { } }); } + + private static boolean isPostStep(Node n) { + if (n instanceof RegularStepNode) { + RegularStepNode r = (RegularStepNode) n; + return r.isPost(); + } + return false; + } } diff --git a/tree/src/main/java/com/flowci/tree/RegularStepNode.java b/tree/src/main/java/com/flowci/tree/RegularStepNode.java index 23c14cdfc..b7dc03326 100644 --- a/tree/src/main/java/com/flowci/tree/RegularStepNode.java +++ b/tree/src/main/java/com/flowci/tree/RegularStepNode.java @@ -17,6 +17,11 @@ public final class RegularStepNode extends Node { public static final boolean ALLOW_FAILURE_DEFAULT = false; + /** + * Indicate the step is post step + */ + private boolean post; + /** * bash script */ diff --git a/tree/src/main/java/com/flowci/tree/yml/FlowYml.java b/tree/src/main/java/com/flowci/tree/yml/FlowYml.java index 7e1dd080b..8f98d1ecd 100644 --- a/tree/src/main/java/com/flowci/tree/yml/FlowYml.java +++ b/tree/src/main/java/com/flowci/tree/yml/FlowYml.java @@ -43,6 +43,9 @@ public class FlowYml extends YmlBase { private List notifications = new LinkedList<>(); + // post steps + private List post = new LinkedList<>(); + public FlowNode toNode(Node parent) { if (!NodePath.validate(name)) { throw new YmlException("Invalid name {0}", name); @@ -60,7 +63,9 @@ public FlowNode toNode(Node parent) { throw new YmlException("The 'steps' section must be defined"); } - setStepsToNode(node); + Set uniqueNames = new HashSet<>(steps.size() + post.size()); + setStepsToParent(node, steps, false, uniqueNames); + setStepsToParent(node, post, true, uniqueNames); return node; } diff --git a/tree/src/main/java/com/flowci/tree/yml/StepYml.java b/tree/src/main/java/com/flowci/tree/yml/StepYml.java index fce9c38ec..5d8ce4e7a 100644 --- a/tree/src/main/java/com/flowci/tree/yml/StepYml.java +++ b/tree/src/main/java/com/flowci/tree/yml/StepYml.java @@ -123,8 +123,7 @@ public Node toNode(Node parent, int index) { if (step.hasPlugin()) { throw new YmlException("The plugin section is not allowed on the step with sub steps"); } - - setStepsToNode(step); + setStepsToParent(step, steps, false, new HashSet<>(steps.size())); } // backward compatible, set script to bash diff --git a/tree/src/main/java/com/flowci/tree/yml/YmlBase.java b/tree/src/main/java/com/flowci/tree/yml/YmlBase.java index 22bf208fe..361a4717d 100644 --- a/tree/src/main/java/com/flowci/tree/yml/YmlBase.java +++ b/tree/src/main/java/com/flowci/tree/yml/YmlBase.java @@ -60,24 +60,6 @@ protected StringVars getVariableMap() { return variables; } - protected void setStepsToNode(T parent) { - int index = 1; - Set uniqueName = new HashSet<>(steps.size()); - - for (StepYml child : steps) { - Node step = child.toNode(parent, index++); - - if (step instanceof RegularStepNode) { - String stepName = step.getName(); - if (!uniqueName.add(stepName)) { - throw new YmlException("Duplicate name {0} in step", stepName); - } - } - - parent.getChildren().add(step); - } - } - void setDockerToNode(T node) { if (hasDocker() && hasDockers()) { throw new YmlException("Only accept either 'docker' or 'dockers' section"); @@ -129,4 +111,21 @@ private boolean hasDocker() { private boolean hasDockers() { return dockers != null && dockers.size() > 0; } + + protected static void setStepsToParent(T parent, List steps, boolean post, Set nameSet) { + int index = 1; + for (StepYml child : steps) { + Node step = child.toNode(parent, index++); + + if (step instanceof RegularStepNode) { + String stepName = step.getName(); + if (!nameSet.add(stepName)) { + throw new YmlException("Duplicate name {0} in step", stepName); + } + ((RegularStepNode) step).setPost(post); + } + + parent.getChildren().add(step); + } + } } diff --git a/tree/src/test/resources/flow-with-post.yml b/tree/src/test/resources/flow-with-post.yml new file mode 100644 index 000000000..45262e08e --- /dev/null +++ b/tree/src/test/resources/flow-with-post.yml @@ -0,0 +1,96 @@ +name: root +envs: + FLOW_WORKSPACE: "echo hello" + FLOW_VERSION: "echo version" + +condition: | + return $FLOWCI_GIT_BRANCH == "develop" || $FLOWCI_GIT_BRANCH == "master"; + +docker: + image: "helloworld:0.1" + +selector: + label: + - ios + - local + +notifications: + - plugin: 'email-notify' + envs: + FLOWCI_SMTP_CONFIG: 'test-config' + +steps: + - parallel: + subflow-A: + selector: + label: [ "linux" ] + steps: + - name: A + plugin: 'A-plugin' + script: | + echo "A" + - name: B + condition: | + return true + script: | + echo "B" + post: + - name: subA-post-1 + bash: "echo sub A post-1" + + subflow-B: + selector: + label: [ "linux" ] + steps: + - name: A + plugin: 'B-plugin' + script: | + echo "A" + + - condition: | + println(FLOW_WORKSPACE) + true + name: step2 + envs: + FLOW_WORKSPACE: "echo step" + FLOW_VERSION: "echo step version" + cache: + key: mycache + paths: + - "./" + - "vendor" + timeout: 3600 + allow_failure: true + + + - name: step3 + steps: + - name: step-3-1 + bash: | + echo "step-3-1" + + - name: step-3-2 + bash: | + echo "step-3-2" + + - name: step4 + allow_failure: false + docker: + image: "ubuntu:18.04" + ports: + - "6400:6400" + - "2700:2700" + entrypoint: [ "/bin/sh" ] + network: host + bash: "echo 2" + pwsh: "echo powershell" + + +post: + - name: post-1 + bash: "echo post-1" + + - name: post-2 + condition: | + return ${FLOWCI_STATUS} == "SUCCESS" + bash: "echo post-2" \ No newline at end of file From 086ebce5e01ad38c689bd28b58fa4dd4af376ce7 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 21 Mar 2021 17:57:31 +0100 Subject: [PATCH 23/46] add post tag in step --- .../main/java/com/flowci/core/job/domain/Step.java | 2 ++ .../core/job/manager/JobActionManagerImpl.java | 12 +++++++----- .../com/flowci/core/job/service/StepServiceImpl.java | 1 + tree/src/main/java/com/flowci/tree/Node.java | 7 ------- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/Step.java b/core/src/main/java/com/flowci/core/job/domain/Step.java index a704a337f..066531d5d 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Step.java +++ b/core/src/main/java/com/flowci/core/job/domain/Step.java @@ -80,6 +80,8 @@ public enum Type { private boolean allowFailure; + private boolean post; + private String plugin; private List dockers; diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index 02dde06b9..e57ef9a86 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -477,15 +477,13 @@ public void accept(JobSmContext context) throws Exception { if (node.isLastChildOfParent()) { String agentId = releaseAgentFromJob(context.job, node, step); - if (assignAgentToWaitingStep(agentId, job, tree, false)) { return; } - releaseAgentToPool(context.job, node, step); } - if (toNextStep(context.job, context.step)) { + if (toNextStep(tree, context.job, context.step)) { return; } @@ -871,8 +869,7 @@ private SimpleSecret getSimpleSecret(String credentialName) { * * @return true if next step dispatched or have to wait for previous steps, false if no more steps or failure */ - private boolean toNextStep(Job job, Step step) throws ScriptException { - NodeTree tree = ymlManager.getTree(job); + private boolean toNextStep(NodeTree tree, Job job, Step step) throws ScriptException { Node node = tree.get(NodePath.create(step.getNodePath())); // current node List next = node.getNext(); @@ -1047,6 +1044,11 @@ private void updateJobContextAndLatestStatus(Job job, Step step) { context.put(Variables.Job.FinishAt, job.finishAtInStr()); context.put(Variables.Job.Steps, stepService.toVarString(job, step)); + // DO NOT update job status from post step + if (step.isPost()) { + return; + } + // DO NOT update job status from context job.setStatusToContext(StatusHelper.convert(step)); job.setErrorToContext(step.getError()); diff --git a/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java index e4c9f6a5d..8a06222d8 100644 --- a/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java @@ -268,6 +268,7 @@ private static Step newInstance(Job job, Node node) { step.setAllowFailure(r.isAllowFailure()); step.setPlugin(r.getPlugin()); step.setType(Step.Type.STEP); + step.setPost(r.isPost()); if (r.hasChildren()) { step.setType(Step.Type.STAGE); diff --git a/tree/src/main/java/com/flowci/tree/Node.java b/tree/src/main/java/com/flowci/tree/Node.java index 8223a8042..516758e8b 100644 --- a/tree/src/main/java/com/flowci/tree/Node.java +++ b/tree/src/main/java/com/flowci/tree/Node.java @@ -182,13 +182,6 @@ public List fetchDockerOptions() { return wrapper.getValue(); } - public void forEachNext(Node node, Consumer onNext) { - for (Node next : node.next) { - onNext.accept(next); - forEachNext(next, onNext); - } - } - public void forEachChildren(Consumer onChild) { forEachChildren(this, onChild); } From fbb7ce0edda7459f79d58698a50a7934ab60abe9 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 21 Mar 2021 22:40:12 +0100 Subject: [PATCH 24/46] fix context changed but hook action still call --- .../java/com/flowci/core/job/domain/Job.java | 5 ++ .../job/manager/JobActionManagerImpl.java | 13 ++++- sm/pom.xml | 8 +++ .../main/java/com/flowci/sm/StateMachine.java | 4 ++ .../com/flowci/sm/test/StateMachineTest.java | 50 +++++++++++++++++++ 5 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 sm/src/test/java/com/flowci/sm/test/StateMachineTest.java diff --git a/core/src/main/java/com/flowci/core/job/domain/Job.java b/core/src/main/java/com/flowci/core/job/domain/Job.java index b85b72b8b..26d2b31a4 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Job.java +++ b/core/src/main/java/com/flowci/core/job/domain/Job.java @@ -117,6 +117,11 @@ public enum Status { */ RUNNING(4), + /** + * Running Post steps, only happened before failure status + */ + RUNNING_POST(4), + /** * Job will be cancelled, but waiting for response from agent */ diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index e57ef9a86..8a8e032f8 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -80,6 +80,7 @@ public class JobActionManagerImpl implements JobActionManager { private static final Status Cancelling = new Status(Job.Status.CANCELLING.name()); private static final Status Queued = new Status(Job.Status.QUEUED.name()); private static final Status Running = new Status(Job.Status.RUNNING.name()); + private static final Status RunningPost = new Status(Job.Status.RUNNING_POST.name()); private static final Status Timeout = new Status(Job.Status.TIMEOUT.name()); private static final Status Failure = new Status(Job.Status.FAILURE.name()); private static final Status Success = new Status(Job.Status.SUCCESS.name()); @@ -112,6 +113,12 @@ public class JobActionManagerImpl implements JobActionManager { private static final Transition RunningToTimeout = new Transition(Running, Timeout); private static final Transition RunningToFailure = new Transition(Running, Failure); + // running post + private static final Transition RunningPostToFailure = new Transition(RunningPost, Failure); + private static final Transition RunningPostToCanceled = new Transition(RunningPost, Cancelled); + private static final Transition RunningPostToTimeout = new Transition(RunningPost, Timeout); + + // cancelling private static final Transition CancellingToCancelled = new Transition(Cancelling, Cancelled); @@ -535,6 +542,8 @@ public void accept(JobSmContext context) { stepService.toStatus(step, Step.Status.EXCEPTION, null, false); setOngoingStepsToSkipped(job); logInfo(job, "finished with status {}", Failure); + + // TOOO: Run post steps } @Override @@ -576,6 +585,8 @@ public void accept(JobSmContext context) { } sm.execute(context.getCurrent(), Cancelling, context); + + // TOOO: Run post steps } @Override @@ -789,7 +800,7 @@ private void setupJobYamlAndSteps(Job job, String yml) { private void setOngoingStepsToSkipped(Job job) { List steps = stepService.list(job, Executed.OngoingStatus); for (Step step : steps) { - if (!step.hasAgent()) { + if (!step.hasAgent() || step.isPost()) { continue; } diff --git a/sm/pom.xml b/sm/pom.xml index 0ad86adea..8a14934ed 100644 --- a/sm/pom.xml +++ b/sm/pom.xml @@ -10,4 +10,12 @@ 4.0.0 sm + + + + junit + junit + test + + \ No newline at end of file diff --git a/sm/src/main/java/com/flowci/sm/StateMachine.java b/sm/src/main/java/com/flowci/sm/StateMachine.java index 4f9474f91..865587326 100644 --- a/sm/src/main/java/com/flowci/sm/StateMachine.java +++ b/sm/src/main/java/com/flowci/sm/StateMachine.java @@ -67,6 +67,10 @@ public void execute(Status current, Status target, T context) { try { action.accept(context); + if (context.getTo() != target) { + return; + } + // execute target hook hooksOnTargetStatus.computeIfPresent(target, (status, hooks) -> { for (Consumer hook : hooks) { diff --git a/sm/src/test/java/com/flowci/sm/test/StateMachineTest.java b/sm/src/test/java/com/flowci/sm/test/StateMachineTest.java new file mode 100644 index 000000000..10e087967 --- /dev/null +++ b/sm/src/test/java/com/flowci/sm/test/StateMachineTest.java @@ -0,0 +1,50 @@ +package com.flowci.sm.test; + +import com.flowci.sm.*; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.atomic.AtomicReference; + +public class StateMachineTest { + + private final Status A = new Status("A"); + private final Status B = new Status("B"); + private final Status C = new Status("C"); + + private final Transition AtoB = new Transition(A, B); + private final Transition AtoC = new Transition(A, C); + + private final StateMachine sm = new StateMachine<>("TEST", null); + + private static class TestContext extends Context { + + } + + @Test + public void should_skip_after_event_if_state_changed() { + sm.add(AtoB, new Action() { + @Override + public void accept(TestContext ctx) throws Exception { + // switch to C + sm.execute(A, C, ctx); + } + }); + + AtomicReference shouldExecute = new AtomicReference<>(Boolean.FALSE); + sm.add(AtoC, new Action() { + @Override + public void accept(TestContext ctx) throws Exception { + shouldExecute.set(Boolean.TRUE); + } + }); + + AtomicReference shouldNotExecute = new AtomicReference<>(Boolean.TRUE); + sm.addHookActionOnTargetStatus(testContext -> shouldNotExecute.set(Boolean.FALSE), B); + + sm.execute(A, B, new TestContext()); + + Assert.assertTrue(shouldExecute.get()); + Assert.assertTrue(shouldNotExecute.get()); + } +} From 234a68ddec71b8902c60b24a1a529cc352aca104 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sun, 21 Mar 2021 23:00:30 +0100 Subject: [PATCH 25/46] check whole from to context --- .../main/java/com/flowci/sm/StateMachine.java | 6 +++- .../com/flowci/sm/test/StateMachineTest.java | 34 ++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/sm/src/main/java/com/flowci/sm/StateMachine.java b/sm/src/main/java/com/flowci/sm/StateMachine.java index 865587326..bd85f334e 100644 --- a/sm/src/main/java/com/flowci/sm/StateMachine.java +++ b/sm/src/main/java/com/flowci/sm/StateMachine.java @@ -67,7 +67,7 @@ public void execute(Status current, Status target, T context) { try { action.accept(context); - if (context.getTo() != target) { + if (!isOnSameContext(current, target, context)) { return; } @@ -86,4 +86,8 @@ public void execute(Status current, Status target, T context) { action.onFinally(context); } } + + private boolean isOnSameContext(Status current, Status target, T context) { + return context.getTo() == target && context.getCurrent() == current; + } } diff --git a/sm/src/test/java/com/flowci/sm/test/StateMachineTest.java b/sm/src/test/java/com/flowci/sm/test/StateMachineTest.java index 10e087967..43adce7ea 100644 --- a/sm/src/test/java/com/flowci/sm/test/StateMachineTest.java +++ b/sm/src/test/java/com/flowci/sm/test/StateMachineTest.java @@ -4,6 +4,7 @@ import org.junit.Assert; import org.junit.Test; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; public class StateMachineTest { @@ -14,6 +15,7 @@ public class StateMachineTest { private final Transition AtoB = new Transition(A, B); private final Transition AtoC = new Transition(A, C); + private final Transition BtoC = new Transition(B, C); private final StateMachine sm = new StateMachine<>("TEST", null); @@ -22,7 +24,33 @@ private static class TestContext extends Context { } @Test - public void should_skip_after_event_if_state_changed() { + public void should_call_hook_only_once_when_current_state_changed() { + sm.add(AtoC, new Action() { + @Override + public void accept(TestContext ctx) throws Exception { + sm.execute(B, C, ctx); + } + }); + + AtomicReference shouldExecute = new AtomicReference<>(Boolean.FALSE); + sm.add(BtoC, new Action() { + @Override + public void accept(TestContext ctx) throws Exception { + shouldExecute.set(Boolean.TRUE); + } + }); + + AtomicInteger shouldCallOnceOnly = new AtomicInteger(0); + sm.addHookActionOnTargetStatus(testContext -> shouldCallOnceOnly.getAndIncrement(), C); + + sm.execute(A, C, new TestContext()); + + Assert.assertTrue(shouldExecute.get()); + Assert.assertEquals(1, shouldCallOnceOnly.get()); + } + + @Test + public void should_skip_target_hook_after_event_if_target_state_changed() { sm.add(AtoB, new Action() { @Override public void accept(TestContext ctx) throws Exception { @@ -42,9 +70,13 @@ public void accept(TestContext ctx) throws Exception { AtomicReference shouldNotExecute = new AtomicReference<>(Boolean.TRUE); sm.addHookActionOnTargetStatus(testContext -> shouldNotExecute.set(Boolean.FALSE), B); + AtomicReference shouldExecuteOnHook = new AtomicReference<>(Boolean.FALSE); + sm.addHookActionOnTargetStatus(testContext -> shouldExecuteOnHook.set(Boolean.TRUE), C); + sm.execute(A, B, new TestContext()); Assert.assertTrue(shouldExecute.get()); Assert.assertTrue(shouldNotExecute.get()); + Assert.assertTrue(shouldExecuteOnHook.get()); } } From 82e7c6373708d8298b0919046634aafc1681dcfa Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Tue, 23 Mar 2021 16:26:04 +0100 Subject: [PATCH 26/46] enable to get next post step --- .../com/flowci/core/job/domain/JobAgent.java | 4 +- .../job/manager/JobActionManagerImpl.java | 22 +++--- .../core/job/service/StepServiceImpl.java | 7 +- tree/src/main/java/com/flowci/tree/Node.java | 25 +------ .../main/java/com/flowci/tree/NodeTree.java | 74 +++++++++++++++++-- .../com/flowci/tree/test/YmlParserTest.java | 38 ++++++++++ tree/src/test/resources/flow-with-post.yml | 38 ++++++---- 7 files changed, 154 insertions(+), 54 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/JobAgent.java b/core/src/main/java/com/flowci/core/job/domain/JobAgent.java index ff6a8ca6f..77431dc5e 100644 --- a/core/src/main/java/com/flowci/core/job/domain/JobAgent.java +++ b/core/src/main/java/com/flowci/core/job/domain/JobAgent.java @@ -95,7 +95,7 @@ public void remove(String agentId) { * Get agent if an agent was occupied within same flow */ public Optional getAgent(Node node) { - FlowNode flow = node.getParentFlowNode(); + FlowNode flow = node.getParent(FlowNode.class); for (Map.Entry> entry : agents.entrySet()) { String agentId = entry.getKey(); Set set = entry.getValue(); @@ -112,7 +112,7 @@ public Optional getAgent(Node node) { * find candidate agents within job, but still need to check Selector */ public List getCandidates(Node node) { - FlowNode flow = node.getParentFlowNode(); + FlowNode flow = node.getParent(FlowNode.class); int flowDepth = flow.getPath().depth(); List candidates = new ArrayList<>(agents.size()); diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index 8a8e032f8..1e2eea909 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -112,13 +112,14 @@ public class JobActionManagerImpl implements JobActionManager { private static final Transition RunningToCanceled = new Transition(Running, Cancelled); private static final Transition RunningToTimeout = new Transition(Running, Timeout); private static final Transition RunningToFailure = new Transition(Running, Failure); + private static final Transition RunningToRunningPost = new Transition(Running, RunningPost); // running post + private static final Transition RunningPostToRunningPost = new Transition(RunningPost, RunningPost); private static final Transition RunningPostToFailure = new Transition(RunningPost, Failure); private static final Transition RunningPostToCanceled = new Transition(RunningPost, Cancelled); private static final Transition RunningPostToTimeout = new Transition(RunningPost, Timeout); - // cancelling private static final Transition CancellingToCancelled = new Transition(Cancelling, Cancelled); @@ -543,7 +544,10 @@ public void accept(JobSmContext context) { setOngoingStepsToSkipped(job); logInfo(job, "finished with status {}", Failure); - // TOOO: Run post steps + + if () + + sm.execute(context.getCurrent(), RunningPost, context); } @Override @@ -586,7 +590,7 @@ public void accept(JobSmContext context) { sm.execute(context.getCurrent(), Cancelling, context); - // TOOO: Run post steps + // TODO: Run post steps } @Override @@ -669,7 +673,7 @@ private boolean waitIfJobNotOnTopPriority(Job job) { } private Optional fetchAgentFromJob(Job job, Node node) { - FlowNode flow = node.getParentFlowNode(); + FlowNode flow = node.getParent(FlowNode.class); Selector selector = flow.fetchSelector(); JobAgent agents = getJobAgent(job.getId()); @@ -697,7 +701,7 @@ private Optional fetchAgentFromJob(Job job, Node node) { } private Optional fetchAgentFromPool(Job job, Node node) { - FlowNode flow = node.getParentFlowNode(); + FlowNode flow = node.getParent(FlowNode.class); Selector selector = flow.fetchSelector(); // find agent outside job, blocking thread @@ -726,7 +730,7 @@ private boolean assignAgentToWaitingStep(String agentId, Job job, NodeTree tree, } Node n = tree.get(waitingForAgentStep.getNodePath()); - FlowNode f = n.getParentFlowNode(); + FlowNode f = n.getParent(FlowNode.class); Selector s = f.fetchSelector(); Optional acquired = agentService.acquire(job.getId(), s, agentId, shouldIdle); @@ -747,7 +751,7 @@ private boolean assignAgentToWaitingStep(String agentId, Job job, NodeTree tree, private String releaseAgentFromJob(Job job, Node node, Step step) { String agentId = step.getAgentId(); - FlowNode flow = node.getParentFlowNode(); + FlowNode flow = node.getParent(FlowNode.class); jobAgentDao.removeFlowFromAgent(job.getId(), agentId, flow.getPathAsString()); return agentId; } @@ -761,14 +765,14 @@ private void releaseAgentToPool(Job job, Node node, Step step) { } NodeTree tree = ymlManager.getTree(job); - Selector currentSelector = node.getParentFlowNode().fetchSelector(); + Selector currentSelector = node.getParent(FlowNode.class).fetchSelector(); // find selectors of pending steps List notStartedSteps = stepService.list(job, Executed.WaitingStatus); Set selectors = new HashSet<>(tree.getSelectors().size()); for (Step s : notStartedSteps) { Node n = tree.get(s.getNodePath()); - Selector selector = n.getParentFlowNode().getSelector(); + Selector selector = n.getParent(FlowNode.class).getSelector(); selectors.add(selector); } diff --git a/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java index 8a06222d8..8b9290885 100644 --- a/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java @@ -158,12 +158,13 @@ public Step toStatus(Step entity, Executed.Status status, String err, boolean al NodeTree tree = ymlManager.getTree(entity.getJobId()); NodePath path = NodePath.create(entity.getNodePath()); Node node = tree.get(path); - node.forEachChildren((childNode) -> { - Step childStep = get(entity.getJobId(), childNode.getPathAsString()); + + for (Node child : node.getChildren()) { + Step childStep = get(entity.getJobId(), child.getPathAsString()); childStep.setStartAt(entity.getStartAt()); childStep.setFinishAt(entity.getFinishAt()); saveStatus(childStep, status, err); - }); + } } String jobId = entity.getJobId(); diff --git a/tree/src/main/java/com/flowci/tree/Node.java b/tree/src/main/java/com/flowci/tree/Node.java index 516758e8b..4c4251929 100644 --- a/tree/src/main/java/com/flowci/tree/Node.java +++ b/tree/src/main/java/com/flowci/tree/Node.java @@ -31,7 +31,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Objects; -import java.util.function.Consumer; import java.util.function.Function; /** @@ -123,11 +122,11 @@ public String getEnv(String name) { } @JsonIgnore - public FlowNode getParentFlowNode() { - ObjectWrapper wrapper = new ObjectWrapper<>(); + public T getParent(Class klass) { + ObjectWrapper wrapper = new ObjectWrapper<>(); this.forEachBottomUp(this, (n) -> { - if (n instanceof FlowNode) { - wrapper.setValue((FlowNode) n); + if (klass.isInstance(n)) { + wrapper.setValue((T) n); return false; } return true; @@ -135,11 +134,6 @@ public FlowNode getParentFlowNode() { return wrapper.getValue(); } - @JsonIgnore - public boolean hasParent() { - return parent != null; - } - @JsonIgnore public boolean hasChildren() { return !getChildren().isEmpty(); @@ -182,17 +176,6 @@ public List fetchDockerOptions() { return wrapper.getValue(); } - public void forEachChildren(Consumer onChild) { - forEachChildren(this, onChild); - } - - private void forEachChildren(Node current, Consumer onChild) { - for (Node child : current.getChildren()) { - onChild.accept(child); - forEachChildren(child, onChild); - } - } - protected final void forEachBottomUp(Node node, Function onNode) { Boolean canContinue = onNode.apply(node); if (!canContinue) { diff --git a/tree/src/main/java/com/flowci/tree/NodeTree.java b/tree/src/main/java/com/flowci/tree/NodeTree.java index 3cf656d24..7c8c12fc5 100644 --- a/tree/src/main/java/com/flowci/tree/NodeTree.java +++ b/tree/src/main/java/com/flowci/tree/NodeTree.java @@ -18,6 +18,7 @@ import com.flowci.exception.ArgumentException; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import lombok.Getter; import java.util.*; @@ -113,6 +114,18 @@ public List skip(NodePath current) { return Lists.newArrayList(nextWithSameParent); } + /** + * Find next post step + */ + public Collection post(NodePath current) { + Node n = get(current); + Collection post = new HashSet<>(); + for (Node next : n.next) { + post.addAll(findNextPost(next)); + } + return post; + } + public Node get(NodePath path) { Node node = flatted.get(path); if (node == null) { @@ -214,11 +227,62 @@ private void buildEndNodes() { }); } - private static boolean isPostStep(Node n) { - if (n instanceof RegularStepNode) { - RegularStepNode r = (RegularStepNode) n; - return r.isPost(); + private Collection findNextPost(Node node) { + if (node instanceof ParallelStepNode) { + Collection output = findPostSteps((ParallelStepNode) node); + if (!output.isEmpty()) { + return output; + } + } + + if (node instanceof FlowNode) { + Collection output = findPostSteps((FlowNode) node); + if (!output.isEmpty()) { + return output; + } + } + + if (node instanceof RegularStepNode) { + Collection output = findPostSteps((RegularStepNode) node); + if (!output.isEmpty()) { + return output; + } + } + + Collection post = new HashSet<>(); + for (Node next : node.next) { + post.addAll(findNextPost(next)); + } + return post; + } + + private Collection findPostSteps(ParallelStepNode p) { + Collection post = new HashSet<>(); + for (Node child : p.getChildren()) { + FlowNode f = (FlowNode) child; + post.addAll(findPostSteps(f)); + } + return post; + } + + private Collection findPostSteps(FlowNode f) { + Collection post = new HashSet<>(); + for (Node child : f.getChildren()) { + if (child instanceof RegularStepNode) { + post.addAll(findPostSteps((RegularStepNode) child)); + } + + if (child instanceof ParallelStepNode) { + post.addAll(findPostSteps((ParallelStepNode) child)); + } + } + return post; + } + + private Collection findPostSteps(RegularStepNode r) { + if (r.isPost()) { + return Sets.newHashSet(r); } - return false; + return Collections.emptyList(); } } diff --git a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java index adfd3d1ae..9a4512b02 100644 --- a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java +++ b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java @@ -19,6 +19,7 @@ import com.flowci.domain.DockerOption; import com.flowci.exception.YmlException; import com.flowci.tree.*; +import com.google.common.collect.Lists; import com.google.common.io.Files; import org.junit.Assert; import org.junit.Before; @@ -319,6 +320,43 @@ public void should_load_parallel_step_yaml() throws IOException { Assert.assertEquals(step2Path, tree.skip(subflowB_APath).get(0).getPath()); } + @Test + public void should_load_and_get_post_steps() throws IOException { + content = loadContent("flow-with-post.yml"); + FlowNode root = YmlParser.load(content); + Assert.assertNotNull(root); + + NodeTree tree = NodeTree.create(root); + Assert.assertNotNull(tree); + + NodePath postOfSubA = NodePath.create("flow/parallel-2/subflow-A/subA-post-1"); + NodePath postOfSubC = NodePath.create("flow/parallel-3/subflow-C/Post-C"); + NodePath postOfSubD = NodePath.create("flow/parallel-3/subflow-D/Post-D"); + NodePath post1OfRoot = NodePath.create("flow/post-1"); + NodePath post2OfRoot = NodePath.create("flow/post-2"); + + List nextFromRoot = Lists.newArrayList(tree.post(NodePath.create("flow"))); + Assert.assertEquals(1, nextFromRoot.size()); + Assert.assertEquals(postOfSubA, nextFromRoot.get(0).getPath()); + + List nextFromSubA = Lists.newArrayList(tree.post(postOfSubA)); + Assert.assertEquals(2, nextFromSubA.size()); + Assert.assertEquals(postOfSubC, nextFromSubA.get(0).getPath()); + Assert.assertEquals(postOfSubD, nextFromSubA.get(1).getPath()); + + List nextFromSubC = Lists.newArrayList(tree.post(postOfSubC)); + Assert.assertEquals(1, nextFromSubC.size()); + Assert.assertEquals(post1OfRoot, nextFromSubC.get(0).getPath()); + + List nextFromSubD = Lists.newArrayList(tree.post(postOfSubD)); + Assert.assertEquals(1, nextFromSubD.size()); + Assert.assertEquals(post1OfRoot, nextFromSubD.get(0).getPath()); + + List nextPostFromRootPost1 = Lists.newArrayList(tree.post(post1OfRoot)); + Assert.assertEquals(1, nextPostFromRootPost1.size()); + Assert.assertEquals(post2OfRoot, nextPostFromRootPost1.get(0).getPath()); + } + private String loadContent(String resource) throws IOException { ClassLoader classLoader = YmlParserTest.class.getClassLoader(); URL url = classLoader.getResource(resource); diff --git a/tree/src/test/resources/flow-with-post.yml b/tree/src/test/resources/flow-with-post.yml index 45262e08e..a87e22565 100644 --- a/tree/src/test/resources/flow-with-post.yml +++ b/tree/src/test/resources/flow-with-post.yml @@ -20,6 +20,10 @@ notifications: FLOWCI_SMTP_CONFIG: 'test-config' steps: + - name: clone + bash: | + echo "git clone" + - parallel: subflow-A: selector: @@ -47,21 +51,27 @@ steps: script: | echo "A" - - condition: | - println(FLOW_WORKSPACE) - true - name: step2 - envs: - FLOW_WORKSPACE: "echo step" - FLOW_VERSION: "echo step version" - cache: - key: mycache - paths: - - "./" - - "vendor" - timeout: 3600 - allow_failure: true + - parallel: + subflow-C: + steps: + - name: C + bash: | + echo "Sub-C" + post: + - name: Post-C + bash: | + echo "Post-C" + + subflow-D: + steps: + - name: D + bash: | + echo "Sub-D" + post: + - name: Post-D + bash: | + echo "Post-D" - name: step3 steps: From c9310e892d76f1a3700dcd3ca8c94f818ac42c41 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Tue, 23 Mar 2021 22:49:54 +0100 Subject: [PATCH 27/46] handle get next post from node tree --- .../java/com/flowci/core/job/domain/Job.java | 2 +- .../job/manager/JobActionManagerImpl.java | 88 +++++++++++++++++-- .../main/java/com/flowci/tree/NodeTree.java | 25 +++++- .../com/flowci/tree/test/YmlParserTest.java | 5 ++ 4 files changed, 109 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/Job.java b/core/src/main/java/com/flowci/core/job/domain/Job.java index 26d2b31a4..3db5e43c2 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Job.java +++ b/core/src/main/java/com/flowci/core/job/domain/Job.java @@ -118,7 +118,7 @@ public enum Status { RUNNING(4), /** - * Running Post steps, only happened before failure status + * Running Post steps, only happened before failure or cancel status */ RUNNING_POST(4), diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index 1e2eea909..d49aef552 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -122,6 +122,7 @@ public class JobActionManagerImpl implements JobActionManager { // cancelling private static final Transition CancellingToCancelled = new Transition(Cancelling, Cancelled); + private static final Transition CancellingToRunningPost = new Transition(Cancelling, RunningPost); private static final long RetryInterval = 10 * 1000; // 10 seconds @@ -186,6 +187,7 @@ public void init(ContextRefreshedEvent ignore) { fromCreated(); fromQueued(); fromRunning(); + fromRunningPost(); fromCancelling(); sm.addHookActionOnTargetStatus(new ActionOnFinishStatus(), Success, Failure, Timeout, Cancelled); @@ -270,6 +272,11 @@ public void toContinue(Job job, Step step) { return; } + if (step.isPost()) { + on(job, Job.Status.RUNNING_POST, (context) -> context.step = step); + return; + } + on(job, Job.Status.RUNNING, (context) -> context.step = step); } @@ -542,12 +549,8 @@ public void accept(JobSmContext context) { Step step = context.step; stepService.toStatus(step, Step.Status.EXCEPTION, null, false); setOngoingStepsToSkipped(job); - logInfo(job, "finished with status {}", Failure); - - if () - - sm.execute(context.getCurrent(), RunningPost, context); + toRunningPostStatusIfNeeded(context); } @Override @@ -585,12 +588,11 @@ public void accept(JobSmContext context) { List ongoingSteps = stepService.list(job, Executed.OngoingStatus); if (jobAgent.allBusyAgents(ongoingSteps).isEmpty()) { setOngoingStepsToSkipped(job); + toRunningPostStatusIfNeeded(context); return; } sm.execute(context.getCurrent(), Cancelling, context); - - // TODO: Run post steps } @Override @@ -605,6 +607,31 @@ public void onFinally(JobSmContext context) { unlockJobAfter(context); } }); + + // from RunningToFailure or RunningToCancelled + sm.add(RunningToRunningPost, new Action() { + @Override + public void accept(JobSmContext context) throws Exception { + doFromCancellingOrRunningToRunningPost(context); + } + + @Override + public void onException(Throwable e, JobSmContext context) { + Job job = context.job; + setJobStatusAndSave(job, Job.Status.FAILURE, e.getMessage()); + } + }); + } + + private void fromRunningPost() { + sm.add(RunningPostToRunningPost, new Action() { + @Override + public void accept(JobSmContext context) throws Exception { + log.debug("---- From -------- To RunningPost"); + + // TODO: continue + } + }); } private void fromCancelling() { @@ -618,7 +645,10 @@ public boolean canRun(JobSmContext context) { public void accept(JobSmContext context) { Job job = context.job; setOngoingStepsToSkipped(job); - setJobStatusAndSave(job, Job.Status.CANCELLED, null); + + if (!toRunningPostStatusIfNeeded(context)) { + setJobStatusAndSave(job, Job.Status.CANCELLED, null); + } } @Override @@ -626,6 +656,30 @@ public void onFinally(JobSmContext context) { unlockJobAfter(context); } }); + + // from CancellingToCancelled + sm.add(CancellingToRunningPost, new Action() { + @Override + public void accept(JobSmContext context) throws Exception { + doFromCancellingOrRunningToRunningPost(context); + } + }); + } + + private void doFromCancellingOrRunningToRunningPost(JobSmContext context) throws ScriptException { + log.debug("---- CancellingOrRunningToRunningPost ----"); + + Job job = context.job; + Step step = context.step; + + NodeTree tree = ymlManager.getTree(job); + Collection post = tree.post(step.getNodePath()); + + // TODO: check flow status for post step ? + + // DO NOT check post size, it has been checked in previous + job.setStatus(Job.Status.RUNNING_POST); + executeJob(job, Lists.newArrayList(post)); } private boolean lockJobBefore(JobSmContext context) { @@ -1025,6 +1079,24 @@ private void toFinishStatus(JobSmContext context) { sm.execute(context.getCurrent(), new Status(statusFromContext.name()), context); } + /** + * To running post status + * Return `true` if post step found + */ + private boolean toRunningPostStatusIfNeeded(JobSmContext context) { + Job job = context.job; + Step step = context.step; + + NodeTree tree = ymlManager.getTree(job); + Collection post = tree.post(step.getNodePath()); + if (post.isEmpty()) { + return false; + } + + sm.execute(context.getCurrent(), RunningPost, context); + return true; + } + private void setJobStatusAndSave(Job job, Job.Status newStatus, String message) { // check status order, just save job if new status is downgrade if (job.getStatus().getOrder() >= newStatus.getOrder()) { diff --git a/tree/src/main/java/com/flowci/tree/NodeTree.java b/tree/src/main/java/com/flowci/tree/NodeTree.java index 7c8c12fc5..26c570397 100644 --- a/tree/src/main/java/com/flowci/tree/NodeTree.java +++ b/tree/src/main/java/com/flowci/tree/NodeTree.java @@ -117,15 +117,29 @@ public List skip(NodePath current) { /** * Find next post step */ - public Collection post(NodePath current) { - Node n = get(current); + public Collection post(NodePath path) { + Node n = get(path); + + // check if step in parallel + if (!isPostStep(n)) { + ParallelStepNode parent = n.getParent(ParallelStepNode.class); + if (parent != null) { + return findPostSteps(parent); + } + } + Collection post = new HashSet<>(); for (Node next : n.next) { post.addAll(findNextPost(next)); } + return post; } + public Collection post(String path) { + return post(NodePath.create(path)); + } + public Node get(NodePath path) { Node node = flatted.get(path); if (node == null) { @@ -285,4 +299,11 @@ private Collection findPostSteps(RegularStepNode r) { } return Collections.emptyList(); } + + private static boolean isPostStep(Node n) { + if (n instanceof RegularStepNode) { + return ((RegularStepNode) n).isPost(); + } + return false; + } } diff --git a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java index 9a4512b02..6a57644bc 100644 --- a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java +++ b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java @@ -339,6 +339,11 @@ public void should_load_and_get_post_steps() throws IOException { Assert.assertEquals(1, nextFromRoot.size()); Assert.assertEquals(postOfSubA, nextFromRoot.get(0).getPath()); + List nextFromSubflowC = Lists.newArrayList(tree.post(NodePath.create("flow/parallel-3/subflow-C/C"))); + Assert.assertEquals(2, nextFromSubflowC.size()); + Assert.assertEquals(postOfSubC, nextFromSubflowC.get(0).getPath()); + Assert.assertEquals(postOfSubD, nextFromSubflowC.get(1).getPath()); + List nextFromSubA = Lists.newArrayList(tree.post(postOfSubA)); Assert.assertEquals(2, nextFromSubA.size()); Assert.assertEquals(postOfSubC, nextFromSubA.get(0).getPath()); From 5766f83fc6ee1fce7d262bb27b12e3328fb9a53f Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Fri, 26 Mar 2021 23:40:52 +0100 Subject: [PATCH 28/46] handle post step --- .../job/manager/JobActionManagerImpl.java | 81 +++++++++++++++---- .../core/job/service/StepServiceImpl.java | 28 ++----- .../main/java/com/flowci/tree/NodeTree.java | 8 +- .../com/flowci/tree/test/YmlParserTest.java | 13 ++- 4 files changed, 83 insertions(+), 47 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index d49aef552..c6ad78a6b 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -478,6 +478,7 @@ public void accept(JobSmContext context) throws Exception { stepService.resultUpdate(step); updateJobContextAndLatestStatus(job, step); + setJobStatusAndSave(job, Job.Status.RUNNING, null); log.debug("Step {} been recorded", step.getNodePath()); if (!step.isSuccess()) { @@ -485,20 +486,11 @@ public void accept(JobSmContext context) throws Exception { return; } - setJobStatusAndSave(job, Job.Status.RUNNING, null); - - NodeTree tree = ymlManager.getTree(job); - Node node = tree.get(step.getNodePath()); - - if (node.isLastChildOfParent()) { - String agentId = releaseAgentFromJob(context.job, node, step); - if (assignAgentToWaitingStep(agentId, job, tree, false)) { - return; - } - releaseAgentToPool(context.job, node, step); + if (releaseAgentOrAssignToWaitingStep(job, step)) { + return; } - if (toNextStep(tree, context.job, context.step)) { + if (toNextStep(job, step, false)) { return; } @@ -625,13 +617,48 @@ public void onException(Throwable e, JobSmContext context) { private void fromRunningPost() { sm.add(RunningPostToRunningPost, new Action() { + @Override + public boolean canRun(JobSmContext context) { + return lockJobBefore(context); + } + @Override public void accept(JobSmContext context) throws Exception { - log.debug("---- From -------- To RunningPost"); + Job job = context.job; + Step step = context.step; - // TODO: continue + stepService.resultUpdate(step); + updateJobContextAndLatestStatus(job, step); + setJobStatusAndSave(job, Job.Status.RUNNING_POST, null); + log.debug("Step {} been recorded", step.getNodePath()); + + if (releaseAgentOrAssignToWaitingStep(job, step)) { + return; + } + + if (toNextStep(job, step, true)) { + return; + } + + toFinishStatus(context); + } + + @Override + public void onFinally(JobSmContext context) { + unlockJobAfter(context); } }); + + Action toActualStatusAction = new Action() { + @Override + public void accept(JobSmContext context) throws Exception { + logInfo(context.job, "RunningPost to {}", context.getTargetToJobStatus()); + } + }; + + sm.add(RunningPostToFailure, toActualStatusAction); + sm.add(RunningPostToTimeout, toActualStatusAction); + sm.add(RunningPostToCanceled, toActualStatusAction); } private void fromCancelling() { @@ -666,6 +693,25 @@ public void accept(JobSmContext context) throws Exception { }); } + /** + * Release agent from job, and try to assign to waiting step, or release to pool + * + * @return true = agent assigned to other waiting step, false = released + */ + private boolean releaseAgentOrAssignToWaitingStep(Job job, Step step) { + NodeTree tree = ymlManager.getTree(job); + Node node = tree.get(step.getNodePath()); + + if (node.isLastChildOfParent()) { + String agentId = releaseAgentFromJob(job, node, step); + if (assignAgentToWaitingStep(agentId, job, tree, false)) { + return true; + } + releaseAgentToPool(job, node, step); + } + return false; + } + private void doFromCancellingOrRunningToRunningPost(JobSmContext context) throws ScriptException { log.debug("---- CancellingOrRunningToRunningPost ----"); @@ -938,10 +984,15 @@ private SimpleSecret getSimpleSecret(String credentialName) { * * @return true if next step dispatched or have to wait for previous steps, false if no more steps or failure */ - private boolean toNextStep(NodeTree tree, Job job, Step step) throws ScriptException { + private boolean toNextStep(Job job, Step step, boolean post) throws ScriptException { + NodeTree tree = ymlManager.getTree(job); Node node = tree.get(NodePath.create(step.getNodePath())); // current node List next = node.getNext(); + if (post) { + next = tree.post(node.getPath()); + } + if (next.isEmpty()) { Collection ends = Sets.newHashSet(tree.ends()); ends.remove(node); // do not check current node diff --git a/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java index 8b9290885..060f78b2c 100644 --- a/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java @@ -34,8 +34,6 @@ import java.util.*; -import static com.google.common.base.Preconditions.checkNotNull; - /** * ExecutedCmd == Step * @@ -151,8 +149,11 @@ public Collection toStatus(Collection steps, Executed.Status status, public Step toStatus(Step entity, Executed.Status status, String err, boolean allChildren) { saveStatus(entity, status, err); - Step parent = getWithNullReturn(entity.getJobId(), entity.getParent()); - updateAllParents(parent, entity); + // update parent status if not post step + if (!entity.isPost()) { + Step parent = getWithNullReturn(entity.getJobId(), entity.getParent()); + updateAllParents(parent, entity); + } if (allChildren) { NodeTree tree = ymlManager.getTree(entity.getJobId()); @@ -177,19 +178,8 @@ public Step toStatus(Step entity, Executed.Status status, String err, boolean al @Override public void resultUpdate(Step stepFromAgent) { - checkNotNull(stepFromAgent.getId()); - Step entity = get(stepFromAgent.getId()); - - // only update properties should from agent - entity.setProcessId(stepFromAgent.getProcessId()); - entity.setCode(stepFromAgent.getCode()); - entity.setStartAt(stepFromAgent.getStartAt()); - entity.setFinishAt(stepFromAgent.getFinishAt()); - entity.setLogSize(stepFromAgent.getLogSize()); - entity.setOutput(stepFromAgent.getOutput()); - // change status and save - toStatus(entity, stepFromAgent.getStatus(), stepFromAgent.getError(), false); + toStatus(stepFromAgent, stepFromAgent.getStatus(), stepFromAgent.getError(), false); } @Override @@ -231,12 +221,8 @@ private void updateAllParents(Step parent, Step current) { } private Step saveStatus(Step step, Step.Status status, String error) { - if (status == step.getStatus()) { - return step; - } - - step.setStatus(status); step.setError(error); + step.setStatus(status); executedCmdDao.save(step); return step; } diff --git a/tree/src/main/java/com/flowci/tree/NodeTree.java b/tree/src/main/java/com/flowci/tree/NodeTree.java index 26c570397..f19c6d683 100644 --- a/tree/src/main/java/com/flowci/tree/NodeTree.java +++ b/tree/src/main/java/com/flowci/tree/NodeTree.java @@ -117,14 +117,14 @@ public List skip(NodePath current) { /** * Find next post step */ - public Collection post(NodePath path) { + public List post(NodePath path) { Node n = get(path); // check if step in parallel if (!isPostStep(n)) { ParallelStepNode parent = n.getParent(ParallelStepNode.class); if (parent != null) { - return findPostSteps(parent); + return Lists.newArrayList(findPostSteps(parent)); } } @@ -133,10 +133,10 @@ public Collection post(NodePath path) { post.addAll(findNextPost(next)); } - return post; + return Lists.newArrayList(post); } - public Collection post(String path) { + public List post(String path) { return post(NodePath.create(path)); } diff --git a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java index 6a57644bc..82a804dd6 100644 --- a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java +++ b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java @@ -19,7 +19,6 @@ import com.flowci.domain.DockerOption; import com.flowci.exception.YmlException; import com.flowci.tree.*; -import com.google.common.collect.Lists; import com.google.common.io.Files; import org.junit.Assert; import org.junit.Before; @@ -335,29 +334,29 @@ public void should_load_and_get_post_steps() throws IOException { NodePath post1OfRoot = NodePath.create("flow/post-1"); NodePath post2OfRoot = NodePath.create("flow/post-2"); - List nextFromRoot = Lists.newArrayList(tree.post(NodePath.create("flow"))); + List nextFromRoot = tree.post(NodePath.create("flow")); Assert.assertEquals(1, nextFromRoot.size()); Assert.assertEquals(postOfSubA, nextFromRoot.get(0).getPath()); - List nextFromSubflowC = Lists.newArrayList(tree.post(NodePath.create("flow/parallel-3/subflow-C/C"))); + List nextFromSubflowC = tree.post(NodePath.create("flow/parallel-3/subflow-C/C")); Assert.assertEquals(2, nextFromSubflowC.size()); Assert.assertEquals(postOfSubC, nextFromSubflowC.get(0).getPath()); Assert.assertEquals(postOfSubD, nextFromSubflowC.get(1).getPath()); - List nextFromSubA = Lists.newArrayList(tree.post(postOfSubA)); + List nextFromSubA = tree.post(postOfSubA); Assert.assertEquals(2, nextFromSubA.size()); Assert.assertEquals(postOfSubC, nextFromSubA.get(0).getPath()); Assert.assertEquals(postOfSubD, nextFromSubA.get(1).getPath()); - List nextFromSubC = Lists.newArrayList(tree.post(postOfSubC)); + List nextFromSubC = tree.post(postOfSubC); Assert.assertEquals(1, nextFromSubC.size()); Assert.assertEquals(post1OfRoot, nextFromSubC.get(0).getPath()); - List nextFromSubD = Lists.newArrayList(tree.post(postOfSubD)); + List nextFromSubD = tree.post(postOfSubD); Assert.assertEquals(1, nextFromSubD.size()); Assert.assertEquals(post1OfRoot, nextFromSubD.get(0).getPath()); - List nextPostFromRootPost1 = Lists.newArrayList(tree.post(post1OfRoot)); + List nextPostFromRootPost1 = tree.post(post1OfRoot); Assert.assertEquals(1, nextPostFromRootPost1.size()); Assert.assertEquals(post2OfRoot, nextPostFromRootPost1.get(0).getPath()); } From 3a0520edb0c229451e44327290bda57c864333da Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sat, 27 Mar 2021 16:09:13 +0100 Subject: [PATCH 29/46] apply list return type of post steps --- .../flowci/core/job/manager/JobActionManagerImpl.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index c6ad78a6b..0ba4b3ca5 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -719,13 +719,11 @@ private void doFromCancellingOrRunningToRunningPost(JobSmContext context) throws Step step = context.step; NodeTree tree = ymlManager.getTree(job); - Collection post = tree.post(step.getNodePath()); - - // TODO: check flow status for post step ? + List post = tree.post(step.getNodePath()); // DO NOT check post size, it has been checked in previous job.setStatus(Job.Status.RUNNING_POST); - executeJob(job, Lists.newArrayList(post)); + executeJob(job, post); } private boolean lockJobBefore(JobSmContext context) { @@ -990,7 +988,7 @@ private boolean toNextStep(Job job, Step step, boolean post) throws ScriptExcept List next = node.getNext(); if (post) { - next = tree.post(node.getPath()); + next = tree.post(step.getNodePath()); } if (next.isEmpty()) { @@ -1139,7 +1137,8 @@ private boolean toRunningPostStatusIfNeeded(JobSmContext context) { Step step = context.step; NodeTree tree = ymlManager.getTree(job); - Collection post = tree.post(step.getNodePath()); + List post = tree.post(step.getNodePath()); + if (post.isEmpty()) { return false; } From b8a55a024d3a4678dbd8c5156c057577efa5aec8 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Mon, 29 Mar 2021 23:09:46 +0200 Subject: [PATCH 30/46] fix missing running post to cancelling status --- .../java/com/flowci/core/job/domain/Job.java | 5 ++++ .../job/manager/JobActionManagerImpl.java | 29 ++++++++++++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/Job.java b/core/src/main/java/com/flowci/core/job/domain/Job.java index 3db5e43c2..21ebdda95 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Job.java +++ b/core/src/main/java/com/flowci/core/job/domain/Job.java @@ -269,6 +269,11 @@ public boolean isRunning() { return status == Status.RUNNING; } + @JsonIgnore + public boolean isRunningPost() { + return status == Status.RUNNING_POST; + } + @JsonIgnore public boolean isCancelling() { return status == Status.CANCELLING; diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index 0ba4b3ca5..fbccd52a1 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -117,6 +117,7 @@ public class JobActionManagerImpl implements JobActionManager { // running post private static final Transition RunningPostToRunningPost = new Transition(RunningPost, RunningPost); private static final Transition RunningPostToFailure = new Transition(RunningPost, Failure); + private static final Transition RunningPostToCancelling = new Transition(RunningPost, Cancelling); private static final Transition RunningPostToCanceled = new Transition(RunningPost, Cancelled); private static final Transition RunningPostToTimeout = new Transition(RunningPost, Timeout); @@ -272,7 +273,7 @@ public void toContinue(Job job, Step step) { return; } - if (step.isPost()) { + if (job.isRunningPost()) { on(job, Job.Status.RUNNING_POST, (context) -> context.step = step); return; } @@ -576,10 +577,10 @@ public boolean canRun(JobSmContext context) { public void accept(JobSmContext context) { Job job = context.job; JobAgent jobAgent = getJobAgent(job.getId()); - List ongoingSteps = stepService.list(job, Executed.OngoingStatus); + + // no busy agents, run post steps directly if needed if (jobAgent.allBusyAgents(ongoingSteps).isEmpty()) { - setOngoingStepsToSkipped(job); toRunningPostStatusIfNeeded(context); return; } @@ -659,6 +660,12 @@ public void accept(JobSmContext context) throws Exception { sm.add(RunningPostToFailure, toActualStatusAction); sm.add(RunningPostToTimeout, toActualStatusAction); sm.add(RunningPostToCanceled, toActualStatusAction); + sm.add(RunningPostToCancelling, new Action() { + @Override + public void accept(JobSmContext context) throws Exception { + sm.execute(context.getCurrent(), Cancelled, context); + } + }); } private void fromCancelling() { @@ -671,8 +678,6 @@ public boolean canRun(JobSmContext context) { @Override public void accept(JobSmContext context) { Job job = context.job; - setOngoingStepsToSkipped(job); - if (!toRunningPostStatusIfNeeded(context)) { setJobStatusAndSave(job, Job.Status.CANCELLED, null); } @@ -901,8 +906,17 @@ private void setupJobYamlAndSteps(Job job, String yml) { private void setOngoingStepsToSkipped(Job job) { List steps = stepService.list(job, Executed.OngoingStatus); - for (Step step : steps) { - if (!step.hasAgent() || step.isPost()) { + Iterator iter = steps.iterator(); + + while (iter.hasNext()) { + Step step = iter.next(); + + if (step.isPost()) { + iter.remove(); + continue; + } + + if (!step.hasAgent()) { continue; } @@ -912,6 +926,7 @@ private void setOngoingStepsToSkipped(Job job) { agentService.dispatch(killCmd, agent); } } + stepService.toStatus(steps, Step.Status.SKIPPED, null); } From 252a7c19fbeb5bc95a32a3bc9a56cea71c935eaa Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Tue, 30 Mar 2021 22:36:18 +0200 Subject: [PATCH 31/46] add killing status on stepg --- .../com/flowci/core/job/domain/Executed.java | 3 +- .../job/manager/JobActionManagerImpl.java | 58 ++++++++++++------- .../core/job/service/JobEventServiceImpl.java | 2 + 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/Executed.java b/core/src/main/java/com/flowci/core/job/domain/Executed.java index ad3fd870d..e1f917dd9 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Executed.java +++ b/core/src/main/java/com/flowci/core/job/domain/Executed.java @@ -15,7 +15,6 @@ public interface Executed { ); Set OngoingStatus = ImmutableSet.of( - Status.PENDING, Status.WAITING_AGENT, Status.RUNNING ); @@ -46,6 +45,8 @@ enum Status { RUNNING(1), + KILLING(1), + SUCCESS(2), SKIPPED(2), diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index fbccd52a1..5545e53d6 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -69,6 +69,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; +import static com.flowci.core.job.domain.Executed.Status.RUNNING; +import static com.flowci.core.job.domain.Executed.Status.WAITING_AGENT; + @Log4j2 @Service public class JobActionManagerImpl implements JobActionManager { @@ -477,7 +480,6 @@ public void accept(JobSmContext context) throws Exception { Job job = context.job; Step step = context.step; - stepService.resultUpdate(step); updateJobContextAndLatestStatus(job, step); setJobStatusAndSave(job, Job.Status.RUNNING, null); log.debug("Step {} been recorded", step.getNodePath()); @@ -523,7 +525,7 @@ public void accept(JobSmContext context) { @Override public void accept(JobSmContext context) { Job job = context.job; - setOngoingStepsToSkipped(job); + killOngoingSteps(job); } @Override @@ -541,7 +543,7 @@ public void accept(JobSmContext context) { Job job = context.job; Step step = context.step; stepService.toStatus(step, Step.Status.EXCEPTION, null, false); - setOngoingStepsToSkipped(job); + killOngoingSteps(job); toRunningPostStatusIfNeeded(context); } @@ -558,7 +560,7 @@ public void onException(Throwable e, JobSmContext context) { public void accept(JobSmContext context) { Job job = context.job; setJobStatusAndSave(job, Job.Status.CANCELLING, null); - setOngoingStepsToSkipped(job); + killOngoingSteps(job); } @Override @@ -591,7 +593,7 @@ public void accept(JobSmContext context) { @Override public void onException(Throwable e, JobSmContext context) { Job job = context.job; - setOngoingStepsToSkipped(job); + killOngoingSteps(job); setJobStatusAndSave(job, Job.Status.CANCELLED, e.getMessage()); } @@ -628,7 +630,6 @@ public void accept(JobSmContext context) throws Exception { Job job = context.job; Step step = context.step; - stepService.resultUpdate(step); updateJobContextAndLatestStatus(job, step); setJobStatusAndSave(job, Job.Status.RUNNING_POST, null); log.debug("Step {} been recorded", step.getNodePath()); @@ -659,7 +660,13 @@ public void accept(JobSmContext context) throws Exception { sm.add(RunningPostToFailure, toActualStatusAction); sm.add(RunningPostToTimeout, toActualStatusAction); - sm.add(RunningPostToCanceled, toActualStatusAction); + sm.add(RunningPostToCanceled, new Action() { + @Override + public void accept(JobSmContext context) throws Exception { + // TODO: send kill to all running steps + } + }); + sm.add(RunningPostToCancelling, new Action() { @Override public void accept(JobSmContext context) throws Exception { @@ -678,6 +685,13 @@ public boolean canRun(JobSmContext context) { @Override public void accept(JobSmContext context) { Job job = context.job; + Step step = context.step; + + if (step == null) { + setJobStatusAndSave(job, Job.Status.CANCELLED, null); + return; + } + if (!toRunningPostStatusIfNeeded(context)) { setJobStatusAndSave(job, Job.Status.CANCELLED, null); } @@ -822,7 +836,7 @@ private Optional fetchAgentFromPool(Job job, Node node) { } private boolean assignAgentToWaitingStep(String agentId, Job job, NodeTree tree, boolean shouldIdle) { - List steps = stepService.list(job, Lists.newArrayList(Executed.Status.WAITING_AGENT)); + List steps = stepService.list(job, Lists.newArrayList(WAITING_AGENT)); if (steps.isEmpty()) { return false; } @@ -904,8 +918,11 @@ private void setupJobYamlAndSteps(Job job, String yml) { job.getContext().merge(root.getEnvironments(), false); } - private void setOngoingStepsToSkipped(Job job) { - List steps = stepService.list(job, Executed.OngoingStatus); + private void killOngoingSteps(Job job) { + List steps = stepService.list(job, Sets.newHashSet(WAITING_AGENT)); + stepService.toStatus(steps, Step.Status.SKIPPED, null); + + steps = stepService.list(job, Sets.newHashSet(RUNNING)); Iterator iter = steps.iterator(); while (iter.hasNext()) { @@ -916,18 +933,17 @@ private void setOngoingStepsToSkipped(Job job) { continue; } - if (!step.hasAgent()) { - continue; - } - - Agent agent = agentService.get(step.getAgentId()); - if (agent.isBusy()) { - CmdIn killCmd = cmdManager.createKillCmd(); - agentService.dispatch(killCmd, agent); + if (step.hasAgent()) { + Agent agent = agentService.get(step.getAgentId()); + if (agent.isBusy()) { + CmdIn killCmd = cmdManager.createKillCmd(); + agentService.dispatch(killCmd, agent); + iter.remove(); // update step status from callback + } } } - stepService.toStatus(steps, Step.Status.SKIPPED, null); + stepService.toStatus(steps, Step.Status.KILLING, null); } private void on(Job job, Job.Status target, Consumer configContext) { @@ -1092,7 +1108,7 @@ private void executeJob(Job job, List nodes) throws ScriptException { continue; } - stepService.toStatus(step, Executed.Status.WAITING_AGENT, null, false); + stepService.toStatus(step, WAITING_AGENT, null, false); } } @@ -1117,7 +1133,7 @@ private boolean runCondition(Job job, Node node) throws ScriptException { private void dispatch(Job job, Node node, Step step, Agent agent) { step.setAgentId(agent.getId()); - stepService.toStatus(step, Step.Status.RUNNING, null, false); + stepService.toStatus(step, RUNNING, null, false); ShellIn cmd = cmdManager.createShellCmd(job, step, node); agentService.dispatch(cmd, agent); diff --git a/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java index e554230b8..7420c3f05 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java @@ -158,8 +158,10 @@ public void handleCmdOutFromAgent(OnCmdOutEvent event) { switch (ind) { case CmdOut.ShellOutInd: ShellOut shellOut = objectMapper.readValue(body, ShellOut.class); + Step step = stepService.get(shellOut.getId()); step.setFrom(shellOut); + stepService.resultUpdate(step); log.info("[Callback]: {}-{} = {}", step.getJobId(), step.getNodePath(), step.getStatus()); handleCallback(step); From 62420d59941746925863a0ea8f5203086fc3349e Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Thu, 1 Apr 2021 14:21:47 +0200 Subject: [PATCH 32/46] enable to find prev post stepg --- .../flowci/core/job/domain/JobSmContext.java | 2 + .../job/manager/JobActionManagerImpl.java | 28 ++++++++----- .../main/java/com/flowci/tree/NodeTree.java | 39 +++++++++++++++---- .../com/flowci/tree/test/YmlParserTest.java | 8 ++++ 4 files changed, 60 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java b/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java index 427afcca4..854aaabe6 100644 --- a/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java +++ b/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java @@ -13,6 +13,8 @@ public class JobSmContext extends Context { public InterLock lock; + public boolean callFromMethod; + public Job.Status getTargetToJobStatus() { String name = this.to.getName(); return Job.Status.valueOf(name); diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index 5545e53d6..c745fc15e 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -288,6 +288,7 @@ public void toContinue(Job job, Step step) { public void toCancelled(Job job, String reason) { on(job, Job.Status.CANCELLED, context -> { context.setError(new CIException(reason)); + context.callFromMethod = true; }); } @@ -525,7 +526,7 @@ public void accept(JobSmContext context) { @Override public void accept(JobSmContext context) { Job job = context.job; - killOngoingSteps(job); + killOngoingSteps(job, false); } @Override @@ -543,7 +544,7 @@ public void accept(JobSmContext context) { Job job = context.job; Step step = context.step; stepService.toStatus(step, Step.Status.EXCEPTION, null, false); - killOngoingSteps(job); + killOngoingSteps(job, false); toRunningPostStatusIfNeeded(context); } @@ -560,7 +561,7 @@ public void onException(Throwable e, JobSmContext context) { public void accept(JobSmContext context) { Job job = context.job; setJobStatusAndSave(job, Job.Status.CANCELLING, null); - killOngoingSteps(job); + killOngoingSteps(job, false); } @Override @@ -593,7 +594,7 @@ public void accept(JobSmContext context) { @Override public void onException(Throwable e, JobSmContext context) { Job job = context.job; - killOngoingSteps(job); + killOngoingSteps(job, false); setJobStatusAndSave(job, Job.Status.CANCELLED, e.getMessage()); } @@ -663,14 +664,23 @@ public void accept(JobSmContext context) throws Exception { sm.add(RunningPostToCanceled, new Action() { @Override public void accept(JobSmContext context) throws Exception { - // TODO: send kill to all running steps + if (!context.callFromMethod) { + return; + } + sm.execute(context.getCurrent(), Cancelling, context); } }); sm.add(RunningPostToCancelling, new Action() { @Override public void accept(JobSmContext context) throws Exception { - sm.execute(context.getCurrent(), Cancelled, context); + if (!context.callFromMethod) { + sm.execute(context.getCurrent(), Cancelled, context); + return; + } + + Job job = context.job; + killOngoingSteps(job, true); } }); } @@ -918,7 +928,7 @@ private void setupJobYamlAndSteps(Job job, String yml) { job.getContext().merge(root.getEnvironments(), false); } - private void killOngoingSteps(Job job) { + private void killOngoingSteps(Job job, boolean includePost) { List steps = stepService.list(job, Sets.newHashSet(WAITING_AGENT)); stepService.toStatus(steps, Step.Status.SKIPPED, null); @@ -928,7 +938,7 @@ private void killOngoingSteps(Job job) { while (iter.hasNext()) { Step step = iter.next(); - if (step.isPost()) { + if (!includePost && step.isPost()) { iter.remove(); continue; } @@ -1031,7 +1041,7 @@ private boolean toNextStep(Job job, Step step, boolean post) throws ScriptExcept } // check prev steps status - Set previous = getStepsStatus(job, tree.prevs(next)); + Set previous = getStepsStatus(job, tree.prevs(next, post)); boolean hasFailure = !Collections.disjoint(previous, Executed.FailureStatus); boolean hasOngoing = !Collections.disjoint(previous, Executed.OngoingStatus); if (hasFailure) { diff --git a/tree/src/main/java/com/flowci/tree/NodeTree.java b/tree/src/main/java/com/flowci/tree/NodeTree.java index f19c6d683..538acd740 100644 --- a/tree/src/main/java/com/flowci/tree/NodeTree.java +++ b/tree/src/main/java/com/flowci/tree/NodeTree.java @@ -31,6 +31,8 @@ public final class NodeTree { private static final int DefaultSize = 20; + private static final int DefaultSizeForPrev = 5; + /** * Create node tree from FlowNode object */ @@ -75,15 +77,25 @@ public Collection ends() { return ends; } - /** - * Get all previous node list from path - */ - public Collection prevs(List nodes) { - List list = new LinkedList<>(); - for (Node n : nodes) { - list.addAll(n.getPrev()); + public Collection prevs(Collection nodes, boolean post) { + Collection ps = prevs(nodes); + + if (!post) { + return ps; } - return list; + + Set prevPost = new HashSet<>(DefaultSizeForPrev); + for (Node p : ps) { + if (isPostStep(p)) { + prevPost.add(p); + } + } + + if (prevPost.isEmpty()) { + return prevs(ps, true); + } + + return prevPost; } /** @@ -152,6 +164,17 @@ public Node get(String nodePath) { return get(NodePath.create(nodePath)); } + /** + * Get all previous node list from path + */ + private Collection prevs(Collection nodes) { + Set list = new HashSet<>(DefaultSizeForPrev); + for (Node n : nodes) { + list.addAll(n.getPrev()); + } + return list; + } + private Node findNextWithSameParent(Node node, Node parent) { for (Node next : node.next) { if (parent.equals((next.parent))) { diff --git a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java index 82a804dd6..4d50283c0 100644 --- a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java +++ b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.Collection; import java.util.List; import static com.flowci.tree.FlowNode.DEFAULT_ROOT_NAME; @@ -359,6 +360,13 @@ public void should_load_and_get_post_steps() throws IOException { List nextPostFromRootPost1 = tree.post(post1OfRoot); Assert.assertEquals(1, nextPostFromRootPost1.size()); Assert.assertEquals(post2OfRoot, nextPostFromRootPost1.get(0).getPath()); + + // get prev post step + Collection prevsOfPost1 = tree.prevs(nextFromSubD, false); + Assert.assertEquals(1, prevsOfPost1.size()); + + prevsOfPost1 = tree.prevs(nextFromSubD, true); + Assert.assertEquals(2, prevsOfPost1.size()); } private String loadContent(String resource) throws IOException { From 35610ad2df90550a49612c29cc95dbc7ec67d3f7 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Thu, 1 Apr 2021 14:41:32 +0200 Subject: [PATCH 33/46] more desc on next stepg --- .../java/com/flowci/core/job/manager/JobActionManagerImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index c745fc15e..ab1802e5f 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -1021,6 +1021,9 @@ private SimpleSecret getSimpleSecret(String credentialName) { /** * Dispatch next step to agent, job will be saved on final function of Running status * + * @param job current job + * @param step current step + * @param post indicate to next post steps or regular steps * @return true if next step dispatched or have to wait for previous steps, false if no more steps or failure */ private boolean toNextStep(Job job, Step step, boolean post) throws ScriptException { From 29441caa7a9ab3853ea27af820e8051394d95d45 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Tue, 6 Apr 2021 22:25:25 +0200 Subject: [PATCH 34/46] support secret in step yml --- tree/src/main/java/com/flowci/tree/RegularStepNode.java | 5 +++++ tree/src/main/java/com/flowci/tree/yml/StepYml.java | 7 +++++-- tree/src/test/java/com/flowci/tree/test/YmlParserTest.java | 2 ++ tree/src/test/resources/flow.yml | 2 ++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tree/src/main/java/com/flowci/tree/RegularStepNode.java b/tree/src/main/java/com/flowci/tree/RegularStepNode.java index b7dc03326..df9da7265 100644 --- a/tree/src/main/java/com/flowci/tree/RegularStepNode.java +++ b/tree/src/main/java/com/flowci/tree/RegularStepNode.java @@ -52,6 +52,11 @@ public final class RegularStepNode extends Node { */ private Set exports = new HashSet<>(0); + /** + * Included secret name in the step + */ + private Set secrets; + /** * Is allow failure */ diff --git a/tree/src/main/java/com/flowci/tree/yml/StepYml.java b/tree/src/main/java/com/flowci/tree/yml/StepYml.java index 5d8ce4e7a..34855189a 100644 --- a/tree/src/main/java/com/flowci/tree/yml/StepYml.java +++ b/tree/src/main/java/com/flowci/tree/yml/StepYml.java @@ -58,12 +58,14 @@ public class StepYml extends YmlBase { private Integer timeout; // timeout in seconds - private List exports = new LinkedList<>(); - private Boolean allow_failure; private Cache cache; + private List exports = new LinkedList<>(); + + private List secrets = new LinkedList<>(); + /** * Only for parallel step, other fields will not valid */ @@ -111,6 +113,7 @@ public Node toNode(Node parent, int index) { step.setExports(Sets.newHashSet(exports)); step.setAllowFailure(allow_failure != null && allow_failure); step.setEnvironments(getVariableMap()); + step.setSecrets(Sets.newHashSet(secrets)); setCacheToNode(step); setDockerToNode(step); diff --git a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java index 4d50283c0..f73d9b321 100644 --- a/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java +++ b/tree/src/test/java/com/flowci/tree/test/YmlParserTest.java @@ -83,6 +83,8 @@ public void should_get_node_from_yml() { Assert.assertEquals("echo step", step1.getEnv("FLOW_WORKSPACE")); Assert.assertEquals("echo step version", step1.getEnv("FLOW_VERSION")); Assert.assertEquals(3600, step1.getTimeout().intValue()); + Assert.assertEquals(1, step1.getSecrets().size()); + Assert.assertTrue(step1.getSecrets().contains("my-secret")); Assert.assertTrue(step1.isAllowFailure()); Assert.assertEquals("println(FLOW_WORKSPACE)\ntrue\n", step1.getCondition()); diff --git a/tree/src/test/resources/flow.yml b/tree/src/test/resources/flow.yml index 23ed14775..4818c7d07 100644 --- a/tree/src/test/resources/flow.yml +++ b/tree/src/test/resources/flow.yml @@ -26,6 +26,8 @@ steps: envs: FLOW_WORKSPACE: "echo step" FLOW_VERSION: "echo step version" + secrets: + - "my-secret" cache: key: mycache paths: From 4b3e12cf657428f1e19b9209fd9392f8972e1a79 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Fri, 9 Apr 2021 22:49:35 +0200 Subject: [PATCH 35/46] validate secret on yml save --- .../core/flow/service/YmlServiceImpl.java | 87 +++++++++++-------- .../core/secret/event/GetSecretEvent.java | 4 +- .../main/java/com/flowci/tree/NodeTree.java | 6 ++ .../java/com/flowci/tree/RegularStepNode.java | 7 +- 4 files changed, 67 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/com/flowci/core/flow/service/YmlServiceImpl.java b/core/src/main/java/com/flowci/core/flow/service/YmlServiceImpl.java index 583108047..040415ea1 100644 --- a/core/src/main/java/com/flowci/core/flow/service/YmlServiceImpl.java +++ b/core/src/main/java/com/flowci/core/flow/service/YmlServiceImpl.java @@ -23,6 +23,7 @@ import com.flowci.core.flow.domain.Flow; import com.flowci.core.flow.domain.Yml; import com.flowci.core.plugin.event.GetPluginEvent; +import com.flowci.core.secret.event.GetSecretEvent; import com.flowci.domain.Vars; import com.flowci.exception.ArgumentException; import com.flowci.exception.DuplicateException; @@ -32,13 +33,15 @@ import com.flowci.tree.YmlParser; import com.flowci.util.StringHelper; import com.github.benmanes.caffeine.cache.Cache; +import com.google.common.collect.ImmutableList; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; import java.util.List; import java.util.Optional; -import java.util.Set; +import java.util.function.Function; + /** * @author yang @@ -46,6 +49,12 @@ @Service public class YmlServiceImpl implements YmlService { + private final List elementCheckers = ImmutableList.of( + new ConditionChecker(), + new PluginChecker(), + new SecretChecker() + ); + @Autowired private Cache flowTreeCache; @@ -102,18 +111,14 @@ public Yml saveYml(Flow flow, String name, String ymlInB64) { FlowNode root = YmlParser.load(yaml); NodeTree tree = NodeTree.create(root); - Optional hasErr = verifyPlugins(tree.getPlugins()); - if (hasErr.isPresent()) { - throw hasErr.get(); - } - - hasErr = verifyConditions(tree.getConditions()); - if (hasErr.isPresent()) { - throw hasErr.get(); + for (NodeElementChecker checker : elementCheckers) { + Optional exception = checker.apply(tree); + if (exception.isPresent()) { + throw exception.get(); + } } Yml ymlObj = getOrCreate(flow.getId(), name, ymlInB64); - try { ymlDao.save(ymlObj); } catch (DuplicateKeyException e) { @@ -155,36 +160,50 @@ private Yml getOrCreate(String flowId, String name, String ymlInB64) { return new Yml(flowId, name, ymlInB64); } - /** - * Check conditions are existed - * - * @param conditions list of condition script - * @return Optional exception - */ - private Optional verifyConditions(Set conditions) { - try { - for (String condition : conditions) { - conditionManager.verify(condition); + private interface NodeElementChecker extends Function> { + + } + + private class ConditionChecker implements NodeElementChecker { + + @Override + public Optional apply(NodeTree tree) { + try { + for (String c : tree.getConditions()) { + conditionManager.verify(c); + } + return Optional.empty(); + } catch (Throwable e) { + return Optional.of(new RuntimeException(e.getMessage())); + } + } + } + + private class PluginChecker implements NodeElementChecker { + + @Override + public Optional apply(NodeTree tree) { + for (String p : tree.getPlugins()) { + GetPluginEvent event = eventManager.publish(new GetPluginEvent(this, p)); + if (event.hasError()) { + return Optional.of(event.getError()); + } } return Optional.empty(); - } catch (Throwable e) { - return Optional.of(new RuntimeException(e.getMessage())); } } - /** - * Check plugins are existed - * - * @param plugins input plugin name list - * @return Optional exception - */ - private Optional verifyPlugins(Set plugins) { - for (String plugin : plugins) { - GetPluginEvent event = eventManager.publish(new GetPluginEvent(this, plugin)); - if (event.hasError()) { - return Optional.of(event.getError()); + private class SecretChecker implements NodeElementChecker { + + @Override + public Optional apply(NodeTree tree) { + for (String s : tree.getSecrets()) { + GetSecretEvent event = eventManager.publish(new GetSecretEvent(this, s)); + if (event.hasError()) { + return Optional.of(event.getError()); + } } + return Optional.empty(); } - return Optional.empty(); } } diff --git a/core/src/main/java/com/flowci/core/secret/event/GetSecretEvent.java b/core/src/main/java/com/flowci/core/secret/event/GetSecretEvent.java index b617698f5..1830de538 100644 --- a/core/src/main/java/com/flowci/core/secret/event/GetSecretEvent.java +++ b/core/src/main/java/com/flowci/core/secret/event/GetSecretEvent.java @@ -30,8 +30,8 @@ public class GetSecretEvent extends AbstractEvent { private final String name; - public GetSecretEvent(Object source, String credentialName) { + public GetSecretEvent(Object source, String secretName) { super(source); - this.name = credentialName; + this.name = secretName; } } diff --git a/tree/src/main/java/com/flowci/tree/NodeTree.java b/tree/src/main/java/com/flowci/tree/NodeTree.java index 538acd740..45f9056cc 100644 --- a/tree/src/main/java/com/flowci/tree/NodeTree.java +++ b/tree/src/main/java/com/flowci/tree/NodeTree.java @@ -55,6 +55,8 @@ public static NodeTree create(FlowNode root) { private final Set plugins = new HashSet<>(DefaultSize); + private final Set secrets = new HashSet<>(DefaultSize); + private int maxHeight = 1; public NodeTree(FlowNode root) { @@ -201,6 +203,10 @@ private void buildMetaData() { if (r.hasPlugin()) { plugins.add(r.getPlugin()); } + + if (r.hasSecrets()) { + secrets.addAll(r.getSecrets()); + } } if (node instanceof FlowNode) { diff --git a/tree/src/main/java/com/flowci/tree/RegularStepNode.java b/tree/src/main/java/com/flowci/tree/RegularStepNode.java index df9da7265..bf8300003 100644 --- a/tree/src/main/java/com/flowci/tree/RegularStepNode.java +++ b/tree/src/main/java/com/flowci/tree/RegularStepNode.java @@ -55,7 +55,7 @@ public final class RegularStepNode extends Node { /** * Included secret name in the step */ - private Set secrets; + private Set secrets = new HashSet<>(0); /** * Is allow failure @@ -86,6 +86,11 @@ public boolean hasCondition() { return !Strings.isNullOrEmpty(condition); } + @JsonIgnore + public boolean hasSecrets() { + return !secrets.isEmpty(); + } + @JsonIgnore public boolean hasTimeout() { return timeout != null; From e97fa949cceec8fa4cf9f43dc9a5a4c3b4cb8c38 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Sat, 10 Apr 2021 21:53:23 +0200 Subject: [PATCH 36/46] support secret in shell in --- core/src/main/java/com/flowci/core/agent/domain/ShellIn.java | 2 ++ .../main/java/com/flowci/core/job/manager/CmdManagerImpl.java | 1 + 2 files changed, 3 insertions(+) diff --git a/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java b/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java index 8ad851a73..d1b8bcfed 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java +++ b/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java @@ -50,6 +50,8 @@ public enum ShellType { private Set envFilters; + private Set secrets; + public ShellIn() { super(Type.SHELL); } diff --git a/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java index 4011d23ab..781f1f169 100644 --- a/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/CmdManagerImpl.java @@ -64,6 +64,7 @@ public ShellIn createShellCmd(Job job, Step step, Node node) { .setInputs(r.fetchEnvs().merge(job.getContext(), false)) .setTimeout(r.fetchTimeout(job.getTimeout())) .setRetry(r.fetchRetry(0)) + .setSecrets(r.getSecrets()) .setCache(r.getCache()); if (r.hasPlugin()) { From 4e496a18edaa8015b2db5e2a84c80b27b9baee2c Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Wed, 14 Apr 2021 15:29:58 +0200 Subject: [PATCH 37/46] remove running post status --- .../java/com/flowci/core/job/domain/Job.java | 12 +- .../com/flowci/core/job/domain/JobAgent.java | 20 -- .../flowci/core/job/domain/JobSmContext.java | 2 - .../core/job/manager/JobActionManager.java | 3 +- .../job/manager/JobActionManagerImpl.java | 225 ++++++------------ .../core/job/service/JobEventServiceImpl.java | 3 +- .../core/job/service/JobServiceImpl.java | 2 +- .../java/com/flowci/core/job/util/Errors.java | 9 + sm/src/main/java/com/flowci/sm/Context.java | 1 + .../main/java/com/flowci/sm/StateMachine.java | 2 +- 10 files changed, 96 insertions(+), 183 deletions(-) create mode 100644 core/src/main/java/com/flowci/core/job/util/Errors.java diff --git a/core/src/main/java/com/flowci/core/job/domain/Job.java b/core/src/main/java/com/flowci/core/job/domain/Job.java index 21ebdda95..1d582f376 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Job.java +++ b/core/src/main/java/com/flowci/core/job/domain/Job.java @@ -117,11 +117,6 @@ public enum Status { */ RUNNING(4), - /** - * Running Post steps, only happened before failure or cancel status - */ - RUNNING_POST(4), - /** * Job will be cancelled, but waiting for response from agent */ @@ -215,6 +210,8 @@ public static Pathable path(Long buildNumber) { private Status status = Status.PENDING; + private boolean onPostSteps = false; + // agent id : info private Map snapshots = new HashMap<>(); @@ -269,11 +266,6 @@ public boolean isRunning() { return status == Status.RUNNING; } - @JsonIgnore - public boolean isRunningPost() { - return status == Status.RUNNING_POST; - } - @JsonIgnore public boolean isCancelling() { return status == Status.CANCELLING; diff --git a/core/src/main/java/com/flowci/core/job/domain/JobAgent.java b/core/src/main/java/com/flowci/core/job/domain/JobAgent.java index 77431dc5e..0752e104e 100644 --- a/core/src/main/java/com/flowci/core/job/domain/JobAgent.java +++ b/core/src/main/java/com/flowci/core/job/domain/JobAgent.java @@ -39,26 +39,6 @@ public Collection all() { return agents.keySet(); } - /** - * All busy agents, which are occupied by flow and assigned to step - */ - public Collection allBusyAgents(Collection ongoingSteps) { - Set busy = new HashSet<>(agents.size()); - this.agents.forEach((k, v) -> { - if (v.isEmpty()) { - return; - } - - for(Step s : ongoingSteps) { - if (s.hasAgent() && s.getAgentId().equals(k)) { - busy.add(k); - return; - } - } - }); - return busy; - } - public boolean isOccupiedByFlow(String agentId) { Set flows = agents.get(agentId); return flows != null && !flows.isEmpty(); diff --git a/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java b/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java index 854aaabe6..427afcca4 100644 --- a/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java +++ b/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java @@ -13,8 +13,6 @@ public class JobSmContext extends Context { public InterLock lock; - public boolean callFromMethod; - public Job.Status getTargetToJobStatus() { String name = this.to.getName(); return Job.Status.valueOf(name); diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManager.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManager.java index 63b1bc65b..fa77b9f87 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManager.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManager.java @@ -2,6 +2,7 @@ import com.flowci.core.job.domain.Step; import com.flowci.core.job.domain.Job; +import com.flowci.exception.CIException; public interface JobActionManager { @@ -15,7 +16,7 @@ public interface JobActionManager { void toContinue(Job job, Step step); - void toCancelled(Job job, String reason); + void toCancelled(Job job, CIException exception); void toTimeout(Job job); } diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index ab1802e5f..e6a5ebd82 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -36,6 +36,7 @@ import com.flowci.core.job.event.JobStatusChangeEvent; import com.flowci.core.job.service.LocalTaskService; import com.flowci.core.job.service.StepService; +import com.flowci.core.job.util.Errors; import com.flowci.core.job.util.StatusHelper; import com.flowci.core.secret.domain.Secret; import com.flowci.core.secret.service.SecretService; @@ -83,7 +84,6 @@ public class JobActionManagerImpl implements JobActionManager { private static final Status Cancelling = new Status(Job.Status.CANCELLING.name()); private static final Status Queued = new Status(Job.Status.QUEUED.name()); private static final Status Running = new Status(Job.Status.RUNNING.name()); - private static final Status RunningPost = new Status(Job.Status.RUNNING_POST.name()); private static final Status Timeout = new Status(Job.Status.TIMEOUT.name()); private static final Status Failure = new Status(Job.Status.FAILURE.name()); private static final Status Success = new Status(Job.Status.SUCCESS.name()); @@ -115,18 +115,10 @@ public class JobActionManagerImpl implements JobActionManager { private static final Transition RunningToCanceled = new Transition(Running, Cancelled); private static final Transition RunningToTimeout = new Transition(Running, Timeout); private static final Transition RunningToFailure = new Transition(Running, Failure); - private static final Transition RunningToRunningPost = new Transition(Running, RunningPost); - - // running post - private static final Transition RunningPostToRunningPost = new Transition(RunningPost, RunningPost); - private static final Transition RunningPostToFailure = new Transition(RunningPost, Failure); - private static final Transition RunningPostToCancelling = new Transition(RunningPost, Cancelling); - private static final Transition RunningPostToCanceled = new Transition(RunningPost, Cancelled); - private static final Transition RunningPostToTimeout = new Transition(RunningPost, Timeout); // cancelling private static final Transition CancellingToCancelled = new Transition(Cancelling, Cancelled); - private static final Transition CancellingToRunningPost = new Transition(Cancelling, RunningPost); + private static final Transition CancellingToRunning = new Transition(Cancelling, Running); private static final long RetryInterval = 10 * 1000; // 10 seconds @@ -191,7 +183,6 @@ public void init(ContextRefreshedEvent ignore) { fromCreated(); fromQueued(); fromRunning(); - fromRunningPost(); fromCancelling(); sm.addHookActionOnTargetStatus(new ActionOnFinishStatus(), Success, Failure, Timeout, Cancelled); @@ -276,19 +267,19 @@ public void toContinue(Job job, Step step) { return; } - if (job.isRunningPost()) { - on(job, Job.Status.RUNNING_POST, (context) -> context.step = step); - return; - } - on(job, Job.Status.RUNNING, (context) -> context.step = step); } @Override - public void toCancelled(Job job, String reason) { + public void toCancelled(Job job, CIException exception) { on(job, Job.Status.CANCELLED, context -> { - context.setError(new CIException(reason)); - context.callFromMethod = true; + context.setError(exception); + + Set currentPath = job.getCurrentPath(); + if (!currentPath.isEmpty()) { + String nodePath = currentPath.iterator().next(); + context.step = stepService.get(job.getId(), nodePath); + } }); } @@ -494,7 +485,7 @@ public void accept(JobSmContext context) throws Exception { return; } - if (toNextStep(job, step, false)) { + if (toNextStep(job, step)) { return; } @@ -526,7 +517,7 @@ public void accept(JobSmContext context) { @Override public void accept(JobSmContext context) { Job job = context.job; - killOngoingSteps(job, false); + killOngoingSteps(job, job.isOnPostSteps()); } @Override @@ -540,13 +531,13 @@ public void onException(Throwable e, JobSmContext context) { // do not lock job since it will be called from RunningToRunning status sm.add(RunningToFailure, new Action() { @Override - public void accept(JobSmContext context) { + public void accept(JobSmContext context) throws ScriptException { Job job = context.job; Step step = context.step; stepService.toStatus(step, Step.Status.EXCEPTION, null, false); - killOngoingSteps(job, false); - toRunningPostStatusIfNeeded(context); + killOngoingSteps(job, job.isOnPostSteps()); + runPostStepsIfNeeded(context); } @Override @@ -561,7 +552,7 @@ public void onException(Throwable e, JobSmContext context) { public void accept(JobSmContext context) { Job job = context.job; setJobStatusAndSave(job, Job.Status.CANCELLING, null); - killOngoingSteps(job, false); + killOngoingSteps(job, job.isOnPostSteps()); } @Override @@ -577,14 +568,18 @@ public boolean canRun(JobSmContext context) { } @Override - public void accept(JobSmContext context) { + public void accept(JobSmContext context) throws ScriptException { Job job = context.job; + + List steps = stepService.list(job, Sets.newHashSet(WAITING_AGENT)); + stepService.toStatus(steps, Step.Status.SKIPPED, null); + JobAgent jobAgent = getJobAgent(job.getId()); - List ongoingSteps = stepService.list(job, Executed.OngoingStatus); + steps = stepService.list(job, Sets.newHashSet(RUNNING)); // no busy agents, run post steps directly if needed - if (jobAgent.allBusyAgents(ongoingSteps).isEmpty()) { - toRunningPostStatusIfNeeded(context); + if (getBusyAgents(jobAgent, steps).isEmpty()) { + runPostStepsIfNeeded(context); return; } @@ -594,7 +589,7 @@ public void accept(JobSmContext context) { @Override public void onException(Throwable e, JobSmContext context) { Job job = context.job; - killOngoingSteps(job, false); + killOngoingSteps(job, job.isOnPostSteps()); setJobStatusAndSave(job, Job.Status.CANCELLED, e.getMessage()); } @@ -603,86 +598,6 @@ public void onFinally(JobSmContext context) { unlockJobAfter(context); } }); - - // from RunningToFailure or RunningToCancelled - sm.add(RunningToRunningPost, new Action() { - @Override - public void accept(JobSmContext context) throws Exception { - doFromCancellingOrRunningToRunningPost(context); - } - - @Override - public void onException(Throwable e, JobSmContext context) { - Job job = context.job; - setJobStatusAndSave(job, Job.Status.FAILURE, e.getMessage()); - } - }); - } - - private void fromRunningPost() { - sm.add(RunningPostToRunningPost, new Action() { - @Override - public boolean canRun(JobSmContext context) { - return lockJobBefore(context); - } - - @Override - public void accept(JobSmContext context) throws Exception { - Job job = context.job; - Step step = context.step; - - updateJobContextAndLatestStatus(job, step); - setJobStatusAndSave(job, Job.Status.RUNNING_POST, null); - log.debug("Step {} been recorded", step.getNodePath()); - - if (releaseAgentOrAssignToWaitingStep(job, step)) { - return; - } - - if (toNextStep(job, step, true)) { - return; - } - - toFinishStatus(context); - } - - @Override - public void onFinally(JobSmContext context) { - unlockJobAfter(context); - } - }); - - Action toActualStatusAction = new Action() { - @Override - public void accept(JobSmContext context) throws Exception { - logInfo(context.job, "RunningPost to {}", context.getTargetToJobStatus()); - } - }; - - sm.add(RunningPostToFailure, toActualStatusAction); - sm.add(RunningPostToTimeout, toActualStatusAction); - sm.add(RunningPostToCanceled, new Action() { - @Override - public void accept(JobSmContext context) throws Exception { - if (!context.callFromMethod) { - return; - } - sm.execute(context.getCurrent(), Cancelling, context); - } - }); - - sm.add(RunningPostToCancelling, new Action() { - @Override - public void accept(JobSmContext context) throws Exception { - if (!context.callFromMethod) { - sm.execute(context.getCurrent(), Cancelled, context); - return; - } - - Job job = context.job; - killOngoingSteps(job, true); - } - }); } private void fromCancelling() { @@ -693,16 +608,9 @@ public boolean canRun(JobSmContext context) { } @Override - public void accept(JobSmContext context) { + public void accept(JobSmContext context) throws ScriptException { Job job = context.job; - Step step = context.step; - - if (step == null) { - setJobStatusAndSave(job, Job.Status.CANCELLED, null); - return; - } - - if (!toRunningPostStatusIfNeeded(context)) { + if (!runPostStepsIfNeeded(context)) { setJobStatusAndSave(job, Job.Status.CANCELLED, null); } } @@ -713,11 +621,11 @@ public void onFinally(JobSmContext context) { } }); - // from CancellingToCancelled - sm.add(CancellingToRunningPost, new Action() { + // To run post steps + sm.add(CancellingToRunning, new Action() { @Override public void accept(JobSmContext context) throws Exception { - doFromCancellingOrRunningToRunningPost(context); + sm.executeInExecutor(Running, context.getTo(), context); } }); } @@ -741,20 +649,6 @@ private boolean releaseAgentOrAssignToWaitingStep(Job job, Step step) { return false; } - private void doFromCancellingOrRunningToRunningPost(JobSmContext context) throws ScriptException { - log.debug("---- CancellingOrRunningToRunningPost ----"); - - Job job = context.job; - Step step = context.step; - - NodeTree tree = ymlManager.getTree(job); - List post = tree.post(step.getNodePath()); - - // DO NOT check post size, it has been checked in previous - job.setStatus(Job.Status.RUNNING_POST); - executeJob(job, post); - } - private boolean lockJobBefore(JobSmContext context) { Job job = context.job; @@ -799,6 +693,33 @@ private boolean waitIfJobNotOnTopPriority(Job job) { } } + /** + * All busy agents, which are occupied by flow and assigned to step + */ + private Collection getBusyAgents(JobAgent jobAgent, Collection ongoingSteps) { + Map> agents = jobAgent.getAgents(); + Set busy = new HashSet<>(agents.size()); + + agents.forEach((agentId, v) -> { + if (v.isEmpty()) { + return; + } + + Agent agent = agentService.get(agentId); + if (!agent.isBusy()) { + return; + } + + for (Step s : ongoingSteps) { + if (s.hasAgent() && s.getAgentId().equals(agentId)) { + busy.add(agent); + return; + } + } + }); + return busy; + } + private Optional fetchAgentFromJob(Job job, Node node) { FlowNode flow = node.getParent(FlowNode.class); Selector selector = flow.fetchSelector(); @@ -1021,17 +942,16 @@ private SimpleSecret getSimpleSecret(String credentialName) { /** * Dispatch next step to agent, job will be saved on final function of Running status * - * @param job current job + * @param job current job * @param step current step - * @param post indicate to next post steps or regular steps * @return true if next step dispatched or have to wait for previous steps, false if no more steps or failure */ - private boolean toNextStep(Job job, Step step, boolean post) throws ScriptException { + private boolean toNextStep(Job job, Step step) throws ScriptException { NodeTree tree = ymlManager.getTree(job); Node node = tree.get(NodePath.create(step.getNodePath())); // current node List next = node.getNext(); - if (post) { + if (job.isOnPostSteps()) { next = tree.post(step.getNodePath()); } @@ -1044,7 +964,7 @@ private boolean toNextStep(Job job, Step step, boolean post) throws ScriptExcept } // check prev steps status - Set previous = getStepsStatus(job, tree.prevs(next, post)); + Set previous = getStepsStatus(job, tree.prevs(next, job.isOnPostSteps())); boolean hasFailure = !Collections.disjoint(previous, Executed.FailureStatus); boolean hasOngoing = !Collections.disjoint(previous, Executed.OngoingStatus); if (hasFailure) { @@ -1130,6 +1050,7 @@ private void executeJob(Job job, List nodes) throws ScriptException { */ private boolean runCondition(Job job, Node node) throws ScriptException { boolean shouldRun = true; + if (job.getTrigger() == Job.Trigger.MANUAL || job.getTrigger() == Job.Trigger.API) { if (node.getPath().isRoot()) { shouldRun = false; @@ -1173,21 +1094,31 @@ private void toFinishStatus(JobSmContext context) { } /** - * To running post status + * To running post status, skip current state machine action and switch job status to Running * Return `true` if post step found */ - private boolean toRunningPostStatusIfNeeded(JobSmContext context) { + private boolean runPostStepsIfNeeded(JobSmContext context) throws ScriptException { + if (context.getError() == Errors.AgentOffline) { + return false; + } + Job job = context.job; Step step = context.step; + Objects.requireNonNull(step, "The step not defined when running post steps"); NodeTree tree = ymlManager.getTree(job); - List post = tree.post(step.getNodePath()); - - if (post.isEmpty()) { + List nextPostSteps = tree.post(step.getNodePath()); + if (nextPostSteps.isEmpty()) { return false; } - sm.execute(context.getCurrent(), RunningPost, context); + context.setSkip(true); + + job.setOnPostSteps(true); + job.resetCurrentPath(); + job.setStatus(Job.Status.RUNNING); + + executeJob(job, nextPostSteps); return true; } diff --git a/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java index 7420c3f05..b69995c8c 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java @@ -37,6 +37,7 @@ import com.flowci.core.job.event.TtyStatusUpdateEvent; import com.flowci.core.job.manager.JobActionManager; import com.flowci.core.job.manager.YmlManager; +import com.flowci.core.job.util.Errors; import com.flowci.tree.FlowNode; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; @@ -145,7 +146,7 @@ public void updateJobAndStepWhenOffline(AgentStatusEvent event) { } Job job = jobService.get(agent.getJobId()); - jobActionManager.toCancelled(job, "Agent unexpected offline"); + jobActionManager.toCancelled(job, Errors.AgentOffline); } @EventListener diff --git a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java index a57b20a5c..19b540908 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java @@ -198,7 +198,7 @@ public void start(Job job) { @Override public void cancel(Job job) { - jobActionManager.toCancelled(job, StringHelper.EMPTY); + jobActionManager.toCancelled(job, null); } @Override diff --git a/core/src/main/java/com/flowci/core/job/util/Errors.java b/core/src/main/java/com/flowci/core/job/util/Errors.java new file mode 100644 index 000000000..9d547d72c --- /dev/null +++ b/core/src/main/java/com/flowci/core/job/util/Errors.java @@ -0,0 +1,9 @@ +package com.flowci.core.job.util; + +import com.flowci.exception.CIException; +import com.flowci.exception.StatusException; + +public abstract class Errors { + + public final static CIException AgentOffline = new StatusException("Agent unexpected offline"); +} diff --git a/sm/src/main/java/com/flowci/sm/Context.java b/sm/src/main/java/com/flowci/sm/Context.java index 02426d682..9683fb127 100644 --- a/sm/src/main/java/com/flowci/sm/Context.java +++ b/sm/src/main/java/com/flowci/sm/Context.java @@ -13,4 +13,5 @@ public abstract class Context { protected Throwable error; + protected boolean skip; } diff --git a/sm/src/main/java/com/flowci/sm/StateMachine.java b/sm/src/main/java/com/flowci/sm/StateMachine.java index bd85f334e..21c78993d 100644 --- a/sm/src/main/java/com/flowci/sm/StateMachine.java +++ b/sm/src/main/java/com/flowci/sm/StateMachine.java @@ -67,7 +67,7 @@ public void execute(Status current, Status target, T context) { try { action.accept(context); - if (!isOnSameContext(current, target, context)) { + if (!isOnSameContext(current, target, context) || context.skip) { return; } From 56832055e15e7d026d83e001842f91ebaaa02b8e Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Wed, 14 Apr 2021 18:04:00 +0200 Subject: [PATCH 38/46] check existing lock from context when lock job --- .../flowci/core/job/domain/JobSmContext.java | 4 +++ .../job/manager/JobActionManagerImpl.java | 25 ++++++++++--------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java b/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java index 427afcca4..36c87fb9f 100644 --- a/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java +++ b/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java @@ -17,4 +17,8 @@ public Job.Status getTargetToJobStatus() { String name = this.to.getName(); return Job.Status.valueOf(name); } + + public boolean hasLock() { + return lock != null; + } } diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java index e6a5ebd82..37507cb73 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java @@ -533,9 +533,6 @@ public void onException(Throwable e, JobSmContext context) { @Override public void accept(JobSmContext context) throws ScriptException { Job job = context.job; - Step step = context.step; - stepService.toStatus(step, Step.Status.EXCEPTION, null, false); - killOngoingSteps(job, job.isOnPostSteps()); runPostStepsIfNeeded(context); } @@ -610,6 +607,7 @@ public boolean canRun(JobSmContext context) { @Override public void accept(JobSmContext context) throws ScriptException { Job job = context.job; + if (!runPostStepsIfNeeded(context)) { setJobStatusAndSave(job, Job.Status.CANCELLED, null); } @@ -620,14 +618,6 @@ public void onFinally(JobSmContext context) { unlockJobAfter(context); } }); - - // To run post steps - sm.add(CancellingToRunning, new Action() { - @Override - public void accept(JobSmContext context) throws Exception { - sm.executeInExecutor(Running, context.getTo(), context); - } - }); } /** @@ -652,6 +642,11 @@ private boolean releaseAgentOrAssignToWaitingStep(Job job, Step step) { private boolean lockJobBefore(JobSmContext context) { Job job = context.job; + if (context.hasLock()) { + log.debug("Job {} has lock from other context", job.getId()); + return true; + } + Optional lock = lock(job.getId(), "LockJobBefore"); if (!lock.isPresent()) { @@ -666,9 +661,14 @@ private boolean lockJobBefore(JobSmContext context) { } private void unlockJobAfter(JobSmContext context) { - Job job = context.job; InterLock lock = context.lock; + if (lock == null) { + return; + } + + Job job = context.job; unlock(lock, job.getId()); + context.lock = null; } private boolean waitIfJobNotOnTopPriority(Job job) { @@ -1117,6 +1117,7 @@ private boolean runPostStepsIfNeeded(JobSmContext context) throws ScriptExceptio job.setOnPostSteps(true); job.resetCurrentPath(); job.setStatus(Job.Status.RUNNING); + job.setStatusToContext(Job.Status.valueOf(context.getTo().getName())); executeJob(job, nextPostSteps); return true; From b0efda5c5dd31991dd509a73e9b24626c197ed60 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Wed, 14 Apr 2021 22:23:07 +0200 Subject: [PATCH 39/46] mv job action to service layer --- .../JobActionService.java} | 4 ++-- .../JobActionServiceImpl.java} | 13 ++++++------- .../core/job/service/JobEventServiceImpl.java | 11 +++++------ .../flowci/core/job/service/JobServiceImpl.java | 17 ++++++++--------- .../flowci/core/test/job/JobServiceTest.java | 8 ++++---- 5 files changed, 25 insertions(+), 28 deletions(-) rename core/src/main/java/com/flowci/core/job/{manager/JobActionManager.java => service/JobActionService.java} (83%) rename core/src/main/java/com/flowci/core/job/{manager/JobActionManagerImpl.java => service/JobActionServiceImpl.java} (98%) diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManager.java b/core/src/main/java/com/flowci/core/job/service/JobActionService.java similarity index 83% rename from core/src/main/java/com/flowci/core/job/manager/JobActionManager.java rename to core/src/main/java/com/flowci/core/job/service/JobActionService.java index fa77b9f87..ec7138c47 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManager.java +++ b/core/src/main/java/com/flowci/core/job/service/JobActionService.java @@ -1,10 +1,10 @@ -package com.flowci.core.job.manager; +package com.flowci.core.job.service; import com.flowci.core.job.domain.Step; import com.flowci.core.job.domain.Job; import com.flowci.exception.CIException; -public interface JobActionManager { +public interface JobActionService { void toLoading(Job job); diff --git a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java b/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java similarity index 98% rename from core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java rename to core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java index 37507cb73..20ba3e569 100644 --- a/core/src/main/java/com/flowci/core/job/manager/JobActionManagerImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.flowci.core.job.manager; +package com.flowci.core.job.service; import com.flowci.core.agent.domain.Agent; import com.flowci.core.agent.domain.AgentProfile; @@ -34,8 +34,8 @@ import com.flowci.core.job.domain.*; import com.flowci.core.job.event.JobReceivedEvent; import com.flowci.core.job.event.JobStatusChangeEvent; -import com.flowci.core.job.service.LocalTaskService; -import com.flowci.core.job.service.StepService; +import com.flowci.core.job.manager.CmdManager; +import com.flowci.core.job.manager.YmlManager; import com.flowci.core.job.util.Errors; import com.flowci.core.job.util.StatusHelper; import com.flowci.core.secret.domain.Secret; @@ -75,7 +75,7 @@ @Log4j2 @Service -public class JobActionManagerImpl implements JobActionManager { +public class JobActionServiceImpl implements JobActionService { private static final Status Pending = new Status(Job.Status.PENDING.name()); private static final Status Created = new Status(Job.Status.CREATED.name()); @@ -112,13 +112,12 @@ public class JobActionManagerImpl implements JobActionManager { private static final Transition RunningToRunning = new Transition(Running, Running); private static final Transition RunningToSuccess = new Transition(Running, Success); private static final Transition RunningToCancelling = new Transition(Running, Cancelling); - private static final Transition RunningToCanceled = new Transition(Running, Cancelled); + private static final Transition RunningToCancelled = new Transition(Running, Cancelled); private static final Transition RunningToTimeout = new Transition(Running, Timeout); private static final Transition RunningToFailure = new Transition(Running, Failure); // cancelling private static final Transition CancellingToCancelled = new Transition(Cancelling, Cancelled); - private static final Transition CancellingToRunning = new Transition(Cancelling, Running); private static final long RetryInterval = 10 * 1000; // 10 seconds @@ -558,7 +557,7 @@ public void onException(Throwable e, JobSmContext context) { } }); - sm.add(RunningToCanceled, new Action() { + sm.add(RunningToCancelled, new Action() { @Override public boolean canRun(JobSmContext context) { return lockJobBefore(context); diff --git a/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java index b69995c8c..11472da69 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java @@ -35,7 +35,6 @@ import com.flowci.core.job.domain.Step; import com.flowci.core.job.event.CreateNewJobEvent; import com.flowci.core.job.event.TtyStatusUpdateEvent; -import com.flowci.core.job.manager.JobActionManager; import com.flowci.core.job.manager.YmlManager; import com.flowci.core.job.util.Errors; import com.flowci.tree.FlowNode; @@ -66,7 +65,7 @@ public class JobEventServiceImpl implements JobEventService { private RabbitOperations jobsQueueManager; @Autowired - private JobActionManager jobActionManager; + private JobActionService jobActionService; @Autowired private ConditionManager conditionManager; @@ -146,7 +145,7 @@ public void updateJobAndStepWhenOffline(AgentStatusEvent event) { } Job job = jobService.get(agent.getJobId()); - jobActionManager.toCancelled(job, Errors.AgentOffline); + jobActionService.toCancelled(job, Errors.AgentOffline); } @EventListener @@ -184,7 +183,7 @@ public void handleCmdOutFromAgent(OnCmdOutEvent event) { @Override public void handleCallback(Step step) { Job job = jobService.get(step.getJobId()); - jobActionManager.toContinue(job, step); + jobActionService.toContinue(job, step); } //==================================================================== @@ -198,7 +197,7 @@ public void startJobDeadLetterConsumer() throws IOException { String jobId = new String(body); try { Job job = jobService.get(jobId); - jobActionManager.toTimeout(job); + jobActionService.toTimeout(job); } catch (Exception e) { log.warn(e); } @@ -224,7 +223,7 @@ private void declareJobQueueAndStartConsumer(Flow flow) { try { Job job = jobService.get(jobId); logInfo(job, "received from queue"); - jobActionManager.toRun(job); + jobActionService.toRun(job); } catch (Exception e) { log.warn(e); } diff --git a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java index 19b540908..bd1f12dfe 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java @@ -26,7 +26,6 @@ import com.flowci.core.job.domain.Job.Trigger; import com.flowci.core.job.event.JobCreatedEvent; import com.flowci.core.job.event.JobDeletedEvent; -import com.flowci.core.job.manager.JobActionManager; import com.flowci.core.job.manager.YmlManager; import com.flowci.core.user.domain.User; import com.flowci.domain.StringVars; @@ -105,7 +104,7 @@ public class JobServiceImpl implements JobService { private FileManager fileManager; @Autowired - private JobActionManager jobActionManager; + private JobActionService jobActionService; @Autowired private StepService stepService; @@ -179,7 +178,7 @@ public Job create(Flow flow, String yml, Trigger trigger, StringVars input) { eventManager.publish(new JobCreatedEvent(this, job)); if (job.isYamlFromRepo()) { - jobActionManager.toLoading(job); + jobActionService.toLoading(job); return job; } @@ -187,18 +186,18 @@ public Job create(Flow flow, String yml, Trigger trigger, StringVars input) { throw new ArgumentException("YAML config is required to start a job"); } - jobActionManager.toCreated(job, yml); + jobActionService.toCreated(job, yml); return job; } @Override public void start(Job job) { - jobActionManager.toStart(job); + jobActionService.toStart(job); } @Override public void cancel(Job job) { - jobActionManager.toCancelled(job, null); + jobActionService.toCancelled(job, null); } @Override @@ -242,8 +241,8 @@ public Job rerun(Flow flow, Job job) { localTaskService.delete(job); ymlManager.delete(job); - jobActionManager.toCreated(job, yml.getRaw()); - jobActionManager.toStart(job); + jobActionService.toCreated(job, yml.getRaw()); + jobActionService.toStart(job); return job; } @@ -270,7 +269,7 @@ public Job rerunFromFailureStep(Flow flow, Job job) { // reset job agent jobAgentDao.save(new JobAgent(job.getId(), flow.getId())); - jobActionManager.toStart(job); + jobActionService.toStart(job); return job; } diff --git a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java index 7f8065259..fec50786d 100644 --- a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java +++ b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java @@ -40,7 +40,7 @@ import com.flowci.core.job.event.JobReceivedEvent; import com.flowci.core.job.event.JobStatusChangeEvent; import com.flowci.core.job.event.StartAsyncLocalTaskEvent; -import com.flowci.core.job.manager.JobActionManager; +import com.flowci.core.job.service.JobActionService; import com.flowci.core.job.manager.YmlManager; import com.flowci.core.job.service.JobEventService; import com.flowci.core.job.service.JobService; @@ -117,7 +117,7 @@ public class JobServiceTest extends ZookeeperScenario { private AgentService agentService; @Autowired - private JobActionManager jobActionManager; + private JobActionService jobActionService; @Autowired private YmlManager ymlManager; @@ -204,7 +204,7 @@ public void should_start_new_job() throws Throwable { Assert.assertEquals(Status.CREATED, job.getStatus()); Assert.assertTrue(job.getCurrentPath().isEmpty()); - jobActionManager.toStart(job); + jobActionService.toStart(job); Assert.assertEquals(Status.QUEUED, job.getStatus()); Assert.assertNotNull(job); @@ -289,7 +289,7 @@ public void should_finish_whole_job() throws InterruptedException, IOException { } }); - jobActionManager.toStart(job); + jobActionService.toStart(job); Assert.assertTrue(counterForStep1.await(10, TimeUnit.SECONDS)); // then: verify step 1 agent From f964682891d0b68023b51a7ee22e1ee5191c4964 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Mon, 3 May 2021 21:54:19 +0200 Subject: [PATCH 40/46] handle post job --- .../core/agent/manager/AgentEventManager.java | 2 +- .../com/flowci/core/job/dao/ExecutedCmdDao.java | 6 ++++-- .../java/com/flowci/core/job/domain/Step.java | 5 +++++ .../core/job/service/JobActionServiceImpl.java | 15 +++++++++++++++ .../com/flowci/core/job/service/StepService.java | 5 +++++ .../flowci/core/job/service/StepServiceImpl.java | 9 +++++++-- 6 files changed, 37 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java b/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java index 0574d7bad..9601747b2 100644 --- a/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java +++ b/core/src/main/java/com/flowci/core/agent/manager/AgentEventManager.java @@ -166,8 +166,8 @@ private void onConnected(WebSocketSession session, String token, byte[] body) { private void onCmdOut(String token, byte[] body) { try { - eventManager.publish(new OnCmdOutEvent(this, body)); log.debug("Agent {} got cmd back: {}", token, new String(body)); + eventManager.publish(new OnCmdOutEvent(this, body)); } catch (Exception e) { log.warn(e); } diff --git a/core/src/main/java/com/flowci/core/job/dao/ExecutedCmdDao.java b/core/src/main/java/com/flowci/core/job/dao/ExecutedCmdDao.java index c3ca25e3c..f91841235 100644 --- a/core/src/main/java/com/flowci/core/job/dao/ExecutedCmdDao.java +++ b/core/src/main/java/com/flowci/core/job/dao/ExecutedCmdDao.java @@ -37,7 +37,9 @@ public interface ExecutedCmdDao extends MongoRepository { Optional findByJobIdAndNodePath(String jobId, String nodePath); - List findByFlowIdAndBuildNumber(String flowId, long buildNumber); + List findAllByJobIdAndNodePathIn(String jobId, Collection nodePaths); - List findByJobIdAndStatusIn(String jobId, Collection statuses); + List findAllByFlowIdAndBuildNumber(String flowId, long buildNumber); + + List findAllByJobIdAndStatusIn(String jobId, Collection statuses); } diff --git a/core/src/main/java/com/flowci/core/job/domain/Step.java b/core/src/main/java/com/flowci/core/job/domain/Step.java index 066531d5d..cb63b7db5 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Step.java +++ b/core/src/main/java/com/flowci/core/job/domain/Step.java @@ -168,6 +168,11 @@ public boolean isOngoing() { return OngoingStatus.contains(status); } + @JsonIgnore + public boolean isKilling() { + return status == Status.KILLING; + } + public void setFrom(ShellOut out) { this.processId = out.getProcessId(); this.containerId = out.getContainerId(); diff --git a/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java index 20ba3e569..71e06e70c 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java @@ -1105,12 +1105,26 @@ private boolean runPostStepsIfNeeded(JobSmContext context) throws ScriptExceptio Step step = context.step; Objects.requireNonNull(step, "The step not defined when running post steps"); + if (job.isOnPostSteps()) { + return true; + } + NodeTree tree = ymlManager.getTree(job); List nextPostSteps = tree.post(step.getNodePath()); if (nextPostSteps.isEmpty()) { return false; } + // check current all steps that should be in finish status + List steps = stepService.listByPath(job, job.getCurrentPath()); + for (Step s : steps) { + if (s.isOngoing() || s.isKilling()) { + log.debug("Step ({} = {}) is ongoing or killing status", s.getNodePath(), s.getStatus()); + return false; + } + } + + log.debug("All current steps finished, will run post steps"); context.setSkip(true); job.setOnPostSteps(true); @@ -1118,6 +1132,7 @@ private boolean runPostStepsIfNeeded(JobSmContext context) throws ScriptExceptio job.setStatus(Job.Status.RUNNING); job.setStatusToContext(Job.Status.valueOf(context.getTo().getName())); + log.debug("Run post steps: {}", nextPostSteps); executeJob(job, nextPostSteps); return true; } diff --git a/core/src/main/java/com/flowci/core/job/service/StepService.java b/core/src/main/java/com/flowci/core/job/service/StepService.java index e4df688bf..24f2a4d8d 100644 --- a/core/src/main/java/com/flowci/core/job/service/StepService.java +++ b/core/src/main/java/com/flowci/core/job/service/StepService.java @@ -54,6 +54,11 @@ public interface StepService { */ List list(Job job, Collection status); + /** + * List steps by given paths + */ + List listByPath(Job job, Collection paths); + /** * Get step list in string, {name}={stats};{name}={stats} * No steps after current node diff --git a/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java index 060f78b2c..fa09ba727 100644 --- a/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/StepServiceImpl.java @@ -98,7 +98,12 @@ public List list(Job job) { @Override public List list(Job job, Collection status) { - return executedCmdDao.findByJobIdAndStatusIn(job.getId(), status); + return executedCmdDao.findAllByJobIdAndStatusIn(job.getId(), status); + } + + @Override + public List listByPath(Job job, Collection paths) { + return executedCmdDao.findAllByJobIdAndNodePathIn(job.getId(), paths); } @Override @@ -229,7 +234,7 @@ private Step saveStatus(Step step, Step.Status status, String error) { private List list(String jobId, String flowId, long buildNumber) { return jobStepCache.get(jobId, - s -> executedCmdDao.findByFlowIdAndBuildNumber(flowId, buildNumber)); + s -> executedCmdDao.findAllByFlowIdAndBuildNumber(flowId, buildNumber)); } private static Step newInstance(Job job, Node node) { From af6248d6274f978a61c4834f02e4f9c8b8ca9947 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Tue, 4 May 2021 17:49:58 +0200 Subject: [PATCH 41/46] support transaction on job action --- .../flowci/core/job/domain/JobSmContext.java | 16 +- .../core/job/service/JobActionService.java | 15 +- .../job/service/JobActionServiceImpl.java | 287 ++++++++---------- .../core/job/service/JobEventServiceImpl.java | 15 +- .../core/job/service/JobServiceImpl.java | 14 +- .../flowci/core/test/job/JobServiceTest.java | 4 +- .../main/java/com/flowci/sm/StateMachine.java | 12 +- 7 files changed, 168 insertions(+), 195 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java b/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java index 36c87fb9f..8e807b44d 100644 --- a/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java +++ b/core/src/main/java/com/flowci/core/job/domain/JobSmContext.java @@ -2,16 +2,24 @@ import com.flowci.sm.Context; import com.flowci.zookeeper.InterLock; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +@Getter +@Setter +@RequiredArgsConstructor public class JobSmContext extends Context { - public Job job; + private final String jobId; - public String yml; + private Job job; - public Step step; + private String yml; - public InterLock lock; + private Step step; + + private InterLock lock; public Job.Status getTargetToJobStatus() { String name = this.to.getName(); diff --git a/core/src/main/java/com/flowci/core/job/service/JobActionService.java b/core/src/main/java/com/flowci/core/job/service/JobActionService.java index ec7138c47..9cf062319 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobActionService.java +++ b/core/src/main/java/com/flowci/core/job/service/JobActionService.java @@ -1,22 +1,21 @@ package com.flowci.core.job.service; import com.flowci.core.job.domain.Step; -import com.flowci.core.job.domain.Job; import com.flowci.exception.CIException; public interface JobActionService { - void toLoading(Job job); + void toLoading(String jobId); - void toCreated(Job job, String yml); + void toCreated(String jobId, String yml); - void toStart(Job job); + void toStart(String jobId); - void toRun(Job job); + void toRun(String jobId); - void toContinue(Job job, Step step); + void toContinue(Step step); - void toCancelled(Job job, CIException exception); + void toCancelled(String jobId, CIException exception); - void toTimeout(Job job); + void toTimeout(String jobId); } diff --git a/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java index 71e06e70c..43b30340f 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java @@ -59,7 +59,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Service; import java.io.IOException; @@ -67,7 +66,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; import static com.flowci.core.job.domain.Executed.Status.RUNNING; @@ -171,9 +169,6 @@ public class JobActionServiceImpl implements JobActionService { @Autowired private StateMachine sm; - // job node execute thread pool - private final Map pool = new ConcurrentHashMap<>(); - @EventListener public void init(ContextRefreshedEvent ignore) { try { @@ -214,7 +209,7 @@ public void onIdleAgent(IdleAgentEvent event) { Optional lock = lock(job.getId(), "LockJobFromIdleAgent"); if (!lock.isPresent()) { - toFailureStatus(job, null, new CIException("Fail to lock job")); + toFailureStatus(job, new CIException("Fail to lock job")); continue; } @@ -227,7 +222,7 @@ public void onIdleAgent(IdleAgentEvent event) { return; } } catch (Exception e) { - toFailureStatus(job, null, new CIException(e.getMessage())); + toFailureStatus(job, new CIException(e.getMessage())); } finally { unlock(lock.get(), "LockJobFromIdleAgent"); } @@ -235,73 +230,76 @@ public void onIdleAgent(IdleAgentEvent event) { } @Override - public void toLoading(Job job) { - on(job, Job.Status.LOADING, null); + public void toLoading(String jobId) { + onTransition(jobId, Loading, null); } @Override - public void toCreated(Job job, String yml) { - on(job, Job.Status.CREATED, context -> { - context.yml = yml; + public void toCreated(String jobId, String yml) { + onTransition(jobId, Created, context -> { + context.setYml(yml); }); } @Override - public void toStart(Job job) { - on(job, Job.Status.QUEUED, null); + public void toStart(String jobId) { + onTransition(jobId, Queued, null); } @Override - public void toRun(Job job) { - if (job.isDone()) { - return; - } - on(job, Job.Status.RUNNING, null); + public void toRun(String jobId) { + onTransition(jobId, Running, null); } @Override - public void toContinue(Job job, Step step) { - if (job.isCancelling()) { - on(job, Job.Status.CANCELLED, (context) -> context.step = step); - return; - } + public void toContinue(Step step) { + onTransition(step.getJobId(), Running, c -> { + c.setStep(step); + + Job job = c.getJob(); + log.debug("---- Job Status {} {} {} {}", job.isOnPostSteps(), step.getNodePath(), job.getStatus(), job.getStatusFromContext()); - on(job, Job.Status.RUNNING, (context) -> context.step = step); + if (job.isCancelling()) { + c.setTo(Cancelled); + } + }); } @Override - public void toCancelled(Job job, CIException exception) { - on(job, Job.Status.CANCELLED, context -> { + public void toCancelled(String jobId, CIException exception) { + onTransition(jobId, Cancelled, context -> { context.setError(exception); + Job job = context.getJob(); Set currentPath = job.getCurrentPath(); + if (!currentPath.isEmpty()) { String nodePath = currentPath.iterator().next(); - context.step = stepService.get(job.getId(), nodePath); + context.setStep(stepService.get(job.getId(), nodePath)); } }); } @Override - public void toTimeout(Job job) { - on(job, Job.Status.TIMEOUT, null); + public void toTimeout(String jobId) { + onTransition(jobId, Timeout, null); } private void fromPending() { - sm.add(PendingToCreated, new Action() { + sm.add(PendingToCreated, new JobActionBase() { @Override public void accept(JobSmContext context) { doFromXToCreated(context); } }); - sm.add(PendingToLoading, new Action() { + sm.add(PendingToLoading, new JobActionBase() { @Override public void accept(JobSmContext context) { - Job job = context.job; + Job job = context.getJob(); setJobStatusAndSave(job, Job.Status.LOADING, null); - context.yml = fetchYamlFromGit(job); + context.setYml(fetchYamlFromGit(job)); sm.execute(Loading, Created, context); } @@ -312,7 +310,7 @@ public void onException(Throwable e, JobSmContext context) { } }); - sm.add(PendingToCancelled, new Action() { + sm.add(PendingToCancelled, new JobActionBase() { @Override public void accept(JobSmContext context) { context.setError(new Exception("cancelled while pending")); @@ -321,14 +319,14 @@ public void accept(JobSmContext context) { } private void fromLoading() { - sm.add(LoadingToFailure, new Action() { + sm.add(LoadingToFailure, new JobActionBase() { @Override public void accept(JobSmContext context) { // handled on ActionOnFinishStatus } }); - sm.add(LoadingToCreated, new Action() { + sm.add(LoadingToCreated, new JobActionBase() { @Override public void accept(JobSmContext context) { doFromXToCreated(context); @@ -337,34 +335,34 @@ public void accept(JobSmContext context) { } private void doFromXToCreated(JobSmContext context) { - Job job = context.job; - String yml = context.yml; + Job job = context.getJob(); + String yml = context.getYml(); setupJobYamlAndSteps(job, yml); setJobStatusAndSave(job, Job.Status.CREATED, StringHelper.EMPTY); } private void fromCreated() { - sm.add(CreatedToTimeout, new Action() { + sm.add(CreatedToTimeout, new JobActionBase() { @Override public void accept(JobSmContext context) { - Job job = context.job; + Job job = context.getJob(); context.setError(new Exception("expired before enqueue")); log.debug("[Job: Timeout] {} has expired", job.getKey()); } }); - sm.add(CreatedToFailure, new Action() { + sm.add(CreatedToFailure, new JobActionBase() { @Override public void accept(JobSmContext context) { // handled on ActionOnFinishStatus } }); - sm.add(CreatedToQueued, new Action() { + sm.add(CreatedToQueued, new JobActionBase() { @Override public void accept(JobSmContext context) { - Job job = context.job; + Job job = context.getJob(); setJobStatusAndSave(job, Job.Status.QUEUED, null); String queue = job.getQueueName(); @@ -383,14 +381,14 @@ public void onException(Throwable e, JobSmContext context) { } private void fromQueued() { - sm.add(QueuedToTimeout, new Action() { + sm.add(QueuedToTimeout, new JobActionBase() { @Override public void accept(JobSmContext context) { // handled on ActionOnFinishStatus } }); - sm.add(QueuedToCancelled, new Action() { + sm.add(QueuedToCancelled, new JobActionBase() { @Override public void accept(JobSmContext context) { context.setError(new Exception("cancelled from queue")); @@ -398,19 +396,20 @@ public void accept(JobSmContext context) { } }); - sm.add(QueuedToRunning, new Action() { + sm.add(QueuedToRunning, new JobActionBase() { + @Override public boolean canRun(JobSmContext context) { - return lockJobBefore(context); + return !context.getJob().isDone(); } @Override public void accept(JobSmContext context) throws Exception { - Job job = context.job; + Job job = context.getJob(); eventManager.publish(new JobReceivedEvent(this, job)); jobPriorityDao.addJob(job.getFlowId(), job.getBuildNumber()); - if (!waitIfJobNotOnTopPriority(job)) { + if (!waitIfJobNotOnTopPriority(context)) { return; } @@ -438,17 +437,12 @@ public void onException(Throwable e, JobSmContext context) { context.setError(e); sm.execute(Queued, Failure, context); } - - @Override - public void onFinally(JobSmContext context) { - unlockJobAfter(context); - } }); - sm.add(QueuedToFailure, new Action() { + sm.add(QueuedToFailure, new JobActionBase() { @Override public void accept(JobSmContext context) { - Job job = context.job; + Job job = context.getJob(); // set current step to exception for (String path : job.getCurrentPath()) { @@ -460,16 +454,16 @@ public void accept(JobSmContext context) { } private void fromRunning() { - sm.add(RunningToRunning, new Action() { + sm.add(RunningToRunning, new JobActionBase() { @Override public boolean canRun(JobSmContext context) { - return lockJobBefore(context); + return !context.getJob().isDone(); } @Override public void accept(JobSmContext context) throws Exception { - Job job = context.job; - Step step = context.step; + Job job = context.getJob(); + Step step = context.getStep(); updateJobContextAndLatestStatus(job, step); setJobStatusAndSave(job, Job.Status.RUNNING, null); @@ -496,57 +490,52 @@ public void onException(Throwable e, JobSmContext context) { context.setError(e); sm.execute(context.getCurrent(), Failure, context); } - - @Override - public void onFinally(JobSmContext context) { - unlockJobAfter(context); - } }); // do not lock job since it will be called from RunningToRunning status - sm.add(RunningToSuccess, new Action() { + sm.add(RunningToSuccess, new JobActionBase() { @Override public void accept(JobSmContext context) { - Job job = context.job; + Job job = context.getJob(); logInfo(job, "finished with status {}", Success); } }); - sm.add(RunningToTimeout, new Action() { + sm.add(RunningToTimeout, new JobActionBase() { @Override public void accept(JobSmContext context) { - Job job = context.job; + Job job = context.getJob(); killOngoingSteps(job, job.isOnPostSteps()); } @Override public void onException(Throwable e, JobSmContext context) { - Job job = context.job; + Job job = context.getJob(); setJobStatusAndSave(job, Job.Status.TIMEOUT, null); } }); // failure from job end or exception // do not lock job since it will be called from RunningToRunning status - sm.add(RunningToFailure, new Action() { + sm.add(RunningToFailure, new JobActionBase() { @Override public void accept(JobSmContext context) throws ScriptException { - Job job = context.job; + Job job = context.getJob(); killOngoingSteps(job, job.isOnPostSteps()); runPostStepsIfNeeded(context); } @Override public void onException(Throwable e, JobSmContext context) { - Job job = context.job; + Job job = context.getJob(); setJobStatusAndSave(job, Job.Status.FAILURE, e.getMessage()); } }); - sm.add(RunningToCancelling, new Action() { + sm.add(RunningToCancelling, new JobActionBase() { @Override public void accept(JobSmContext context) { - Job job = context.job; + Job job = context.getJob(); setJobStatusAndSave(job, Job.Status.CANCELLING, null); killOngoingSteps(job, job.isOnPostSteps()); } @@ -557,15 +546,10 @@ public void onException(Throwable e, JobSmContext context) { } }); - sm.add(RunningToCancelled, new Action() { - @Override - public boolean canRun(JobSmContext context) { - return lockJobBefore(context); - } - + sm.add(RunningToCancelled, new JobActionBase() { @Override public void accept(JobSmContext context) throws ScriptException { - Job job = context.job; + Job job = context.getJob(); List steps = stepService.list(job, Sets.newHashSet(WAITING_AGENT)); stepService.toStatus(steps, Step.Status.SKIPPED, null); @@ -584,38 +568,22 @@ public void accept(JobSmContext context) throws ScriptException { @Override public void onException(Throwable e, JobSmContext context) { - Job job = context.job; + Job job = context.getJob(); killOngoingSteps(job, job.isOnPostSteps()); setJobStatusAndSave(job, Job.Status.CANCELLED, e.getMessage()); } - - @Override - public void onFinally(JobSmContext context) { - unlockJobAfter(context); - } }); } private void fromCancelling() { - sm.add(CancellingToCancelled, new Action() { - @Override - public boolean canRun(JobSmContext context) { - return lockJobBefore(context); - } - + sm.add(CancellingToCancelled, new JobActionBase() { @Override public void accept(JobSmContext context) throws ScriptException { - Job job = context.job; - + Job job = context.getJob(); if (!runPostStepsIfNeeded(context)) { setJobStatusAndSave(job, Job.Status.CANCELLED, null); } } - - @Override - public void onFinally(JobSmContext context) { - unlockJobAfter(context); - } }); } @@ -638,42 +606,13 @@ private boolean releaseAgentOrAssignToWaitingStep(Job job, Step step) { return false; } - private boolean lockJobBefore(JobSmContext context) { - Job job = context.job; + private boolean waitIfJobNotOnTopPriority(JobSmContext context) { + Job job = context.getJob(); - if (context.hasLock()) { - log.debug("Job {} has lock from other context", job.getId()); - return true; - } - - Optional lock = lock(job.getId(), "LockJobBefore"); - - if (!lock.isPresent()) { - toFailureStatus(context.job, context.step, new CIException("Fail to lock job")); - return false; - } - - context.lock = lock.get(); - context.job = reload(job.getId()); - log.debug("Job {} is locked", job.getId()); - return true; - } - - private void unlockJobAfter(JobSmContext context) { - InterLock lock = context.lock; - if (lock == null) { - return; - } - - Job job = context.job; - unlock(lock, job.getId()); - context.lock = null; - } - - private boolean waitIfJobNotOnTopPriority(Job job) { while (true) { if (job.isExpired()) { - on(job, Job.Status.TIMEOUT, (c) -> c.setError(new Exception("time out while queueing"))); + context.setError(new Exception("time out while queueing")); + sm.execute(context.getCurrent(), Timeout, context); return false; } @@ -687,7 +626,7 @@ private boolean waitIfJobNotOnTopPriority(Job job) { } ThreadHelper.sleep(RetryInterval); - job = reload(job.getId()); + job = getJob(job.getId()); log.debug("Job {}/{} wait since not on top priority", job.getFlowName(), job.getBuildNumber()); } } @@ -832,11 +771,13 @@ private void releaseAgentToPool(Job job, Node node, Step step) { jobAgentDao.removeAgent(job.getId(), agentId); } - private void toFailureStatus(Job job, Step step, CIException e) { - on(job, Job.Status.FAILURE, (c) -> { - c.step = step; - c.setError(e); - }); + private void toFailureStatus(Job job, CIException e) { + JobSmContext context = new JobSmContext(job.getId()); + context.setJob(job); + context.setCurrent(new Status(job.getStatus().name())); + context.setTo(Failure); + context.setError(e); + sm.execute(context); } private void setupJobYamlAndSteps(Job job, String yml) { @@ -876,18 +817,29 @@ private void killOngoingSteps(Job job, boolean includePost) { stepService.toStatus(steps, Step.Status.KILLING, null); } - private void on(Job job, Job.Status target, Consumer configContext) { - Status current = new Status(job.getStatus().name()); - Status to = new Status(target.name()); + private void onTransition(String jobId, Status to, Consumer onContext) { + Optional lock = lock(jobId, "Before Transition"); + + if (!lock.isPresent()) { + Job job = getJob(jobId); + toFailureStatus(job, new CIException("Fail to lock job")); + return; + } + + log.debug("Job {} is locked", jobId); + Job job = getJob(jobId); - JobSmContext context = new JobSmContext(); - context.job = job; + JobSmContext context = new JobSmContext(jobId); + context.setLock(lock.get()); + context.setJob(job); + context.setCurrent(new Status(job.getStatus().name())); + context.setTo(to); - if (configContext != null) { - configContext.accept(context); + if (onContext != null) { + onContext.accept(context); } - sm.execute(current, to, context); + sm.execute(context); } private String fetchYamlFromGit(Job job) { @@ -1083,7 +1035,7 @@ private void setSkipStatusToStep(Step step) { } private void toFinishStatus(JobSmContext context) { - Job job = context.job; + Job job = context.getJob(); Job.Status statusFromContext = job.getStatusFromContext(); String error = job.getErrorFromContext(); @@ -1101,8 +1053,8 @@ private boolean runPostStepsIfNeeded(JobSmContext context) throws ScriptExceptio return false; } - Job job = context.job; - Step step = context.step; + Job job = context.getJob(); + Step step = context.getStep(); Objects.requireNonNull(step, "The step not defined when running post steps"); if (job.isOnPostSteps()) { @@ -1133,6 +1085,7 @@ private boolean runPostStepsIfNeeded(JobSmContext context) throws ScriptExceptio job.setStatusToContext(Job.Status.valueOf(context.getTo().getName())); log.debug("Run post steps: {}", nextPostSteps); + log.debug("---- Job Status Before Post {} {}", job.getStatus(), job.getStatusFromContext()); executeJob(job, nextPostSteps); return true; } @@ -1193,21 +1146,21 @@ private void logInfo(Job job, String message, Object... params) { log.info("[Job] " + job.getKey() + " " + message, params); } - private Job reload(String jobId) { + private Job getJob(String jobId) { return jobDao.findById(jobId).get(); } - private Optional lock(String key, String message) { - String path = zk.makePath("/job-locks", key); + private Optional lock(String jobId, String message) { + String path = zk.makePath("/job-locks", jobId); Optional lock = zk.lock(path, DefaultJobLockTimeout); - lock.ifPresent(interLock -> log.debug("Lock: {} - {}", key, message)); + lock.ifPresent(interLock -> log.debug("Lock: {} - {}", jobId, message)); return lock; } - private void unlock(InterLock lock, String key) { + private void unlock(InterLock lock, String jobId) { try { zk.release(lock); - log.debug("Unlock: {}", key); + log.debug("Unlock: {}", jobId); } catch (Exception warn) { log.warn(warn); } @@ -1217,14 +1170,13 @@ private class ActionOnFinishStatus implements Consumer { @Override public void accept(JobSmContext context) { - Job job = context.job; + Job job = context.getJob(); // save job with status Throwable error = context.getError(); String message = error == null ? "" : error.getMessage(); setJobStatusAndSave(job, context.getTargetToJobStatus(), message); jobPriorityDao.removeJob(job.getFlowId(), job.getBuildNumber()); - pool.remove(job.getId()); JobAgent agents = getJobAgent(job.getId()); agentService.release(agents.all()); @@ -1232,4 +1184,19 @@ public void accept(JobSmContext context) { localTaskService.executeAsync(job); } } + + private abstract class JobActionBase extends Action { + + @Override + public void onFinally(JobSmContext context) { + InterLock lock = context.getLock(); + if (lock == null) { + return; + } + + Job job = context.getJob(); + unlock(lock, job.getId()); + context.setLock(null); + } + } } diff --git a/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java index 11472da69..fcc88f82a 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java @@ -144,8 +144,7 @@ public void updateJobAndStepWhenOffline(AgentStatusEvent event) { return; } - Job job = jobService.get(agent.getJobId()); - jobActionService.toCancelled(job, Errors.AgentOffline); + jobActionService.toCancelled(agent.getJobId(), Errors.AgentOffline); } @EventListener @@ -182,8 +181,7 @@ public void handleCmdOutFromAgent(OnCmdOutEvent event) { @Override public void handleCallback(Step step) { - Job job = jobService.get(step.getJobId()); - jobActionService.toContinue(job, step); + jobActionService.toContinue(step); } //==================================================================== @@ -196,8 +194,7 @@ public void startJobDeadLetterConsumer() throws IOException { jobsQueueManager.startConsumer(deadLetterQueue, true, (header, body, envelope) -> { String jobId = new String(body); try { - Job job = jobService.get(jobId); - jobActionService.toTimeout(job); + jobActionService.toTimeout(jobId); } catch (Exception e) { log.warn(e); } @@ -219,11 +216,9 @@ private void declareJobQueueAndStartConsumer(Flow flow) { jobsQueueManager.declare(queue, true, 255, rabbitProperties.getJobDlExchange()); jobsQueueManager.startConsumer(queue, false, (header, body, envelope) -> { - String jobId = new String(body); try { - Job job = jobService.get(jobId); - logInfo(job, "received from queue"); - jobActionService.toRun(job); + String jobId = new String(body); + jobActionService.toRun(jobId); } catch (Exception e) { log.warn(e); } diff --git a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java index bd1f12dfe..cd4ccedd3 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java @@ -178,7 +178,7 @@ public Job create(Flow flow, String yml, Trigger trigger, StringVars input) { eventManager.publish(new JobCreatedEvent(this, job)); if (job.isYamlFromRepo()) { - jobActionService.toLoading(job); + jobActionService.toLoading(job.getId()); return job; } @@ -186,18 +186,18 @@ public Job create(Flow flow, String yml, Trigger trigger, StringVars input) { throw new ArgumentException("YAML config is required to start a job"); } - jobActionService.toCreated(job, yml); + jobActionService.toCreated(job.getId(), yml); return job; } @Override public void start(Job job) { - jobActionService.toStart(job); + jobActionService.toStart(job.getId()); } @Override public void cancel(Job job) { - jobActionService.toCancelled(job, null); + jobActionService.toCancelled(job.getId(), null); } @Override @@ -241,8 +241,8 @@ public Job rerun(Flow flow, Job job) { localTaskService.delete(job); ymlManager.delete(job); - jobActionService.toCreated(job, yml.getRaw()); - jobActionService.toStart(job); + jobActionService.toCreated(job.getId(), yml.getRaw()); + jobActionService.toStart(job.getId()); return job; } @@ -269,7 +269,7 @@ public Job rerunFromFailureStep(Flow flow, Job job) { // reset job agent jobAgentDao.save(new JobAgent(job.getId(), flow.getId())); - jobActionService.toStart(job); + jobActionService.toStart(job.getId()); return job; } diff --git a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java index fec50786d..1263014a1 100644 --- a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java +++ b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java @@ -204,7 +204,7 @@ public void should_start_new_job() throws Throwable { Assert.assertEquals(Status.CREATED, job.getStatus()); Assert.assertTrue(job.getCurrentPath().isEmpty()); - jobActionService.toStart(job); + jobActionService.toStart(job.getId()); Assert.assertEquals(Status.QUEUED, job.getStatus()); Assert.assertNotNull(job); @@ -289,7 +289,7 @@ public void should_finish_whole_job() throws InterruptedException, IOException { } }); - jobActionService.toStart(job); + jobActionService.toStart(job.getId()); Assert.assertTrue(counterForStep1.await(10, TimeUnit.SECONDS)); // then: verify step 1 agent diff --git a/sm/src/main/java/com/flowci/sm/StateMachine.java b/sm/src/main/java/com/flowci/sm/StateMachine.java index 21c78993d..fb8a88063 100644 --- a/sm/src/main/java/com/flowci/sm/StateMachine.java +++ b/sm/src/main/java/com/flowci/sm/StateMachine.java @@ -40,13 +40,17 @@ public void add(Transition t, Action action) { map.put(t.getTo(), action); } - public void executeInExecutor(Status current, Status target, T context) { - executor.execute(() -> execute(current, target, context)); - } - public void execute(Status current, Status target, T context) { context.setCurrent(current); context.setTo(target); + execute(context); + } + + public void execute(T context) { + Status current = context.getCurrent(); + Status target = context.getTo(); + Objects.requireNonNull(current, "SM current status is missing"); + Objects.requireNonNull(target, "SM target status is missing"); Map> actionMap = actions.get(current); if (Objects.isNull(actionMap)) { From f78747c2fa34d8805be90f94392675896a591646 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Wed, 5 May 2021 17:23:50 +0200 Subject: [PATCH 42/46] fix wrong return value on post func --- .../flowci/core/agent/domain/ShellOut.java | 2 + .../core/job/service/JobActionService.java | 4 +- .../job/service/JobActionServiceImpl.java | 55 +++++++------------ .../core/job/service/JobEventService.java | 4 +- .../core/job/service/JobEventServiceImpl.java | 18 ++---- .../flowci/core/job/service/JobService.java | 16 +++++- .../core/job/service/JobServiceImpl.java | 25 +++++++++ .../flowci/core/test/job/JobServiceTest.java | 34 ++++++------ 8 files changed, 88 insertions(+), 70 deletions(-) diff --git a/core/src/main/java/com/flowci/core/agent/domain/ShellOut.java b/core/src/main/java/com/flowci/core/agent/domain/ShellOut.java index 9e7485e41..9e1a47e9b 100644 --- a/core/src/main/java/com/flowci/core/agent/domain/ShellOut.java +++ b/core/src/main/java/com/flowci/core/agent/domain/ShellOut.java @@ -5,11 +5,13 @@ import com.flowci.domain.Vars; import lombok.Getter; import lombok.Setter; +import lombok.experimental.Accessors; import java.util.Date; @Getter @Setter +@Accessors(chain = true) public final class ShellOut implements Executed, CmdOut { private String id; diff --git a/core/src/main/java/com/flowci/core/job/service/JobActionService.java b/core/src/main/java/com/flowci/core/job/service/JobActionService.java index 9cf062319..8574ff484 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobActionService.java +++ b/core/src/main/java/com/flowci/core/job/service/JobActionService.java @@ -1,6 +1,6 @@ package com.flowci.core.job.service; -import com.flowci.core.job.domain.Step; +import com.flowci.core.agent.domain.ShellOut; import com.flowci.exception.CIException; public interface JobActionService { @@ -13,7 +13,7 @@ public interface JobActionService { void toRun(String jobId); - void toContinue(Step step); + void toContinue(String jobId, ShellOut shellOut); void toCancelled(String jobId, CIException exception); diff --git a/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java index 43b30340f..2ba21f9f6 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java @@ -16,10 +16,7 @@ package com.flowci.core.job.service; -import com.flowci.core.agent.domain.Agent; -import com.flowci.core.agent.domain.AgentProfile; -import com.flowci.core.agent.domain.CmdIn; -import com.flowci.core.agent.domain.ShellIn; +import com.flowci.core.agent.domain.*; import com.flowci.core.agent.event.IdleAgentEvent; import com.flowci.core.agent.service.AgentService; import com.flowci.core.common.domain.Variables; @@ -50,7 +47,6 @@ import com.flowci.util.ObjectsHelper; import com.flowci.util.StringHelper; import com.flowci.zookeeper.InterLock; -import com.flowci.zookeeper.ZookeeperClient; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -119,17 +115,12 @@ public class JobActionServiceImpl implements JobActionService { private static final long RetryInterval = 10 * 1000; // 10 seconds - private static final int DefaultJobLockTimeout = 20; // seconds - @Autowired private Path repoDir; @Autowired private Path tmpDir; - @Autowired - private ZookeeperClient zk; - @Autowired private JobDao jobDao; @@ -157,6 +148,9 @@ public class JobActionServiceImpl implements JobActionService { @Autowired private LocalTaskService localTaskService; + @Autowired + private JobService jobService; + @Autowired private AgentService agentService; @@ -207,7 +201,7 @@ public void onIdleAgent(IdleAgentEvent event) { continue; } - Optional lock = lock(job.getId(), "LockJobFromIdleAgent"); + Optional lock = jobService.lock(job.getId()); if (!lock.isPresent()) { toFailureStatus(job, new CIException("Fail to lock job")); continue; @@ -224,7 +218,7 @@ public void onIdleAgent(IdleAgentEvent event) { } catch (Exception e) { toFailureStatus(job, new CIException(e.getMessage())); } finally { - unlock(lock.get(), "LockJobFromIdleAgent"); + jobService.unlock(lock.get(), job.getId()); } } } @@ -252,10 +246,14 @@ public void toRun(String jobId) { } @Override - public void toContinue(Step step) { - onTransition(step.getJobId(), Running, c -> { - c.setStep(step); + public void toContinue(String jobId, ShellOut so) { + onTransition(jobId, Running, c -> { + Step step = stepService.get(so.getId()); + step.setFrom(so); + stepService.resultUpdate(step); + log.info("[Callback]: {}-{} = {}", step.getJobId(), step.getNodePath(), step.getStatus()); + c.setStep(step); Job job = c.getJob(); log.debug("---- Job Status {} {} {} {}", job.isOnPostSteps(), step.getNodePath(), job.getStatus(), job.getStatusFromContext()); @@ -467,7 +465,7 @@ public void accept(JobSmContext context) throws Exception { updateJobContextAndLatestStatus(job, step); setJobStatusAndSave(job, Job.Status.RUNNING, null); - log.debug("Step {} been recorded", step.getNodePath()); + log.debug("Step {} {} been recorded", step.getNodePath(), step.getStatus()); if (!step.isSuccess()) { toFinishStatus(context); @@ -818,7 +816,7 @@ private void killOngoingSteps(Job job, boolean includePost) { } private void onTransition(String jobId, Status to, Consumer onContext) { - Optional lock = lock(jobId, "Before Transition"); + Optional lock = jobService.lock(jobId); if (!lock.isPresent()) { Job job = getJob(jobId); @@ -1070,9 +1068,12 @@ private boolean runPostStepsIfNeeded(JobSmContext context) throws ScriptExceptio // check current all steps that should be in finish status List steps = stepService.listByPath(job, job.getCurrentPath()); for (Step s : steps) { + log.debug("Step {} status ------------- {}", s.getNodePath(), s.getStatus()); + if (s.isOngoing() || s.isKilling()) { + context.setSkip(true); log.debug("Step ({} = {}) is ongoing or killing status", s.getNodePath(), s.getStatus()); - return false; + return true; } } @@ -1150,22 +1151,6 @@ private Job getJob(String jobId) { return jobDao.findById(jobId).get(); } - private Optional lock(String jobId, String message) { - String path = zk.makePath("/job-locks", jobId); - Optional lock = zk.lock(path, DefaultJobLockTimeout); - lock.ifPresent(interLock -> log.debug("Lock: {} - {}", jobId, message)); - return lock; - } - - private void unlock(InterLock lock, String jobId) { - try { - zk.release(lock); - log.debug("Unlock: {}", jobId); - } catch (Exception warn) { - log.warn(warn); - } - } - private class ActionOnFinishStatus implements Consumer { @Override @@ -1195,7 +1180,7 @@ public void onFinally(JobSmContext context) { } Job job = context.getJob(); - unlock(lock, job.getId()); + jobService.unlock(lock, job.getId()); context.setLock(null); } } diff --git a/core/src/main/java/com/flowci/core/job/service/JobEventService.java b/core/src/main/java/com/flowci/core/job/service/JobEventService.java index d0f03b17b..5be4c719c 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobEventService.java +++ b/core/src/main/java/com/flowci/core/job/service/JobEventService.java @@ -16,12 +16,12 @@ package com.flowci.core.job.service; -import com.flowci.core.job.domain.Step; +import com.flowci.core.agent.domain.ShellOut; public interface JobEventService { /** * Handle the executed cmd form callback queue */ - void handleCallback(Step execCmd); + void handleCallback(ShellOut shellOut); } diff --git a/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java index fcc88f82a..f9f738883 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobEventServiceImpl.java @@ -32,7 +32,6 @@ import com.flowci.core.flow.event.FlowDeletedEvent; import com.flowci.core.flow.event.FlowInitEvent; import com.flowci.core.job.domain.Job; -import com.flowci.core.job.domain.Step; import com.flowci.core.job.event.CreateNewJobEvent; import com.flowci.core.job.event.TtyStatusUpdateEvent; import com.flowci.core.job.manager.YmlManager; @@ -157,13 +156,7 @@ public void handleCmdOutFromAgent(OnCmdOutEvent event) { switch (ind) { case CmdOut.ShellOutInd: ShellOut shellOut = objectMapper.readValue(body, ShellOut.class); - - Step step = stepService.get(shellOut.getId()); - step.setFrom(shellOut); - stepService.resultUpdate(step); - - log.info("[Callback]: {}-{} = {}", step.getJobId(), step.getNodePath(), step.getStatus()); - handleCallback(step); + handleCallback(shellOut); break; case CmdOut.TtyOutInd: @@ -180,8 +173,9 @@ public void handleCmdOutFromAgent(OnCmdOutEvent event) { } @Override - public void handleCallback(Step step) { - jobActionService.toContinue(step); + public void handleCallback(ShellOut so) { + String jobId = stepService.get(so.getId()).getJobId(); + jobActionService.toContinue(jobId, so); } //==================================================================== @@ -206,10 +200,6 @@ public void startJobDeadLetterConsumer() throws IOException { // %% Utils //==================================================================== - private void logInfo(Job job, String message, Object... params) { - log.info("[Job] " + job.getKey() + " " + message, params); - } - private void declareJobQueueAndStartConsumer(Flow flow) { try { final String queue = flow.getQueueName(); diff --git a/core/src/main/java/com/flowci/core/job/service/JobService.java b/core/src/main/java/com/flowci/core/job/service/JobService.java index 367df7f88..203730e20 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobService.java +++ b/core/src/main/java/com/flowci/core/job/service/JobService.java @@ -17,12 +17,17 @@ package com.flowci.core.job.service; import com.flowci.core.flow.domain.Flow; -import com.flowci.core.job.domain.*; +import com.flowci.core.job.domain.Job; import com.flowci.core.job.domain.Job.Trigger; +import com.flowci.core.job.domain.JobDesc; +import com.flowci.core.job.domain.JobItem; +import com.flowci.core.job.domain.JobYml; import com.flowci.domain.StringVars; +import com.flowci.zookeeper.InterLock; import org.springframework.data.domain.Page; import javax.annotation.Nullable; +import java.util.Optional; /** * @author yang @@ -94,5 +99,14 @@ public interface JobService { * Delete all jobs of the flow within an executor */ void delete(Flow flow); + + /** + * Lock job by id + * + * @param jobId + */ + Optional lock(String jobId); + + void unlock(InterLock lock, String jobId); } diff --git a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java index cd4ccedd3..201365e7c 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java @@ -36,6 +36,8 @@ import com.flowci.store.FileManager; import com.flowci.tree.FlowNode; import com.flowci.util.StringHelper; +import com.flowci.zookeeper.InterLock; +import com.flowci.zookeeper.ZookeeperClient; import com.google.common.collect.Maps; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; @@ -65,6 +67,8 @@ public class JobServiceImpl implements JobService { private static final Sort SortByBuildNumber = Sort.by(Direction.DESC, "buildNumber"); + private static final int DefaultJobLockTimeout = 20; // seconds + //==================================================================== // %% Spring injection //==================================================================== @@ -115,6 +119,9 @@ public class JobServiceImpl implements JobService { @Autowired private SettingService settingService; + @Autowired + private ZookeeperClient zk; + //==================================================================== // %% Public functions //==================================================================== @@ -298,6 +305,24 @@ public void delete(Flow flow) { }); } + @Override + public Optional lock(String jobId) { + String path = zk.makePath("/job-locks", jobId); + Optional lock = zk.lock(path, DefaultJobLockTimeout); + lock.ifPresent(interLock -> log.debug("Lock: {}", jobId)); + return lock; + } + + @Override + public void unlock(InterLock lock, String jobId) { + try { + zk.release(lock); + log.debug("Unlock: {}", jobId); + } catch (Exception warn) { + log.warn(warn); + } + } + //==================================================================== // %% Utils //==================================================================== diff --git a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java index 1263014a1..67582857b 100644 --- a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java +++ b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java @@ -17,10 +17,7 @@ package com.flowci.core.test.job; import com.flowci.core.agent.dao.AgentDao; -import com.flowci.core.agent.domain.Agent; -import com.flowci.core.agent.domain.CmdIn; -import com.flowci.core.agent.domain.AgentOption; -import com.flowci.core.agent.domain.ShellIn; +import com.flowci.core.agent.domain.*; import com.flowci.core.agent.event.AgentStatusEvent; import com.flowci.core.agent.event.CmdSentEvent; import com.flowci.core.agent.service.AgentService; @@ -40,12 +37,8 @@ import com.flowci.core.job.event.JobReceivedEvent; import com.flowci.core.job.event.JobStatusChangeEvent; import com.flowci.core.job.event.StartAsyncLocalTaskEvent; -import com.flowci.core.job.service.JobActionService; import com.flowci.core.job.manager.YmlManager; -import com.flowci.core.job.service.JobEventService; -import com.flowci.core.job.service.JobService; -import com.flowci.core.job.service.LocalTaskService; -import com.flowci.core.job.service.StepService; +import com.flowci.core.job.service.*; import com.flowci.core.plugin.dao.PluginDao; import com.flowci.core.plugin.domain.Plugin; import com.flowci.core.test.ZookeeperScenario; @@ -315,7 +308,7 @@ public void should_finish_whole_job() throws InterruptedException, IOException { firstStep.setFinishAt(new Date()); executedCmdDao.save(firstStep); - jobEventService.handleCallback(firstStep); + jobEventService.handleCallback(getShellOutFromStep(firstStep)); // then: verify step 2 agent Assert.assertTrue(counterForStep2.await(10, TimeUnit.SECONDS)); @@ -340,7 +333,7 @@ public void should_finish_whole_job() throws InterruptedException, IOException { secondStep.setFinishAt(new Date()); executedCmdDao.save(secondStep); - jobEventService.handleCallback(secondStep); + jobEventService.handleCallback(getShellOutFromStep(secondStep)); // // then: should job with SUCCESS status and sent notification task Assert.assertEquals(Status.SUCCESS, jobService.get(job.getId()).getStatus()); @@ -363,8 +356,9 @@ public void should_handle_cmd_callback_for_success_status() { firstStep.setStatus(Step.Status.SUCCESS); firstStep.setOutput(output); + executedCmdDao.save(firstStep); - jobEventService.handleCallback(firstStep); + jobEventService.handleCallback(getShellOutFromStep(firstStep)); // then: job context should be updated job = jobDao.findById(job.getId()).get(); @@ -381,7 +375,7 @@ public void should_handle_cmd_callback_for_success_status() { secondStep.setStatus(Step.Status.SUCCESS); secondStep.setOutput(output); executedCmdDao.save(secondStep); - jobEventService.handleCallback(secondStep); + jobEventService.handleCallback(getShellOutFromStep(secondStep)); // then: job context should be updated job = jobDao.findById(job.getId()).get(); @@ -405,7 +399,7 @@ public void should_handle_cmd_callback_for_failure_status() throws IOException { // when: cmd of first node with failure firstStep.setStatus(Step.Status.EXCEPTION); executedCmdDao.save(firstStep); - jobEventService.handleCallback(firstStep); + jobEventService.handleCallback(getShellOutFromStep(firstStep)); // then: job should be failure job = jobDao.findById(job.getId()).get(); @@ -432,7 +426,7 @@ public void should_handle_cmd_callback_for_failure_status_but_allow_failure() th firstStep.setStatus(Step.Status.EXCEPTION); firstStep.setOutput(output); executedCmdDao.save(firstStep); - jobEventService.handleCallback(firstStep); + jobEventService.handleCallback(getShellOutFromStep(firstStep)); // then: job status should be running and current path should be change to second node job = jobDao.findById(job.getId()).get(); @@ -450,7 +444,7 @@ public void should_handle_cmd_callback_for_failure_status_but_allow_failure() th secondCmd.setStatus(Step.Status.TIMEOUT); secondCmd.setOutput(output); executedCmdDao.save(secondCmd); - jobEventService.handleCallback(secondCmd); + jobEventService.handleCallback(getShellOutFromStep(secondCmd)); // then: job should be timeout with error message job = jobDao.findById(job.getId()).get(); @@ -547,4 +541,12 @@ private Job prepareJobForRunningStatus(Agent agent) { return jobDao.save(job); } + + private ShellOut getShellOutFromStep(Step step) { + return new ShellOut() + .setId(step.getId()) + .setStatus(step.getStatus()) + .setOutput(step.getOutput()) + .setFinishAt(new Date()); + } } \ No newline at end of file From 53b691b404cebe81b3b1a4cc3bfbcc2670683fb1 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Wed, 5 May 2021 17:40:37 +0200 Subject: [PATCH 43/46] fix job ut --- .../java/com/flowci/core/job/service/JobServiceImpl.java | 5 +++-- .../java/com/flowci/core/test/job/JobServiceTest.java | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java index 201365e7c..f2087e766 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobServiceImpl.java @@ -194,7 +194,7 @@ public Job create(Flow flow, String yml, Trigger trigger, StringVars input) { } jobActionService.toCreated(job.getId(), yml); - return job; + return get(job.getId()); } @Override @@ -239,6 +239,7 @@ public Job rerun(Flow flow, Job job) { context.put(GIT_COMMIT_ID, lastCommitId); context.put(Variables.Job.TriggerBy, sessionManager.get().getEmail()); context.merge(root.getEnvironments(), false); + jobDao.save(job); // reset job agent jobAgentDao.save(new JobAgent(job.getId(), flow.getId())); @@ -250,7 +251,7 @@ public Job rerun(Flow flow, Job job) { jobActionService.toCreated(job.getId(), yml.getRaw()); jobActionService.toStart(job.getId()); - return job; + return get(job.getId()); } @Override diff --git a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java index 67582857b..48c9906f8 100644 --- a/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java +++ b/core/src/test/java/com/flowci/core/test/job/JobServiceTest.java @@ -192,12 +192,13 @@ public void should_start_new_job() throws Throwable { // when: create and start job Job job = jobService.create(flow, yml.getRaw(), Trigger.MANUAL, StringVars.EMPTY); - NodeTree tree = ymlManager.getTree(job); + job = jobService.get(job.getId()); Assert.assertEquals(Status.CREATED, job.getStatus()); Assert.assertTrue(job.getCurrentPath().isEmpty()); jobActionService.toStart(job.getId()); + job = jobService.get(job.getId()); Assert.assertEquals(Status.QUEUED, job.getStatus()); Assert.assertNotNull(job); @@ -485,8 +486,9 @@ public void should_cancel_job_if_agent_offline() throws IOException, Interrupted Assert.assertEquals(Status.CANCELLED, job.getStatus()); // then: step should be skipped - for (Step cmd : stepService.list(job)) { - Assert.assertEquals(Step.Status.SKIPPED, cmd.getStatus()); + List steps = stepService.list(job); + for (Step step : steps) { + Assert.assertEquals(Step.Status.PENDING, step.getStatus()); } } From 67e337669a200d2e81ad151ac7ee98e977501dd9 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Wed, 5 May 2021 22:46:51 +0200 Subject: [PATCH 44/46] remove running or finished post steps --- .../main/java/com/flowci/core/job/domain/Step.java | 5 +++++ .../core/job/service/JobActionServiceImpl.java | 13 ++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/flowci/core/job/domain/Step.java b/core/src/main/java/com/flowci/core/job/domain/Step.java index cb63b7db5..f6427ac91 100644 --- a/core/src/main/java/com/flowci/core/job/domain/Step.java +++ b/core/src/main/java/com/flowci/core/job/domain/Step.java @@ -173,6 +173,11 @@ public boolean isKilling() { return status == Status.KILLING; } + @JsonIgnore + public boolean isFinished() { + return FinishStatus.contains(status); + } + public void setFrom(ShellOut out) { this.processId = out.getProcessId(); this.containerId = out.getContainerId(); diff --git a/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java index 2ba21f9f6..330c4bd91 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java @@ -913,7 +913,8 @@ private boolean toNextStep(Job job, Step step) throws ScriptException { } // check prev steps status - Set previous = getStepsStatus(job, tree.prevs(next, job.isOnPostSteps())); + Collection prevs = tree.prevs(next, job.isOnPostSteps()); + Set previous = getStepsStatus(job, prevs); boolean hasFailure = !Collections.disjoint(previous, Executed.FailureStatus); boolean hasOngoing = !Collections.disjoint(previous, Executed.OngoingStatus); if (hasFailure) { @@ -1065,6 +1066,16 @@ private boolean runPostStepsIfNeeded(JobSmContext context) throws ScriptExceptio return false; } + // remove running or finished post steps + Iterator iterator = nextPostSteps.iterator(); + while(iterator.hasNext()) { + Node postNode = iterator.next(); + Step postStep = stepService.get(job.getId(), postNode.getPathAsString()); + if (postStep.isOngoing() || postStep.isKilling() || postStep.isFinished()) { + iterator.remove(); + } + } + // check current all steps that should be in finish status List steps = stepService.listByPath(job, job.getCurrentPath()); for (Step s : steps) { From fb5cc10ec352015900fdd71fbec181f1ef382747 Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Wed, 5 May 2021 23:03:33 +0200 Subject: [PATCH 45/46] check step pending status before go next --- .../com/flowci/core/job/service/JobActionServiceImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java b/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java index 330c4bd91..13d659e0c 100644 --- a/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java +++ b/core/src/main/java/com/flowci/core/job/service/JobActionServiceImpl.java @@ -916,13 +916,15 @@ private boolean toNextStep(Job job, Step step) throws ScriptException { Collection prevs = tree.prevs(next, job.isOnPostSteps()); Set previous = getStepsStatus(job, prevs); boolean hasFailure = !Collections.disjoint(previous, Executed.FailureStatus); - boolean hasOngoing = !Collections.disjoint(previous, Executed.OngoingStatus); if (hasFailure) { return false; } + boolean hasOngoing = !Collections.disjoint(previous, Executed.OngoingStatus); + boolean hasWaiting = !Collections.disjoint(previous, Executed.WaitingStatus); + // do not execute next - if (hasOngoing) { + if (hasOngoing || hasWaiting) { return true; } From c165980dedf20ddc8c547709a877d54c15bdff4f Mon Sep 17 00:00:00 2001 From: gy2006 <32008001@qq.com> Date: Thu, 6 May 2021 20:34:43 +0200 Subject: [PATCH 46/46] update ver to 0.21.21 --- core/src/main/resources/application.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/resources/application.properties b/core/src/main/resources/application.properties index 596116916..ed15459a8 100644 --- a/core/src/main/resources/application.properties +++ b/core/src/main/resources/application.properties @@ -2,7 +2,7 @@ logging.level.org.apache.zookeeper=ERROR logging.level.org.apache.curator.framework=ERROR logging.level.com.flowci.core=${FLOWCI_LOG_LEVEL:INFO} -info.app.version=0.21.05 +info.app.version=0.21.21 info.app.name=flow.ci server.port=${FLOWCI_SERVER_PORT:8080}