Merge pull request #407 from FlowCI/develop

Develop
FlowCI · Aug 18, 2021 · bee8fd7 · bee8fd7
2 parents 462e88d + 57a4c95
commit bee8fd7
Show file tree

Hide file tree

Showing 38 changed files with 301 additions and 144 deletions.
diff --git a/core/src/main/java/com/flowci/core/Application.java b/core/src/main/java/com/flowci/core/Application.java
@@ -16,15 +16,34 @@
 
 package com.flowci.core;
 
+import com.flowci.util.StringHelper;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.event.ContextRefreshedEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.ResourceLoader;
+
+import java.io.IOException;
 
 /**
  * @author yang
  */
+@Log4j2
 @SpringBootApplication
 public class Application {
 
+    @Autowired
+    private ResourceLoader resourceLoader;
+
+    @EventListener(ContextRefreshedEvent.class)
+    public void printBanner() throws IOException {
+        Resource r = resourceLoader.getResource("classpath:welcome.txt");
+        log.info(StringHelper.toString(r.getInputStream()));
+    }
+
     public static void main(String[] args) {
         SpringApplication.run(Application.class, args);
     }

diff --git a/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java b/core/src/main/java/com/flowci/core/agent/domain/ShellIn.java
@@ -52,6 +52,8 @@ public enum ShellType {
 
     private Set<String> secrets;
 
+    private Set<String> configs;
+
     public ShellIn() {
         super(Type.SHELL);
     }

diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentHostServiceImpl.java
@@ -47,7 +47,6 @@
 import lombok.AllArgsConstructor;
 import lombok.extern.log4j.Log4j2;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.event.ContextRefreshedEvent;
 import org.springframework.context.event.EventListener;
 import org.springframework.core.env.Environment;
 import org.springframework.core.task.TaskExecutor;
@@ -56,6 +55,7 @@
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import javax.annotation.PostConstruct;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.*;
@@ -114,6 +114,12 @@ public class AgentHostServiceImpl implements AgentHostService {
         mapping.put(K8sAgentHost.class, new K8sHostAdaptor());
     }
 
+    @PostConstruct
+    public void init() {
+        autoCreateLocalAgentHost();
+        syncAgents();
+    }
+
     //====================================================================
     //        %% Public functions
     //====================================================================
@@ -378,12 +384,6 @@ public void scheduleCollect() {
     //        %% Internal events
     //====================================================================
 
-    @EventListener
-    public void onContextReady(ContextRefreshedEvent event) {
-        autoCreateLocalAgentHost();
-        syncAgents();
-    }
-
     @EventListener
     public void onNoIdleAgent(NoIdleAgentEvent event) {
         Set<String> agentTags = event.getSelector().getLabel();

diff --git a/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java b/core/src/main/java/com/flowci/core/agent/service/AgentServiceImpl.java
@@ -42,11 +42,11 @@
 import com.google.common.collect.Sets;
 import lombok.extern.log4j.Log4j2;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.event.ContextRefreshedEvent;
 import org.springframework.context.event.EventListener;
 import org.springframework.dao.DuplicateKeyException;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.PostConstruct;
 import java.io.IOException;
 import java.time.Instant;
 import java.util.*;
@@ -108,7 +108,7 @@ public class AgentServiceImpl implements AgentService {
     @Autowired
     private SocketPushManager socketPushManager;
 
-    @EventListener(ContextRefreshedEvent.class)
+    @PostConstruct
     public void initAgentStatus() {
         taskManager.run("init-agent-status", true, () -> {
             for (Agent agent : agentDao.findAll()) {
@@ -122,7 +122,7 @@ public void initAgentStatus() {
         });
     }
 
-    @EventListener(ContextRefreshedEvent.class)
+    @PostConstruct
     public void subscribeIdleAgentQueue() throws IOException {
         idleAgentQueueManager.startConsumer(idleAgentQueue, false, (header, body, envelope) -> {
             String agentId = new String(body);
@@ -152,6 +152,22 @@ public void subscribeIdleAgentQueue() throws IOException {
         }, null);
     }
 
+    @PostConstruct
+    public void lockNodeCleanup() {
+        List<String> children = zk.children(zkProperties.getAgentRoot());
+        for (String path : children) {
+            String agentId = Util.getAgentIdFromLockPath(path);
+            Optional<Agent> optional = agentDao.findById(agentId);
+
+            if (!optional.isPresent()) {
+                try {
+                    zk.delete(path, true);
+                } catch (Throwable ignore) {
+                }
+            }
+        }
+    }
+
     //====================================================================
     //        %% Public Methods
     //====================================================================
@@ -369,22 +385,6 @@ public void dispatch(CmdIn cmd, Agent agent) {
     //        %% Spring Event Listener
     //====================================================================
 
-    @EventListener(ContextRefreshedEvent.class)
-    public void lockNodeCleanup() {
-        List<String> children = zk.children(zkProperties.getAgentRoot());
-        for (String path : children) {
-            String agentId = Util.getAgentIdFromLockPath(path);
-            Optional<Agent> optional = agentDao.findById(agentId);
-
-            if (!optional.isPresent()) {
-                try {
-                    zk.delete(path, true);
-                } catch (Throwable ignore) {
-                }
-            }
-        }
-    }
-
     @EventListener
     public void onConnected(OnConnectedEvent event) {
         Optional<InterLock> lock = lock();

diff --git a/core/src/main/java/com/flowci/core/api/OpenRestController.java b/core/src/main/java/com/flowci/core/api/OpenRestController.java
@@ -73,7 +73,7 @@ public Secret getSecret(@PathVariable String name) {
 
         if (secret instanceof RSASecret) {
             RSASecret rsa = (RSASecret) secret;
-            rsa.setPublicKey(null);
+            rsa.setPrivateKey(null);
         }
 
         return secret;

diff --git a/core/src/main/java/com/flowci/core/common/config/AppProperties.java b/core/src/main/java/com/flowci/core/common/config/AppProperties.java
@@ -23,6 +23,7 @@
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.PropertySource;
+import org.springframework.core.io.Resource;
 import org.springframework.validation.annotation.Validated;
 
 import javax.validation.constraints.NotBlank;
@@ -51,6 +52,9 @@ public class AppProperties {
     @Length(max = 16, min = 16)
     private String secret;
 
+    // indicate load yaml template and plugin from where
+    private String resourceDomain;
+
     private boolean autoLocalAgentHost;
 
     private boolean defaultSmtpConfig;
@@ -106,7 +110,7 @@ public Minio minio() {
     @Data
     public static class Flow {
 
-        private String templatesUrl;
+        private Resource templatesUrl;
     }
 
     @Data
@@ -118,7 +122,7 @@ public static class Job {
     @Data
     public static class Plugin {
 
-        private String defaultRepo;
+        private Resource defaultRepo;
 
         private Boolean autoUpdate;
     }

diff --git a/core/src/main/java/com/flowci/core/common/domain/Settings.java b/core/src/main/java/com/flowci/core/common/domain/Settings.java
@@ -32,6 +32,11 @@ public static class Action {
 
     private String serverUrl;
 
+    /**
+     * Indicate load resource(cn) from where
+     */
+    private String source;
+
     public Settings() {
         setId(DefaultId);
     }

diff --git a/core/src/main/java/com/flowci/core/common/domain/SourceWithDomain.java b/core/src/main/java/com/flowci/core/common/domain/SourceWithDomain.java
@@ -0,0 +1,28 @@
+package com.flowci.core.common.domain;
+
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.flowci.util.StringHelper;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.io.Serializable;
+
+@Getter
+@Setter
+public abstract class SourceWithDomain implements Serializable {
+
+    private static final String DOMAIN_CN = "cn";
+
+    @JsonAlias("url")
+    private String source;
+
+    @JsonAlias("url_cn")
+    @JsonProperty("source_cn")
+    private String sourceCn;
+
+    public String getSourceWithDomain(String domain) {
+        if (!StringHelper.hasValue(domain)) return source;
+        return DOMAIN_CN.equalsIgnoreCase(domain) ? sourceCn : source;
+    }
+}
diff --git a/core/src/main/java/com/flowci/core/common/domain/Variables.java b/core/src/main/java/com/flowci/core/common/domain/Variables.java
@@ -29,6 +29,7 @@ public abstract static class App {
 
         public static final String Host = "FLOWCI_SERVER_HOST";
 
+        public static final String ResourceDomain = "FLOWCI_RESOURCE_DOMAIN";
     }
 
     public abstract static class Flow {

diff --git a/core/src/main/java/com/flowci/core/common/manager/SocketPushManager.java b/core/src/main/java/com/flowci/core/common/manager/SocketPushManager.java
@@ -23,11 +23,10 @@
 import com.flowci.core.common.rabbit.RabbitOperations;
 import lombok.extern.log4j.Log4j2;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.event.ContextRefreshedEvent;
-import org.springframework.context.event.EventListener;
 import org.springframework.messaging.simp.SimpMessagingTemplate;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.PostConstruct;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
@@ -56,7 +55,7 @@ public class SocketPushManager {
     @Autowired
     private RabbitOperations broadcastQueueManager;
 
-    @EventListener(ContextRefreshedEvent.class)
+    @PostConstruct
     public void subscribeBroadcastQueue() throws IOException {
         broadcastQueueManager.startConsumer(wsBroadcastQueue, true, (headers, body, envelope) -> {
             try {

diff --git a/core/src/main/java/com/flowci/core/common/manager/SpringEventManagerImpl.java b/core/src/main/java/com/flowci/core/common/manager/SpringEventManagerImpl.java
@@ -27,10 +27,9 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationEvent;
 import org.springframework.context.ApplicationEventPublisher;
-import org.springframework.context.event.ContextRefreshedEvent;
-import org.springframework.context.event.EventListener;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.PostConstruct;
 import java.io.IOException;
 import java.util.Map;
 
@@ -55,7 +54,7 @@ public class SpringEventManagerImpl implements SpringEventManager {
     @Autowired
     private RabbitOperations broadcastQueueManager;
 
-    @EventListener(ContextRefreshedEvent.class)
+    @PostConstruct
     public void subscribeBroadcastQueue() throws IOException {
         broadcastQueueManager.startConsumer(eventBroadcastQueue, true, (headers, body, envelope) -> {
             try {

diff --git a/core/src/main/java/com/flowci/core/common/service/SettingServiceImpl.java b/core/src/main/java/com/flowci/core/common/service/SettingServiceImpl.java
@@ -1,5 +1,6 @@
 package com.flowci.core.common.service;
 
+import com.flowci.core.common.config.AppProperties;
 import com.flowci.core.common.dao.SettingsDao;
 import com.flowci.core.common.domain.Settings;
 import com.flowci.core.common.domain.Variables;
@@ -18,6 +19,9 @@ public class SettingServiceImpl implements SettingService {
     @Autowired
     private Environment environment;
 
+    @Autowired
+    private AppProperties appProperties;
+
     @Autowired
     private ServerProperties serverProperties;
 
@@ -49,6 +53,7 @@ public void setDefaultValue() {
             }
 
             s.setServerUrl(serverUrl);
+            s.setSource(appProperties.getResourceDomain());
             settingsDao.save(s);
         });
     }

diff --git a/core/src/main/java/com/flowci/core/config/service/ConfigServiceImpl.java b/core/src/main/java/com/flowci/core/config/service/ConfigServiceImpl.java
@@ -16,12 +16,12 @@
 import lombok.extern.log4j.Log4j2;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.event.ContextRefreshedEvent;
 import org.springframework.context.event.EventListener;
 import org.springframework.core.io.Resource;
 import org.springframework.dao.DuplicateKeyException;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.PostConstruct;
 import java.io.IOException;
 import java.util.List;
 import java.util.Optional;
@@ -42,8 +42,8 @@ public class ConfigServiceImpl implements ConfigService {
     @Autowired
     private SpringEventManager eventManager;
 
-    @EventListener
-    public void onInit(ContextRefreshedEvent ignore) {
+    @PostConstruct
+    public void onInit() {
         try {
             Config config = ConfigParser.parse(defaultSmtpConfigYml.getInputStream());
             Optional<Config> optional = configDao.findByName(config.getName());

diff --git a/core/src/main/java/com/flowci/core/flow/config/FlowConfig.java b/core/src/main/java/com/flowci/core/flow/config/FlowConfig.java
@@ -20,14 +20,14 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.flowci.core.common.config.AppProperties;
 import com.flowci.core.common.helper.CacheHelper;
-import com.flowci.core.common.manager.HttpRequestManager;
 import com.flowci.core.flow.domain.Template;
 import com.flowci.tree.NodeTree;
 import com.github.benmanes.caffeine.cache.Cache;
 import lombok.extern.log4j.Log4j2;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.Resource;
 import org.springframework.scheduling.TaskScheduler;
 import org.springframework.scheduling.concurrent.ConcurrentTaskScheduler;
 
@@ -58,13 +58,13 @@ public Cache<String, List<String>> gitBranchCache() {
     }
 
     @Bean("templates")
-    public List<Template> getTemplates(HttpRequestManager httpManager) throws IOException {
-        String body = httpManager.get(flowProperties.getTemplatesUrl());
+    public List<Template> getTemplates() throws IOException {
+        Resource r = flowProperties.getTemplatesUrl();
 
         TypeReference<List<Template>> typeRef = new TypeReference<List<Template>>() {
         };
 
-        List<Template> list = objectMapper.readValue(body, typeRef);
+        List<Template> list = objectMapper.readValue(r.getInputStream(), typeRef);
         log.info("Templates is loaded from {}", flowProperties.getTemplatesUrl());
         return list;
     }

diff --git a/core/src/main/java/com/flowci/core/flow/controller/YmlController.java b/core/src/main/java/com/flowci/core/flow/controller/YmlController.java
@@ -62,14 +62,13 @@ public void saveYml(@PathVariable String flowName,
                         @RequestBody RequestMessage<String> body) {
         Flow flow = flowService.get(flowName);
         String yamlInB64 = body.getData();
-        ymlService.saveYml(flow, ymlName, yamlInB64);
+        ymlService.saveYmlFromB64(flow, ymlName, yamlInB64);
     }
 
     @GetMapping(value = "/{flowName}/yml/{ymlName}", produces = MediaType.APPLICATION_JSON_VALUE)
     @Action(FlowAction.GET_YML)
     public String getYml(@PathVariable String flowName, @PathVariable String ymlName) {
         Flow flow = flowService.get(flowName);
-        String yamlInB64 = ymlService.getYmlString(flow.getId(), ymlName);
-        return yamlInB64;
+        return ymlService.getYmlString(flow.getId(), ymlName);
     }
 }