support appoint maxmemory policy

and make it work through app lifecycle

cachecloud-web/sql/3.1.sql

@@ -1136,7 +1136,7 @@ CREATE TABLE `system_resource` (
 --  Records of `system_resource`
 -- ----------------------------
 BEGIN;
-INSERT INTO `system_resource1` VALUES (1,'cachecloud-init.sh','容器初始化脚本',2,'2020-07-15 18:35:41','/script','',0,1,NULL,NULL,NULL,0),(2,'x.x.x.x',NULL,1,'2020-08-10 10:31:51','/opt/download/software/cachecloud/resource','http://x.x.x.x/software/cachecloud/resource',0,1,'admin',0,NULL,0),(4,'cachecloud-env.sh','宿主环境脚本',2,'2020-07-15 18:36:28','/script','',0,1,NULL,NULL,NULL,0),(5,'id_rsa','私钥文件',4,'2020-07-07 10:45:39','/ssh','',0,1,NULL,NULL,NULL,0),(6,'id_rsa.pub','公钥文件',4,'2020-07-07 10:45:45','/ssh','',0,1,NULL,NULL,NULL,0),(12,'redis-4.0.14','redis 4.0.14资源包',3,'2020-08-10 09:52:41','/redis','http://download.redis.io/releases/redis-4.0.14.tar.gz',0,1,'admin',532,NULL,0),(21,'/script','脚本目录管理',6,'2020-08-10 10:51:34','',NULL,0,1,'admin',0,NULL,0),(28,'/ssh','ssh目录',6,'2020-07-20 17:55:03',NULL,NULL,0,1,'admin',0,NULL,0),(29,'redis-3.0.7','redis3.0.7 资源包',3,'2020-08-10 09:53:32','/redis','http://download.redis.io/releases/redis-3.0.7.tar.gz',0,1,'admin',529,NULL,0),(31,'redis-3.2.12','redis 3.2.12 资源包',3,'2020-08-10 15:08:21','/redis','http://download.redis.io/releases/redis-3.2.12.tar.gz',0,1,'admin',530,NULL,0),(32,'/redis','redis资源包管理',6,'2020-07-20 17:54:59',NULL,NULL,0,1,'admin',0,NULL,0),(33,'/tool','迁移工具资源包',6,'2020-07-20 17:54:53',NULL,NULL,0,1,'admin',0,NULL,0),(37,'redis-5.0.9','redis5.0.9 资源包',3,'2020-08-10 09:51:41','/redis','http://download.redis.io/releases/redis-5.0.9.tar.gz',0,1,'admin',533,NULL,0),(40,'redis-shake-2.0.3','redis 2.0.3\n修复fix 5.0迁移类型问题',7,'2020-08-11 10:53:26','/tool','https://github.com/alibaba/RedisShake/releases/download/release-v2.0.3-20200724/redis-shake-v2.0.3.tar.gz',0,1,'admin',518,NULL,0);
+INSERT INTO `system_resource` VALUES (1,'cachecloud-init.sh','容器初始化脚本',2,'2020-07-15 18:35:41','/script','',0,1,NULL,NULL,NULL,0),(2,'x.x.x.x',NULL,1,'2020-08-10 10:31:51','/opt/download/software/cachecloud/resource','http://x.x.x.x/software/cachecloud/resource',0,1,'admin',0,NULL,0),(4,'cachecloud-env.sh','宿主环境脚本',2,'2020-07-15 18:36:28','/script','',0,1,NULL,NULL,NULL,0),(5,'id_rsa','私钥文件',4,'2020-07-07 10:45:39','/ssh','',0,1,NULL,NULL,NULL,0),(6,'id_rsa.pub','公钥文件',4,'2020-07-07 10:45:45','/ssh','',0,1,NULL,NULL,NULL,0),(12,'redis-4.0.14','redis 4.0.14资源包',3,'2020-08-10 09:52:41','/redis','http://download.redis.io/releases/redis-4.0.14.tar.gz',0,1,'admin',532,NULL,0),(21,'/script','脚本目录管理',6,'2020-08-10 10:51:34','',NULL,0,1,'admin',0,NULL,0),(28,'/ssh','ssh目录',6,'2020-07-20 17:55:03',NULL,NULL,0,1,'admin',0,NULL,0),(29,'redis-3.0.7','redis3.0.7 资源包',3,'2020-08-10 09:53:32','/redis','http://download.redis.io/releases/redis-3.0.7.tar.gz',0,1,'admin',529,NULL,0),(31,'redis-3.2.12','redis 3.2.12 资源包',3,'2020-08-10 15:08:21','/redis','http://download.redis.io/releases/redis-3.2.12.tar.gz',0,1,'admin',530,NULL,0),(32,'/redis','redis资源包管理',6,'2020-07-20 17:54:59',NULL,NULL,0,1,'admin',0,NULL,0),(33,'/tool','迁移工具资源包',6,'2020-07-20 17:54:53',NULL,NULL,0,1,'admin',0,NULL,0),(37,'redis-5.0.9','redis5.0.9 资源包',3,'2020-08-10 09:51:41','/redis','http://download.redis.io/releases/redis-5.0.9.tar.gz',0,1,'admin',533,NULL,0),(40,'redis-shake-2.0.3','redis 2.0.3\n修复fix 5.0迁移类型问题',7,'2020-08-11 10:53:26','/tool','https://github.com/alibaba/RedisShake/releases/download/release-v2.0.3-20200724/redis-shake-v2.0.3.tar.gz',0,1,'admin',518,NULL,0);
 COMMIT;
 
 --

cachecloud-web/sql/update 3.1 to 3.2.sql

@@ -0,0 +1,2 @@
+-- app_desc change
+ALTER TABLE app_desc ADD maxmemory_policy tinyint(4) DEFAULT NULL COMMENT '淘汰策略(0：noeviction; 1:allkeys-lru;2:allkeys-lfu;3:volatile-lru;4:volatile-lfu;5:allkeys-random;6:volatile-random;7:volatile-ttl)';

cachecloud-web/src/main/java/com/sohu/cache/constant/AppDescEnum.java

@@ -1,5 +1,9 @@
 package com.sohu.cache.constant;
 
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
 /**
  * app详情枚举(以后有关app详情的都往这里面迁)
  * 
@@ -58,4 +62,55 @@ public String getInfo() {
     }
 
 
+    /**
+     * 应用内存淘汰策略
+     */
+    public static enum MaxmemoryPolicyType {
+        NOEVICTION(0, "noeviction","不淘汰，占满写入失败"),
+        ALLKEYSLRU(1, "allkeys-lru","所有键-最近最少使用"),
+        ALLKEYSLFU(2, "allkeys-lfu","所有键-最少频率使用"),
+        VOLATILELRU(3, "volatile-lru","有过期时间的键-最近最少使用"),
+        VOLATILELFU(4, "volatile-lfu","有过期时间的键-最少频率使用"),
+        ALLKEYSRANDOM(5, "allkeys-random","所有键-随机"),
+        VOLATILERANDOM(6, "volatile-random","有过期时间的键-随机"),
+        VOLATILETTL(7, "volatile-ttl","有过期时间的键-剩余时间最短");
+
+        private int type;
+
+        private String name;
+
+        private String desc;
+
+        private MaxmemoryPolicyType(int type, String name, String desc) {
+           this.type = type;
+           this.name = name;
+           this.desc = desc;
+        }
+
+        public int getType(){
+            return type;
+        }
+
+        public String getName(){
+            return name;
+        }
+
+        public String getDesc(){
+            return desc;
+        }
+
+        public static MaxmemoryPolicyType getByType(int type){
+            Optional<MaxmemoryPolicyType> policyTypeOptional = Arrays.asList(MaxmemoryPolicyType.values()).stream().filter(maxmemoryPolicyType -> maxmemoryPolicyType.type == type).findFirst();
+            if(policyTypeOptional.isPresent()){
+                return policyTypeOptional.get();
+            }
+            return null;
+        }
+
+        public static List<MaxmemoryPolicyType> getAll(){
+            MaxmemoryPolicyType[] values = MaxmemoryPolicyType.values();
+            return Arrays.asList(values);
+        }
+    }
+
 }

cachecloud-web/src/main/java/com/sohu/cache/entity/AppDesc.java

@@ -162,6 +162,11 @@ public class AppDesc implements Serializable {
      */
     private String customPassword;
 
+    /**
+     * 最大内存淘汰策略
+     * 见AppDescEnum.MaxmemoryPolicyType
+     */
+    private Integer maxmemoryPolicy;
     public String getAuthPassword() {
         if(StringUtils.isNotBlank(customPassword)){
             return customPassword;
@@ -309,4 +314,15 @@ public Date getPassedTime() {
     public void setPassedTime(Date passedTime) {
         this.passedTime = (Date) passedTime.clone();
     }
+
+    public String getMaxmemoryPolicyDesc() {
+        if(maxmemoryPolicy != null){
+            AppDescEnum.MaxmemoryPolicyType policyType = AppDescEnum.MaxmemoryPolicyType.getByType(maxmemoryPolicy);
+            if(policyType != null){
+                return policyType.getName();
+
+            }
+        }
+        return null;
+    }
 }

cachecloud-web/src/main/java/com/sohu/cache/log/CustLogAppenderInit.java

@@ -4,20 +4,20 @@
 import ch.qos.logback.classic.LoggerContext;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.ApplicationArguments;
+import org.springframework.boot.ApplicationRunner;
 import org.springframework.stereotype.Component;
 
-import javax.annotation.PostConstruct;
-
 @Component
-public class CustLogAppenderInit {
+public class CustLogAppenderInit implements ApplicationRunner{
 
     @Autowired
     private TaskFlowRecordAppender taskFlowRecordAppender;
 
     private final org.slf4j.Logger logger = LoggerFactory.getLogger(CustLogAppenderInit.class);
 
-    @PostConstruct
-    public void init() {
+    @Override
+    public void run(ApplicationArguments args) throws Exception {
         logger.warn("custLogAppender init begin!");
         if (LoggerFactory.getILoggerFactory() instanceof LoggerContext) {
             LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
@@ -32,5 +32,4 @@ public void init() {
                     LoggerFactory.getILoggerFactory());
         }
     }
-
 }

cachecloud-web/src/main/java/com/sohu/cache/redis/RedisConfigTemplateService.java

@@ -87,10 +87,11 @@ public interface RedisConfigTemplateService {
      * 普通节点配置
      * @param port
      * @param maxMemory
+     * @param maxMemoryPolicy
      * @param versionId Redis版本
      * @return 普通配置列表
      */
-    List<String> handleCommonConfig(String host, int port, int maxMemory, int versionId);
+    List<String> handleCommonConfig(String host, int port, int maxMemory, String maxMemoryPolicy, int versionId);
 
     /**
      * sentinel节点配置(兼容k8s)

cachecloud-web/src/main/java/com/sohu/cache/redis/impl/RedisCenterImpl.java

@@ -1563,6 +1563,9 @@ public Object executeAdminCommand(AppDesc appDesc, ProtocolCommand command, Stri
                 if(instanceInfo.getType() == ConstUtils.CACHE_REDIS_SENTINEL){
                     continue;
                 }
+                if(instanceInfo.isOffline()){
+                    continue;
+                }
                 host = instanceInfo.getIp();
                 port = instanceInfo.getPort();
                 BooleanEnum isMaster = this.isMaster(appId, host, port);

cachecloud-web/src/main/java/com/sohu/cache/redis/impl/RedisConfigTemplateServiceImpl.java

@@ -163,7 +163,7 @@ public int updateStatus(long id, int status) {
 
 
     @Override
-    public List<String> handleCommonConfig(String host, int port, int maxMemory, int versionId) {
+    public List<String> handleCommonConfig(String host, int port, int maxMemory, String maxMemoryPolicy, int versionId) {
         //2018-08-24
         List<InstanceConfig> instanceConfigList = getByVesionAndType(ConstUtils.CACHE_REDIS_STANDALONE, versionId);
         if (CollectionUtils.isEmpty(instanceConfigList)) {
@@ -193,6 +193,10 @@ public List<String> handleCommonConfig(String host, int port, int maxMemory, int
                 configValue = String.format(configValue, percent);
             } else if (RedisConfigEnum.BIND.getKey().equals(configKey)) {
                 configValue = String.format(configValue, host);
+            } else if (RedisConfigEnum.MAXMEMORY_POLICY.getKey().equals(configKey)) {
+                if(StringUtils.isNotEmpty(maxMemoryPolicy)){
+                    configValue = maxMemoryPolicy;
+                }
             }
             configs.add(combineConfigKeyValue(configKey, configValue));
         }

cachecloud-web/src/main/java/com/sohu/cache/redis/impl/RedisDeployCenterImpl.java

@@ -1,6 +1,7 @@
 package com.sohu.cache.redis.impl;
 
 import com.google.common.collect.Lists;
+import com.sohu.cache.constant.AppDescEnum;
 import com.sohu.cache.constant.ClusterOperateResult;
 import com.sohu.cache.constant.InstanceStatusEnum;
 import com.sohu.cache.dao.AppDao;
@@ -570,7 +571,7 @@ private boolean runInstanceWithDefaultConfig(AppDesc appDesc, String host, Integ
             throw new RuntimeException(String.format("machine: %s version :%s is installed %s", host, redisResource, installStatus));
         }
         // 生成配置
-        List<String> configs = handleCommonConfig(host, port, maxMemory, appDesc.getVersionId());
+        List<String> configs = handleCommonConfig(host, port, maxMemory, appDesc.getMaxmemoryPolicy(), appDesc.getVersionId());
         if (isCluster) {
             configs.addAll(handleClusterConfig(port, appDesc.getVersionId()));
         }
@@ -633,7 +634,7 @@ public boolean bornConfigAndRunNode(AppDesc appDesc, InstanceInfo instanceInfo,
         SystemResource redisResource = resourceService.getResourceById(appDesc.getVersionId());
         String redisDir = redisResource == null ? ConstUtils.REDIS_DEFAULT_DIR : ConstUtils.getRedisDir(redisResource.getName());
         // 生成配置
-        List<String> configs = handleCommonConfig(host, port, maxMemory, appDesc.getVersionId());
+        List<String> configs = handleCommonConfig(host, port, maxMemory, appDesc.getMaxmemoryPolicy(), appDesc.getVersionId());
         if (isCluster) {
             configs.addAll(handleClusterConfig(port, appDesc.getVersionId()));
         }
@@ -757,10 +758,17 @@ private boolean runSentinel(AppDesc appDesc, String sentinelHost, String masterN
      * @param maxMemory
      * @return
      */
-    public List<String> handleCommonConfig(String host, int port, int maxMemory, int versionId) {
+    public List<String> handleCommonConfig(String host, int port, int maxMemory, Integer maxMemoryPolicyType, int versionId) {
         List<String> configs = null;
         try {
-            configs = redisConfigTemplateService.handleCommonConfig(host, port, maxMemory, versionId);
+            String maxMemoryPolicy = null;
+            if(maxMemoryPolicyType != null){
+                AppDescEnum.MaxmemoryPolicyType policyType = AppDescEnum.MaxmemoryPolicyType.getByType(maxMemoryPolicyType);
+                if(policyType != null){
+                    maxMemoryPolicy = policyType.getName();
+                }
+            }
+            configs = redisConfigTemplateService.handleCommonConfig(host, port, maxMemory, maxMemoryPolicy, versionId);
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
         }

cachecloud-web/src/main/java/com/sohu/cache/task/tasks/install/RedisServerInstallTask.java

@@ -1,5 +1,6 @@
 package com.sohu.cache.task.tasks.install;
 
+import com.sohu.cache.constant.AppDescEnum;
 import com.sohu.cache.entity.AppDesc;
 import com.sohu.cache.entity.SystemResource;
 import com.sohu.cache.protocol.RedisProtocol;
@@ -143,7 +144,7 @@ public TaskFlowStatusEnum pushConfig() {
         //实例基准目录
         String instanceRemoteBasePath = ConstUtils.CACHECLOUD_BASE_DIR;
         AppDesc appDesc = appService.getByAppId(appId);
-        List<String> configList = handleCommonConfig(host, port, maxMemory, appDesc.getVersionId(),isCluster);
+        List<String> configList = handleCommonConfig(host, port, maxMemory, appDesc.getMaxmemoryPolicy(), appDesc.getVersionId(), isCluster);
         if (CollectionUtils.isEmpty(configList)) {
             logger.error(marker, "appId {} port {} maxmemory {} versionId:{} instanceRemoteBasePath {} configList is empty", appId, port, maxMemory, appDesc.getVersionId(), instanceRemoteBasePath);
             return TaskFlowStatusEnum.ABORT;
@@ -232,9 +233,16 @@ public TaskFlowStatusEnum checkIsRun() {
      * @version 1.0
      * @date 2019/1/9
      */
-    private List<String> handleCommonConfig(String host, int port, int maxMemory, int versionId, boolean isCluster) {
+    private List<String> handleCommonConfig(String host, int port, int maxMemory, Integer maxMemoryPolicyType, int versionId, boolean isCluster) {
         try {
-            List<String> configs = redisConfigTemplateService.handleCommonConfig(host, port, maxMemory, versionId);
+            String maxMemoryPolicy = null;
+            if(maxMemoryPolicyType != null){
+                AppDescEnum.MaxmemoryPolicyType policyType = AppDescEnum.MaxmemoryPolicyType.getByType(maxMemoryPolicyType);
+                if(policyType != null){
+                    maxMemoryPolicy = policyType.getName();
+                }
+            }
+            List<String> configs = redisConfigTemplateService.handleCommonConfig(host, port, maxMemory, maxMemoryPolicy, versionId);
             if (isCluster) {
                 configs.addAll(redisConfigTemplateService.handleClusterConfig(port, versionId));
             }

cachecloud-web/src/main/java/com/sohu/cache/util/ConstUtils.java

@@ -45,7 +45,7 @@ public class ConstUtils {
     /**
      * 服务端版本
      */
-    public static final String CACHECLOUD_VERSION = "3.1.0";
+    public static final String CACHECLOUD_VERSION = "3.2.0";
     /**
      * 逗号
      */

cachecloud-web/src/main/java/com/sohu/cache/web/controller/AppController.java

@@ -1021,6 +1021,7 @@ public ModelAndView doAppInit(HttpServletRequest request,
         model.addAttribute("roomList", roomList);
         model.addAttribute("versionList", versionList);
         model.addAttribute("allModules", allModules);
+        model.addAttribute("policyList", AppDescEnum.MaxmemoryPolicyType.getAll());
 
         return new ModelAndView("app/jobIndex/appInitIndex");
     }

cachecloud-web/src/main/java/com/sohu/cache/web/controller/RedisConfigTemplateController.java

@@ -256,7 +256,7 @@ public ModelAndView preview(HttpServletRequest request, HttpServletResponse resp
         // 根据类型生成配置模板
         List<String> configList = new ArrayList<String>();
         if (ConstUtils.CACHE_REDIS_STANDALONE == type) {
-            configList = redisConfigTemplateService.handleCommonConfig(host, port, maxMemory, versionId);
+            configList = redisConfigTemplateService.handleCommonConfig(host, port, maxMemory, null, versionId);
         } else if (ConstUtils.CACHE_REDIS_SENTINEL == type) {
             configList = redisConfigTemplateService.handleSentinelConfig(masterName, host, port, host, sentinelPort, versionId);
         } else if (ConstUtils.CACHE_TYPE_REDIS_CLUSTER == type) {

cachecloud-web/src/main/resources/mapper/AppDao.xml

@@ -6,12 +6,16 @@
     <sql id="app_desc_add_fields">
         app_id,name,user_id,status,intro,create_time,passed_time,type,officer,
         ver_id,is_test,has_back_store,need_persistence,need_hot_back_up,forecase_qps,
-        forecast_obj_num,mem_alert_value,client_machine_room,app_key,client_conn_alert_value,hit_precent_alert_value,is_access_monitor,important_level,password,version_id,custom_password
+        forecast_obj_num,mem_alert_value,client_machine_room,app_key,client_conn_alert_value,
+        hit_precent_alert_value,is_access_monitor,important_level,password,version_id,
+        custom_password,maxmemory_policy
     </sql>
     <sql id="app_desc_fields">
         app_id,name,user_id,status,intro,create_time,passed_time,type,officer,
         ver_id,is_test,has_back_store,need_persistence,need_hot_back_up,forecase_qps,
-        forecast_obj_num,mem_alert_value,client_machine_room,app_key,client_conn_alert_value,hit_precent_alert_value,is_access_monitor,important_level,password "pkey",version_id,custom_password
+        forecast_obj_num,mem_alert_value,client_machine_room,app_key,client_conn_alert_value,
+        hit_precent_alert_value,is_access_monitor,important_level,password "pkey",version_id,
+        custom_password, maxmemory_policy
     </sql>
     <!--通过appId查询app的信息-->
     <select id="getAppDescById" resultType="AppDesc" parameterType="long">
@@ -47,7 +51,8 @@
             #{appId},#{name},#{userId},#{status},#{intro},#{createTime},#{passedTime},
 			#{type},#{officer},#{verId},#{isTest},#{hasBackStore},#{needPersistence},
 			#{needHotBackUp},#{forecaseQps},#{forecastObjNum},#{memAlertValue},#{clientMachineRoom},
-            #{appKey},#{clientConnAlertValue},#{hitPrecentAlertValue},#{isAccessMonitor},#{importantLevel},#{pkey},#{versionId},#{customPassword}
+            #{appKey},#{clientConnAlertValue},#{hitPrecentAlertValue},#{isAccessMonitor},
+            #{importantLevel},#{pkey},#{versionId},#{customPassword},#{maxmemoryPolicy}
             )
     </insert>
 

cachecloud-web/src/main/resources/mapper/AppDataMigrateStatusDao.xml

@@ -155,7 +155,7 @@
     </select>
 
     <update id="updateStatus">
-    	update app_data_migrate_status set status = #{status}, end_time=now() where id = #{id}
+    	update app_data_migrate_status set status = #{status}, end_time=now() where id = #{id} and status != 1
     </update>
 
     <select id="getAllOnMigrateId" resultType="long">

cachecloud-web/src/main/resources/spring/spring-mvc.xml

@@ -27,6 +27,9 @@
         <!-- 后台管理员验证 -->
         <mvc:interceptor>
             <mvc:mapping path="/manage/app/*"/>
+            <mvc:mapping path="/manage/app/**"/>
+            <mvc:mapping path="/manage/import/**"/>
+            <mvc:mapping path="/manage/task/**"/>
             <mvc:mapping path="/manage/instance/*"/>
             <mvc:mapping path="/manage/user/*"/>
             <mvc:mapping path="/manage/quartz/*"/>

cachecloud-web/src/main/webapp/WEB-INF/jsp/app/appDetail.jsp

@@ -117,6 +117,8 @@
                 <tr>
                     <td>应用描述</td>
                     <td>${appDetail.appDesc.intro}</td>
+                    <td>淘汰策略</td>
+                    <td>${appDetail.appDesc.maxmemoryPolicyDesc}</td>
                 </tr>
                 </tbody>
             </table>