[Opt][Incomplete]Optimize history processing

.github/workflows/buildapp-dev.yml

@@ -7,7 +7,7 @@ on:
   workflow_dispatch:
   push:
     branches:
-      - dev
+      - main
 
 
 jobs:
@@ -30,49 +30,25 @@ jobs:
       run: go mod download
 
     - name: Run build script
-      run: bash build.sh dev win
-
-  build_linux:
-    runs-on: ubuntu-20.04
-    steps:
-
-    - name: Checkout code
-      uses: actions/checkout@v3
-
-    - name: Set up Go
-      uses: actions/setup-go@v4
-      with:
-        go-version: '1.21'
-
-    - name: Install UPX
-      run: sudo apt-get install upx
-
-    - name: download modules
-      run: go mod download
-
-    - name: Run build script
-      run: bash build.sh dev linux
-
-  build_darwin:
-    runs-on: ubuntu-20.04
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
-
-    - name: Set up Go
-      uses: actions/setup-go@v4
-      with:
-        go-version: '1.21'
-
-    - name: Install UPX
-      run: sudo apt-get install upx
-
-    - name: download modules
-      run: go mod download
-
-    - name: Run build script
-      run: bash build.sh dev darwin
-
+    # 传入commit sha
+      run: bash build.sh ${{ github.sha }}
+
+    # 安装 rclone
+    # dev版本不另外发release
+    - name: Install rclone
+      run: |
+        cd ~
+        curl https://rclone.org/install.sh | sudo bash
+    # 配置 rclone
+    - name: Configure rclone
+      run: |
+        mkdir -p ~/.config/rclone
+        echo ${{ secrets.RCLONECONFIG }} > ~/.config/rclone
+    - name: Sync to OneDrive
+      run: |
+        sudo timedatectl set-timezone "Asia/Shanghai"
+        rclone mkdir one:/share/Mirouter-ui/dev/${{ github.sha }}
+        rclone sync ./build one:/share/Mirouter-ui/dev/${{ github.sha }} 
 
   builddocker:
     runs-on: ubuntu-20.04

.github/workflows/buildapp.yml

@@ -135,7 +135,23 @@ jobs:
                 "${{ steps.create_release.outputs.upload_url }}=$(basename $file)"
             fi
           done
-
+
+      # 安装 rclone
+      # dev版本不另外发release
+      - name: Install rclone
+        run: |
+          cd ~
+          curl https://rclone.org/install.sh | sudo bash
+      # 配置 rclone
+      - name: Configure rclone
+        run: |
+          mkdir -p ~/.config/rclone
+          echo ${{ secrets.RCLONECONFIG }} > ~/.config/rclone
+      - name: Sync to OneDrive
+        run: |
+          sudo timedatectl set-timezone "Asia/Shanghai"
+          rclone mkdir one:/share/Mirouter-ui/${{ github.event.inputs.version }}
+          rclone sync ./build one:/share/Mirouter-ui/${{ github.event.inputs.version }}
   builddocker:
     runs-on: ubuntu-20.04
     steps:

README.md

@@ -110,6 +110,10 @@
 | sampletime        | 300 | 采样时间间隔(s)                    |
 | maxsaved         | 8640                     | 最多记录条数                                |
 
+> [!NOTE]  
+> 保存数据条数过多可能会造成前端页面卡顿
+> 同时，为了减小历史数据拟合时产生的误差，sampletime应不超过600
+
 命令行参数：
 
 | 参数            | 解释                             |

modules/database/base.go

@@ -2,12 +2,12 @@ package database
 
 import (
 	"io"
-	"math"
 	"strconv"
 
 	"encoding/json"
 	"fmt"
 	"main/modules/config"
+	"math"
 	"net/http"
 
 	"github.com/glebarez/sqlite"
@@ -29,6 +29,25 @@ type History struct {
 	DownTotal float64
 	DeviceNum int
 }
+type DevicesHistory struct {
+	gorm.Model
+	Mac       string
+	UpSpeed   float64
+	DownSpeed float64
+	UpTotal   float64
+	DownTotal float64
+}
+type DeviceInfo struct {
+	DevName          string `json:"devname"`
+	Download         int64  `json:"download,string"`
+	DownSpeed        int    `json:"downspeed,string"`
+	Mac              string `json:"mac"`
+	MaxDownloadSpeed int    `json:"maxdownloadspeed,string"`
+	MaxUploadSpeed   int    `json:"maxuploadspeed,string"`
+	Online           int    `json:"online,string"`
+	Upload           int64  `json:"upload,string"`
+	UpSpeed          int    `json:"upspeed,string"`
+}
 
 type Dev struct {
 	Password   string `json:"password"`
@@ -51,7 +70,8 @@ func CheckDatabase(databasepath string) {
 	// Check if the history table exists, if not, create it
 	err = db.AutoMigrate(&History{})
 	checkErr(err)
-
+	err = db.AutoMigrate(&DevicesHistory{})
+	checkErr(err)
 	// Perform CRUD operations on the history table using db.Create, db.First, db.Update, db.Delete methods
 }
 
@@ -68,7 +88,7 @@ func Savetodb(databasepath string, dev []config.Dev, tokens map[int]string, maxs
 	for i, d := range dev {
 		ip := d.IP
 		routerNum := i
-		cpu, cpu_tp, mem, upSpeed, downSpeed, upTotal, downTotal, deviceNum := getDeviceStats(i, tokens, ip)
+		cpu, cpu_tp, mem, upSpeed, downSpeed, upTotal, downTotal, deviceNum, devs := getRouterStats(i, tokens, ip)
 		var count int64
 		db.Model(&History{}).Where("router_num = ?", routerNum).Count(&count)
 		if count >= int64(maxsaved) {
@@ -88,6 +108,40 @@ func Savetodb(databasepath string, dev []config.Dev, tokens map[int]string, maxs
 			DownTotal: downTotal,
 			DeviceNum: deviceNum,
 		})
+		for _, dev := range devs {
+			devMap := dev.(map[string]interface{})
+
+			data, err := json.Marshal(devMap)
+			checkErr(err)
+
+			var info DeviceInfo
+			err = json.Unmarshal(data, &info)
+			checkErr(err)
+			mac := info.Mac
+			upSpeed := float64(info.UpSpeed) / 1024 / 1024
+			downSpeed := float64(info.DownSpeed) / 1024 / 1024
+			upTotal := float64(info.Upload) / 1024 / 1024
+			downTotal := float64(info.Download) / 1024 / 1024
+			db.Model(&DevicesHistory{}).Where("mac = ?", routerNum).Count(&count)
+			if count >= int64(maxsaved) {
+				logrus.Debug("删除历史数据")
+				db.Exec("DELETE FROM histories WHERE mac = ? AND created_at = (SELECT MIN(created_at) FROM histories WHERE mac = ? );", mac, mac)
+
+			}
+			db.Create(&History{
+				Ip:        ip,
+				RouterNum: routerNum,
+				Cpu:       cpu,
+				Cpu_tp:    cpu_tp,
+				Mem:       mem,
+				UpSpeed:   upSpeed,
+				DownSpeed: downSpeed,
+				UpTotal:   upTotal,
+				DownTotal: downTotal,
+				DeviceNum: deviceNum,
+			})
+		}
+
 	}
 }
 
@@ -96,28 +150,16 @@ func Getdata(databasepath string, routernum int) []History {
 	checkErr(err)
 	var history []History
 	db.Where("router_num = ?", routernum).Find(&history)
+	// 处理浮点数精度问题
+	for i := range history {
+		history[i].Cpu = round(history[i].Cpu, .5, 2)
+	}
 	return history
 }
 
-// getDeviceStats retrieves the device statistics from the specified router.
-//
-// Parameters:
-// - routernum: The router number.
-// - tokens: A map containing the tokens.
-// - ip: The IP address of the router.
-//
-// Returns:
-// - cpuload: The Cpu load.
-// - cpu_tp: The Cpu temperature.
-// - memusage: The memory usage.
-// - upspeed: The upload speed.
-// - downspeed: The download speed.
-// - uploadtotal: The total upload amount.
-// - downloadtotal: The total download amount.
-// - devicenum_now: The number of online devices.
-func getDeviceStats(routernum int, tokens map[int]string, ip string) (float64, int, float64, float64, float64, float64, float64, int) {
+func getRouterStats(routernum int, tokens map[int]string, ip string) (float64, int, float64, float64, float64, float64, float64, int, []interface{}) {
 	if tokens[routernum] == "" {
-		return 0, 0, 0, 0, 0, 0, 0, 0
+		return 0, 0, 0, 0, 0, 0, 0, 0, []interface{}{}
 	}
 	url := fmt.Sprintf("http://%s/cgi-bin/luci/;stok=%s/api/misystem/status", ip, tokens[routernum])
 	resp, err := http.Get(url)
@@ -131,20 +173,35 @@ func getDeviceStats(routernum int, tokens map[int]string, ip string) (float64, i
 	downspeed, _ := strconv.ParseFloat(result["wan"].(map[string]interface{})["downspeed"].(string), 64)
 	uploadtotal, _ := strconv.ParseFloat(result["wan"].(map[string]interface{})["upload"].(string), 64)
 	downloadtotal, _ := strconv.ParseFloat(result["wan"].(map[string]interface{})["download"].(string), 64)
-	cpuload := roundToOneDecimal(result["cpu"].(map[string]interface{})["load"].(float64) * 100)
+	cpuload := result["cpu"].(map[string]interface{})["load"].(float64) * 100
 	cpu_tp := int(result["temperature"].(float64))
-	memusage := roundToOneDecimal(result["mem"].(map[string]interface{})["usage"].(float64) * 100)
+	memusage := result["mem"].(map[string]interface{})["usage"].(float64) * 100
 	devicenum_now := int(result["count"].(map[string]interface{})["online"].(float64))
+	devs := result["devs"].([]interface{})
 
-	return cpuload, cpu_tp, memusage, upspeed, downspeed, uploadtotal, downloadtotal, devicenum_now
+	return cpuload, cpu_tp, memusage, upspeed, downspeed, uploadtotal, downloadtotal, devicenum_now, devs
 }
 
-func roundToOneDecimal(num float64) float64 {
-	return math.Round(num*100) / 100
-}
+// func roundToOneDecimal(num float64) float64 {
+// 	return math.Round(num*100) / 100
+// }
 
 func checkErr(err error) {
 	if err != nil {
 		logrus.Debug(err)
 	}
 }
+func round(val float64, roundOn float64, places int) (newVal float64) {
+	var rounder float64
+	pow := math.Pow(10, float64(places))
+	intermed := val * pow
+	_, frac := math.Modf(intermed)
+
+	if frac >= roundOn {
+		rounder = math.Ceil(intermed)
+	} else {
+		rounder = math.Floor(intermed)
+	}
+	newVal = rounder / pow
+	return
+}