diff --git a/auto-check/auto-check.yaml b/auto-check/auto-check.yaml index 3112a74..736edd8 100644 --- a/auto-check/auto-check.yaml +++ b/auto-check/auto-check.yaml @@ -16,13 +16,16 @@ ssh: # --- 压力测试 --- stress: - types: "cpu,memory" + types: "cpu,memory,disk,full" # 单项在前、综合(full)最后 duration: 30s threads: 4 + temp_limit: 90 # 温度上限(°C)，超过判失败，0=不判定 + auto_install: true # 缺工具时在线安装（apt/dnf/yum 等），需要目标机可联网 + use_mirror: false # 在线安装是否切换国内镜像源(清华TUNA)。国内网络卡顿/超时再改 true # --- 工作流 --- workflow: - interval: 60s + interval: 180s # --- 报告与打印 --- report: diff --git a/auto-check/go.mod b/auto-check/go.mod index 4794628..a4fa172 100644 --- a/auto-check/go.mod +++ b/auto-check/go.mod @@ -5,12 +5,12 @@ go 1.25.0 require ( github.com/spf13/cobra v1.10.2 golang.org/x/crypto v0.54.0 + golang.org/x/text v0.41.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.10 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.41.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/auto-check/go.sum b/auto-check/go.sum index 302718b..fd2f91f 100644 --- a/auto-check/go.sum +++ b/auto-check/go.sum @@ -16,6 +16,7 @@ golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/auto-check/pkg/config/config.go b/auto-check/pkg/config/config.go index b91b7f8..8576774 100644 --- a/auto-check/pkg/config/config.go +++ b/auto-check/pkg/config/config.go @@ -38,13 +38,17 @@ type SSHConfig struct { Password string `yaml:"password"` KeyFile string `yaml:"key"` Port int `yaml:"port"` + SudoPwd string `yaml:"sudo_password"` // 非 root 用户安装工具时使用的 sudo 密码 } // StressConfig 压力测试 type StressConfig struct { - Types string `yaml:"types"` - Duration time.Duration `yaml:"duration"` - Threads int `yaml:"threads"` + Types string `yaml:"types"` + Duration time.Duration `yaml:"duration"` + Threads int `yaml:"threads"` + TempLimit float64 `yaml:"temp_limit"` // 温度上限(°C)，0=不判定 + AutoInstall bool `yaml:"auto_install"` // 缺工具时尝试在线安装（apt/dnf/yum 等） + UseMirror bool `yaml:"use_mirror"` // 在线安装时切换国内镜像源（清华 TUNA），国内网络卡顿/超时再开启 } // ReportConfig 报告 @@ -83,9 +87,12 @@ func Default() *Config { Port: 22, }, Stress: StressConfig{ - Types: "cpu,memory", - Duration: 30 * time.Second, - Threads: 4, + Types: "cpu,memory", + Duration: 30 * time.Second, + Threads: 4, + TempLimit: 90, + AutoInstall: true, + UseMirror: false, }, Workflow: WorkflowConfig{ Interval: 10 * time.Second, diff --git a/auto-check/pkg/report/device.go b/auto-check/pkg/report/device.go index f128210..ac5f556 100644 --- a/auto-check/pkg/report/device.go +++ b/auto-check/pkg/report/device.go @@ -18,11 +18,14 @@ func BuildDeviceReport(dev *model.Device, rpt *stress.Report) *Report { b.AddHost(HostResult{MAC: dev.MAC, IP: dev.IP}) for _, res := range rpt.Results { b.AddStressResult(dev.MAC, StressResult{ - Type: string(res.Type), - Status: res.Status, - Duration: res.Duration.String(), - Error: res.Error, - Output: res.Output, + Type: string(res.Type), + Status: res.Status, + Duration: res.Duration.String(), + Score: res.Score, + ScoreUnit: stress.ScoreUnit(res.Type), + Metrics: res.OrderedMetrics(), + Error: res.Error, + Output: res.Output, }) } b.EndPhase(rpt.Summary()) diff --git a/auto-check/pkg/report/report.go b/auto-check/pkg/report/report.go index 27187fa..59d08c9 100644 --- a/auto-check/pkg/report/report.go +++ b/auto-check/pkg/report/report.go @@ -7,6 +7,8 @@ import ( "path/filepath" "strings" "time" + + "auto-check/pkg/stress" ) // PhaseResult 单阶段结果 @@ -19,23 +21,25 @@ type PhaseResult struct { // HostResult 单台设备结果（MAC 为唯一标识） type HostResult struct { - MAC string `json:"mac" yaml:"mac"` - IP string `json:"ip" yaml:"ip"` - Alive bool `json:"alive" yaml:"alive"` - OpenPort int `json:"open_port,omitempty" yaml:"open_port,omitempty"` - SSHLogin bool `json:"ssh_login" yaml:"ssh_login"` - SSHErr string `json:"ssh_error,omitempty" yaml:"ssh_error,omitempty"` - Stress []StressResult `json:"stress,omitempty" yaml:"stress,omitempty"` - Curves map[string][]float64 `json:"curves,omitempty" yaml:"curves,omitempty"` // 指标名 → 时间序列（渲染为 sparkline） + MAC string `json:"mac" yaml:"mac"` + IP string `json:"ip" yaml:"ip"` + Alive bool `json:"alive" yaml:"alive"` + OpenPort int `json:"open_port,omitempty" yaml:"open_port,omitempty"` + SSHLogin bool `json:"ssh_login" yaml:"ssh_login"` + SSHErr string `json:"ssh_error,omitempty" yaml:"ssh_error,omitempty"` + Stress []StressResult `json:"stress,omitempty" yaml:"stress,omitempty"` } // StressResult 单项压力测试结果 type StressResult struct { - Type string `json:"type" yaml:"type"` - Status string `json:"status" yaml:"status"` // pass / fail / skip - Duration string `json:"duration" yaml:"duration"` - Output string `json:"output,omitempty" yaml:"output,omitempty"` - Error string `json:"error,omitempty" yaml:"error,omitempty"` + Type string `json:"type" yaml:"type"` + Status string `json:"status" yaml:"status"` // pass / fail / skip + Duration string `json:"duration" yaml:"duration"` + Score float64 `json:"score,omitempty" yaml:"score,omitempty"` + ScoreUnit string `json:"score_unit,omitempty" yaml:"score_unit,omitempty"` + Metrics []stress.MetricItem `json:"metrics,omitempty" yaml:"metrics,omitempty"` + Output string `json:"output,omitempty" yaml:"output,omitempty"` + Error string `json:"error,omitempty" yaml:"error,omitempty"` } // Report 检测报告 @@ -209,6 +213,12 @@ func (r *Report) ToText() string { icon = "✗" } sb.WriteString(fmt.Sprintf(" │ %s %-10s 耗时: %s\n", icon, s.Type, s.Duration)) + if s.Score > 0 { + sb.WriteString(fmt.Sprintf(" │ 跑分: %.1f %s\n", s.Score, s.ScoreUnit)) + } + for _, m := range s.Metrics { + sb.WriteString(fmt.Sprintf(" │ %s: %.1f%s\n", m.Label, m.Value, m.Unit)) + } if s.Error != "" { sb.WriteString(fmt.Sprintf(" │ 错误: %s\n", s.Error)) } diff --git a/auto-check/pkg/report/template.go b/auto-check/pkg/report/template.go index c97a3b8..1e6d7ca 100644 --- a/auto-check/pkg/report/template.go +++ b/auto-check/pkg/report/template.go @@ -2,7 +2,6 @@ package report import ( "fmt" - "math" "strings" "text/template" "time" @@ -50,29 +49,6 @@ func RenderTemplate(tpl string, data interface{}, cfg TemplateConfig) (string, e }, // 分隔线 "hr": func(sharp rune) string { return strings.Repeat(string(sharp), cfg.Width) }, - // 曲线图（时间序列 → sparkline） - "sparkline": func(data []float64, width int) string { - return Sparkline(data, width) - }, - // 曲线摘要: 最高/最低/平均 - "curveSummary": func(data []float64) string { - if len(data) == 0 { - return "无数据" - } - min, max := math.MaxFloat64, -math.MaxFloat64 - var sum float64 - for _, v := range data { - if v < min { - min = v - } - if v > max { - max = v - } - sum += v - } - avg := sum / float64(len(data)) - return fmt.Sprintf("max=%.1f min=%.1f avg=%.1f", max, min, avg) - }, } t, err := template.New("report").Funcs(funcMap).Parse(tpl) @@ -86,76 +62,3 @@ func RenderTemplate(tpl string, data interface{}, cfg TemplateConfig) (string, e } return sb.String(), nil } - -// ============================ -// Sparkline — 80mm 热敏纸曲线图 -// ============================ - -// sparkChars 8 级块状字符（低→高） -var sparkChars = []rune("▁▂▃▄▅▆▇█") - -// Sparkline 将时间序列渲染为一行块状曲线 -// width 为字符宽度（80mm 纸 ≤42） -func Sparkline(data []float64, width int) string { - if len(data) == 0 { - return "(无数据)" - } - if width <= 0 { - width = 42 - } - if width > 42 { - width = 42 - } - - // 单点 - if len(data) == 1 { - return fmt.Sprintf("▊ %.1f", data[0]) - } - - // 按宽度采样（超出宽度时等距抽取） - step := float64(len(data)) / float64(width) - sampled := make([]float64, 0, width) - for i := 0; i < width && int(float64(i)*step) < len(data); i++ { - idx := int(float64(i) * step) - if idx >= len(data) { - break - } - sampled = append(sampled, data[idx]) - } - - // 归一化到 0-7 - min, max := math.MaxFloat64, -math.MaxFloat64 - for _, v := range sampled { - if v < min { - min = v - } - if v > max { - max = v - } - } - - var sb strings.Builder - if max-min < 1e-9 { - // 全相同值 - idx := 0 - if max > 0 { - idx = 3 - } - for range sampled { - sb.WriteRune(sparkChars[idx]) - } - return sb.String() - } - - for _, v := range sampled { - idx := int((v - min) / (max - min) * 7) - if idx < 0 { - idx = 0 - } - if idx > 7 { - idx = 7 - } - sb.WriteRune(sparkChars[idx]) - } - return sb.String() -} diff --git a/auto-check/pkg/report/templates/report_80mm.tmpl b/auto-check/pkg/report/templates/report_80mm.tmpl index 850e07d..1a5d7b5 100644 --- a/auto-check/pkg/report/templates/report_80mm.tmpl +++ b/auto-check/pkg/report/templates/report_80mm.tmpl @@ -18,9 +18,8 @@ {{ end }}{{ end }}{{ if eq .Phase "SSH 登录" }}{{ range .Hosts }}{{ if .SSHLogin }}✓{{ else }}✗{{ end }} {{ padRight .MAC 19 }} {{ padRight .IP 15 }} {{ .SSHErr }} {{ end }}{{ end }}{{ if eq .Phase "压力测试" }}{{ range .Hosts }}{{ padRight .MAC 19 }} {{ padRight .IP 15 }} {{ range .Stress }}{{ if eq .Status "pass" }}✓{{ else }}✗{{ end }} {{ padRight .Type 12 }} {{ .Status }} {{ .Duration }} -{{ if .Error }} ERR: {{ trunc .Error 34 }}{{ end }}{{ end }}{{ range $name, $data := .Curves }} [{{ $name }}] {{ curveSummary $data }} - {{ sparkline $data 38 }} -{{ end }}{{ end }}{{ end }}{{ end }} +{{ if .Score }} 跑分: {{ printf "%.1f" .Score }} {{ .ScoreUnit }}{{ end }}{{ range .Metrics }} {{ .Label }}: {{ printf "%.1f" .Value }}{{ .Unit }} +{{ end }}{{ if .Error }} ERR: {{ trunc .Error 34 }}{{ end }}{{ end }}{{ end }}{{ end }}{{ end }} {{ "" }} {{ hr '═' }} 报告结束 diff --git a/auto-check/pkg/sshclient/sshclient.go b/auto-check/pkg/sshclient/sshclient.go index c12573e..71fd6eb 100644 --- a/auto-check/pkg/sshclient/sshclient.go +++ b/auto-check/pkg/sshclient/sshclient.go @@ -4,6 +4,7 @@ import ( "fmt" "net" "os" + "strings" "time" "golang.org/x/crypto/ssh" @@ -16,6 +17,7 @@ type Config struct { KeyFile string Port int Timeout time.Duration + SudoPwd string // 非 root 用户执行特权命令(sudo)时的密码 } // NewConfig 创建配置 @@ -54,9 +56,61 @@ func RunCommand(client *ssh.Client, command string) (string, error) { out, err := session.CombinedOutput(command) if err != nil { - return string(out), fmt.Errorf("命令执行失败: %w", err) + return cleanOutput(out), fmt.Errorf("命令执行失败: %w", err) } - return string(out), nil + return cleanOutput(out), nil +} + +// isNoiseLine 判断某行是否为登录横幅/欢迎语等噪声（fnOS 的 admin 用户会把 +// "Could not chdir to home directory" 之类写入 stdout，污染命令解析）。 +func isNoiseLine(line string) bool { + s := strings.TrimSpace(line) + if s == "" { + return false + } + prefixes := []string{ + "Could not chdir to", + "Last login:", + "Welcome to", + "/etc/motd", + "Permission denied", + "bash: line 1:", + } + for _, p := range prefixes { + if strings.HasPrefix(s, p) { + return true + } + } + // 某些横幅是 "Could: command not found" 这类由前面噪声行残生， + // 形如 "xxx: command not found" 且前半段是噪声关键词的也过滤 + if strings.Contains(s, ": command not found") { + return true + } + return false +} + +// cleanOutput 过滤掉登录横幅等噪声行，仅保留真实命令输出 +func cleanOutput(raw []byte) string { + lines := strings.Split(string(raw), "\n") + var kept []string + for _, ln := range lines { + if isNoiseLine(ln) { + continue + } + kept = append(kept, ln) + } + return strings.Join(kept, "\n") +} + +// EnsureHome 确保远端登录用户的 HOME 目录存在，消除 fnOS "Could not chdir to +// home directory" 登录横幅（该横幅会写入 stdout 污染命令输出）。 +func EnsureHome(client *ssh.Client) { + out, err := RunCommand(client, "echo $HOME") + home := strings.TrimSpace(out) + if err != nil || home == "" || home == "/" { + return + } + RunCommand(client, fmt.Sprintf("mkdir -p %s", home)) } // buildSSHConfig 构建认证配置 diff --git a/auto-check/pkg/stress/chart.go b/auto-check/pkg/stress/chart.go index f34becd..f3a1ff2 100644 --- a/auto-check/pkg/stress/chart.go +++ b/auto-check/pkg/stress/chart.go @@ -5,127 +5,264 @@ import ( "fmt" "os" "strings" + "time" ) -// chartData 给 JS 用的数据 -type chartData struct { - Labels []string `json:"labels"` - CPU []float64 `json:"cpu"` - Temp []float64 `json:"temp"` - Title string `json:"title"` +// metricSpec 单条曲线规格 +type metricSpec struct { + Key string `json:"key"` + Label string `json:"label"` + Color string `json:"color"` + Unit string `json:"unit"` } -// WriteChartHTML 根据采样数据生成折线图 HTML 文件，返回文件路径 -func WriteChartHTML(ip string, result Result, reportDir string) (string, error) { - if len(result.Samples) == 0 { - return "", nil - } +// chartSpec 图表规格（左轴 / 右轴） +type chartSpec struct { + Left []metricSpec + Right []metricSpec + LeftPct bool + LeftTitle string + RightTitle string +} - cd := chartData{ - Title: fmt.Sprintf("%s - %s 压测曲线 (%s)", ip, result.Type, result.Duration), - } - for _, s := range result.Samples { - cd.Labels = append(cd.Labels, s.Time) - cd.CPU = append(cd.CPU, s.CPU) - cd.Temp = append(cd.Temp, s.Temp) - } +// kvPair 指标明细 +type kvPair struct { + Label string `json:"label"` + Value string `json:"value"` +} - dataJSON, err := json.Marshal(cd) +// metricSeries 单条曲线序列 +type metricSeries struct { + Label string `json:"label"` + Color string `json:"color"` + Data []float64 `json:"data"` + Unit string `json:"unit"` +} + +// chartPayload 图表数据 +type chartPayload struct { + Labels []string `json:"labels"` + LeftSets []metricSeries `json:"left_sets"` + RightSets []metricSeries `json:"right_sets"` + LeftTitle string `json:"left_title"` + RightTitle string `json:"right_title"` + LeftPct bool `json:"left_pct"` +} + +// testCard 单个测试卡片数据 +type testCard struct { + Name string `json:"name"` + Label string `json:"label"` + Status string `json:"status"` + Duration string `json:"duration"` + Score float64 `json:"score"` + ScoreUnit string `json:"score_unit"` + ScoreHint string `json:"score_hint"` + Metrics []kvPair `json:"metrics"` + Output string `json:"output"` + Chart *chartPayload `json:"chart,omitempty"` +} + +// reportHTMLData 报告整体数据 +type reportHTMLData struct { + Title string `json:"title"` + IP string `json:"ip"` + Time string `json:"time"` + Duration string `json:"duration"` + Overall string `json:"overall"` + Summary string `json:"summary"` + SysInfo map[string]string `json:"sys_info"` + Cards []testCard `json:"cards"` +} + +// WriteReportHTML 生成综合 HTML 报告（跑分卡片 + 多曲线图），返回文件路径 +func WriteReportHTML(ip string, report *Report, reportDir string) (string, error) { + data := buildReportData(ip, report) + dataJSON, err := json.Marshal(data) if err != nil { return "", err } - html := strings.Replace(chartHTMLTemplate, "__DATA__", string(dataJSON), 1) + html := strings.Replace(reportHTMLTemplate, "__DATA__", string(dataJSON), 1) - os.MkdirAll(reportDir, 0755) - filename := fmt.Sprintf("%s/chart-%s-%s.html", reportDir, result.Type, SanitizeFilename(ip)) + if err := os.MkdirAll(reportDir, 0755); err != nil { + return "", err + } + filename := fmt.Sprintf("%s/report-%s.html", reportDir, SanitizeFilename(ip)) if err := os.WriteFile(filename, []byte(html), 0644); err != nil { return "", err } return filename, nil } +func buildReportData(ip string, report *Report) reportHTMLData { + data := reportHTMLData{ + Title: fmt.Sprintf("%s · 硬件压力测试报告", ip), + IP: ip, + Time: report.StartTime.Format("2006-01-02 15:04:05"), + Duration: report.Duration.Round(time.Millisecond).String(), + Overall: Status(report), + Summary: report.Summary(), + SysInfo: report.SysInfo, + } + for _, res := range report.Results { + data.Cards = append(data.Cards, buildCard(res)) + } + return data +} + +func buildCard(res Result) testCard { + card := testCard{ + Name: string(res.Type), + Label: testLabel(res.Type), + Status: res.Status, + Duration: res.Duration.Round(time.Millisecond).String(), + Score: res.Score, + ScoreUnit: ScoreUnit(res.Type), + ScoreHint: scoreHint(res.Type), + Output: res.Output, + } + for _, name := range sortedMetricNames(res.Metrics) { + card.Metrics = append(card.Metrics, kvPair{ + Label: MetricLabel(name), + Value: fmt.Sprintf("%.1f%s", res.Metrics[name], MetricUnit(name)), + }) + } + if len(res.Samples) > 0 { + if payload, ok := buildChart(res); ok { + card.Chart = &payload + } + } + return card +} + +func buildChart(res Result) (chartPayload, bool) { + spec := chartSpecFor(res.Type) + if len(spec.Left) == 0 && len(spec.Right) == 0 { + return chartPayload{}, false + } + + payload := chartPayload{ + LeftTitle: spec.LeftTitle, + RightTitle: spec.RightTitle, + LeftPct: spec.LeftPct, + } + for _, s := range res.Samples { + payload.Labels = append(payload.Labels, s.Time) + } + + hasData := func(key string) bool { + for _, s := range res.Samples { + if s.HasMetric(key) { + return true + } + } + return false + } + buildSeries := func(specs []metricSpec) []metricSeries { + var out []metricSeries + for _, m := range specs { + if !hasData(m.Key) { + continue + } + series := metricSeries{Label: m.Label, Color: m.Color, Unit: m.Unit} + for _, s := range res.Samples { + series.Data = append(series.Data, s.Values[m.Key]) + } + out = append(out, series) + } + return out + } + + payload.LeftSets = buildSeries(spec.Left) + payload.RightSets = buildSeries(spec.Right) + return payload, true +} + +// testLabel 测试项中文名 +func testLabel(t TestType) string { + switch t { + case TestCPU: + return "CPU 压力测试" + case TestMemory: + return "内存压力测试" + case TestDiskIO: + return "磁盘 IO 测试" + case TestMemNative: + return "内存精压测试" + case TestFull: + return "综合工况测试" + case MonitorTemp: + return "温度监控" + case MonitorDmesg: + return "内核日志检查" + default: + return string(t) + } +} + +// scoreHint 跑分含义说明 +func scoreHint(t TestType) string { + switch t { + case TestCPU: + return "每秒操作数（bogo ops/s）" + case TestMemory: + return "每秒操作数（bogo ops/s）" + case TestDiskIO: + return "读写带宽合计（MB/s）" + case TestFull: + return "综合每秒操作数（bogo ops/s）" + case TestMemNative: + return "内存带宽（MB/s）" + default: + return "" + } +} + +// chartSpecFor 返回某类测试的曲线规格 +func chartSpecFor(t TestType) chartSpec { + switch t { + case TestCPU: + return chartSpec{ + Left: []metricSpec{{Key: "cpu", Label: "CPU 利用率", Color: "#f97316", Unit: "%"}}, + Right: []metricSpec{{Key: "temp", Label: "温度", Color: "#ef4444", Unit: "°C"}}, + LeftPct: true, + LeftTitle: "利用率 (%)", + RightTitle: "温度 (°C)", + } + case TestMemory: + return chartSpec{ + Left: []metricSpec{{Key: "mem_used_pct", Label: "内存使用率", Color: "#8b5cf6", Unit: "%"}}, + Right: []metricSpec{{Key: "temp", Label: "温度", Color: "#ef4444", Unit: "°C"}}, + LeftPct: true, + LeftTitle: "使用率 (%)", + RightTitle: "温度 (°C)", + } + case TestDiskIO: + return chartSpec{ + Left: []metricSpec{{Key: "iops_total", Label: "IOPS", Color: "#22c55e", Unit: ""}}, + Right: []metricSpec{{Key: "bw_total", Label: "吞吐", Color: "#3b82f6", Unit: "MB/s"}}, + LeftPct: false, + LeftTitle: "IOPS", + RightTitle: "吞吐 (MB/s)", + } + case TestFull: + return chartSpec{ + Left: []metricSpec{ + {Key: "cpu", Label: "CPU 利用率", Color: "#f97316", Unit: "%"}, + {Key: "mem_used_pct", Label: "内存使用率", Color: "#8b5cf6", Unit: "%"}, + }, + Right: []metricSpec{{Key: "temp", Label: "温度", Color: "#ef4444", Unit: "°C"}}, + LeftPct: true, + LeftTitle: "利用率 (%)", + RightTitle: "温度 (°C)", + } + default: + return chartSpec{} + } +} + // SanitizeFilename 清理 IP 中的特殊字符，用于文件名 func SanitizeFilename(ip string) string { return strings.ReplaceAll(ip, ":", "_") } - -const chartHTMLTemplate = ` - - - - -压测曲线 - - - - -
-

- -
- CPU 使用率 (%) - 温度 (°C) -
-
- - -` diff --git a/auto-check/pkg/stress/metrics.go b/auto-check/pkg/stress/metrics.go index e62cfa4..2294d04 100644 --- a/auto-check/pkg/stress/metrics.go +++ b/auto-check/pkg/stress/metrics.go @@ -1,6 +1,7 @@ package stress import ( + "fmt" "time" ) @@ -10,10 +11,11 @@ import ( // Metric 测试指标（定义测什么、用什么工具、怎么判结果） type Metric struct { - Name string // 指标名称（cpu/memory/disk/memnative/full） - Tool string // 依赖的工具（stress-ng/stressapptest） - Enabled bool // 是否启用 - IsMonitor bool // 是否为监控指标（temp/dmesg，自动附加） + Name string // 指标名称（cpu/memory/disk/memnative/full） + Tool string // 依赖的工具（stress-ng/stressapptest/fio） + Enabled bool // 是否启用 + IsMonitor bool // 是否为监控指标（temp/dmesg，自动附加） + DisableReason string // 禁用时展示的原因 } // BuildMetrics 根据配置和可用工具，生成要执行的指标列表 @@ -28,16 +30,36 @@ func BuildMetrics(types []TestType, tools ToolSet) []Metric { case TestMemory: m.Tool = "stress-ng" case TestDiskIO: - m.Tool = "stress-ng" + // disk 优先 fio；无 fio 可降级 stress-ng + switch { + case tools.Has("fio"): + m.Tool = "fio" + case tools.Has("stress-ng"): + m.Tool = "stress-ng" + default: + m.Tool = "fio" + } case TestMemNative: m.Tool = "stressapptest" case TestFull: m.Tool = "stress-ng" default: m.Enabled = false + m.DisableReason = "未知测试类型" } - if m.Tool != "" && !tools.Has(m.Tool) { - m.Enabled = false // 工具不可用，禁用 + + // 根据工具可用性决定是否禁用，并记录原因 + switch t { + case TestDiskIO: + if !tools.Has("fio") && !tools.Has("stress-ng") { + m.Enabled = false + m.DisableReason = "缺少 fio 或 stress-ng" + } + default: + if m.Tool != "" && !tools.Has(m.Tool) { + m.Enabled = false + m.DisableReason = fmt.Sprintf("缺少 %s", m.Tool) + } } metrics = append(metrics, m) } @@ -61,6 +83,7 @@ type Config struct { DiskSizeMB int // 0=默认1024 DiskDir string // 空=自动临时目录 TempLogInt time.Duration // 0=10s + TempLimit float64 // 温度上限(°C)，0=不判定 } // DefaultConfig 默认配置 @@ -70,5 +93,6 @@ func DefaultConfig() Config { Threads: 4, DiskSizeMB: 1024, TempLogInt: 10 * time.Second, + TempLimit: 90, } } diff --git a/auto-check/pkg/stress/report_html.go b/auto-check/pkg/stress/report_html.go new file mode 100644 index 0000000..74d2cd2 --- /dev/null +++ b/auto-check/pkg/stress/report_html.go @@ -0,0 +1,126 @@ +package stress + +const reportHTMLTemplate = ` + + + + +硬件压力测试报告 + + + + +
+
+
+
+ 设备 + 测试时间 + 总耗时 +
+
+
+ + +
+
+
+
+ + +` diff --git a/auto-check/pkg/stress/runner.go b/auto-check/pkg/stress/runner.go index bf3aaf8..f6f5cbd 100644 --- a/auto-check/pkg/stress/runner.go +++ b/auto-check/pkg/stress/runner.go @@ -4,6 +4,7 @@ import ( "fmt" "net" "os" + "sort" "strconv" "strings" "time" @@ -24,18 +25,20 @@ var Results = make(map[string]*Report) // Runner 压力测试执行器 type Runner struct { - cfg Config - types []TestType - client *ssh.Client - tools ToolSet - metrics []Metric + cfg Config + types []TestType + client *ssh.Client + tools ToolSet + metrics []Metric + installFailed []string // 自动安装仍失败的工具 } -// NewRunner 创建执行器（检测工具 → 构建指标 → 生成脚本 → 执行） -func NewRunner(cfg Config, types []TestType, client *ssh.Client) *Runner { - tools := DetectTools(client) +// NewRunner 创建执行器（检查/在线安装 → 构建指标 → 生成脚本 → 执行） +func NewRunner(cfg Config, types []TestType, client *ssh.Client, autoInstall bool, sudoPwd string) *Runner { + // 1. 检测工具，缺失时按发行版在线安装（可配置国内镜像源加速） + tools, failed := EnsureTools(client, requiredTools, autoInstall, sudoPwd) metrics := BuildMetrics(types, tools) - return &Runner{cfg: cfg, types: types, client: client, tools: tools, metrics: metrics} + return &Runner{cfg: cfg, types: types, client: client, tools: tools, metrics: metrics, installFailed: failed} } // Run 执行压力测试（生成脚本 → 远程执行 → 解析结果） @@ -49,8 +52,11 @@ func (r *Runner) Run(ip string) *Report { fmt.Printf("%s(%s) ", name, info.Version) } fmt.Println() + if len(r.installFailed) > 0 { + fmt.Printf(" [安装] 以下工具自动安装失败，相关测试将跳过: %v\n", r.installFailed) + } - // 显示将执行的指标 + // 显示将执行的指标及被跳过的原因 fmt.Printf(" 指标: ") for _, m := range r.metrics { if m.Enabled { @@ -58,6 +64,11 @@ func (r *Runner) Run(ip string) *Report { } } fmt.Println() + for _, m := range r.metrics { + if !m.Enabled && !m.IsMonitor { + fmt.Printf(" [跳过] %s: %s\n", m.Name, m.DisableReason) + } + } // 按需读取各测试脚本，拼装成完整脚本 script := r.assembleScript() @@ -76,20 +87,19 @@ func (r *Runner) Run(ip string) *Report { r.parseResults(report, output) } - // 生成采样折线图 - for _, res := range report.Results { - if len(res.Samples) > 0 { - chartPath, err := WriteChartHTML(ip, res, "reports") - if err != nil { - fmt.Printf(" [图表] 生成失败: %v\n", err) - } else { - fmt.Printf(" [图表] %s\n", chartPath) - } + report.EndTime = time.Now() + report.Duration = report.EndTime.Sub(report.StartTime) + + // 生成综合 HTML 报告（跑分卡片 + 曲线图） + if len(report.Results) > 0 { + reportPath, err := WriteReportHTML(ip, report, "reports") + if err != nil { + fmt.Printf(" [报告] 生成失败: %v\n", err) + } else { + fmt.Printf(" [报告] %s\n", reportPath) } } - report.EndTime = time.Now() - report.Duration = report.EndTime.Sub(report.StartTime) fmt.Printf("\n════════ [%s] 压力测试完成 ════════\n", ip) fmt.Println(report.ToText()) return report @@ -103,7 +113,11 @@ func (r *Runner) parseResults(report *Report, output string) { var currentDuration string var currentOutput []string var currentSamples []Sample + var currentScore float64 + var currentMetrics map[string]float64 + var sampleHeader []string inSamples := false + inSysInfo := false flush := func() { if currentTest == "" { @@ -118,6 +132,14 @@ func (r *Runner) parseResults(report *Report, output string) { case "error": status = StatusError } + // 温度超限判定（仅对 temp 监控） + if TestType(currentTest) == MonitorTemp && status == StatusPass && r.cfg.TempLimit > 0 { + if maxT, ok := currentMetrics["max_temp"]; ok && maxT > r.cfg.TempLimit { + status = StatusFail + currentOutput = append(currentOutput, + fmt.Sprintf("温度超限: 最高 %.1f°C > 阈值 %.1f°C", maxT, r.cfg.TempLimit)) + } + } duration, _ := time.ParseDuration(currentDuration) report.AddResult(Result{ Type: TestType(currentTest), @@ -125,12 +147,17 @@ func (r *Runner) parseResults(report *Report, output string) { Output: strings.Join(currentOutput, "\n"), Duration: duration, Samples: currentSamples, + Score: currentScore, + Metrics: currentMetrics, }) currentTest = "" currentStatus = "" currentDuration = "" currentOutput = nil currentSamples = nil + currentScore = 0 + currentMetrics = nil + sampleHeader = nil } for _, line := range lines { @@ -139,6 +166,7 @@ func (r *Runner) parseResults(report *Report, output string) { // 采样数据块 if line == "===SAMPLES===" { inSamples = true + sampleHeader = nil continue } if line == "===END_SAMPLES===" { @@ -147,14 +175,46 @@ func (r *Runner) parseResults(report *Report, output string) { } if inSamples { parts := strings.Split(line, ",") - if len(parts) >= 3 { - cpu, _ := strconv.ParseFloat(strings.TrimSpace(parts[1]), 64) - temp, _ := strconv.ParseFloat(strings.TrimSpace(parts[2]), 64) - currentSamples = append(currentSamples, Sample{ - Time: strings.TrimSpace(parts[0]), - CPU: cpu, - Temp: temp, - }) + if sampleHeader == nil { + // 首行为表头：time,指标1,指标2,... + for _, p := range parts { + sampleHeader = append(sampleHeader, strings.TrimSpace(p)) + } + continue + } + if len(parts) < 1 || strings.TrimSpace(parts[0]) == "" { + continue + } + s := Sample{Time: strings.TrimSpace(parts[0]), Values: map[string]float64{}} + for i := 1; i < len(parts) && i < len(sampleHeader); i++ { + name := sampleHeader[i] + if name == "" || name == "time" { + continue + } + if v, err := strconv.ParseFloat(strings.TrimSpace(parts[i]), 64); err == nil { + s.Values[name] = v + } + } + currentSamples = append(currentSamples, s) + continue + } + + if line == "===SYSTEM_INFO===" { + inSysInfo = true + continue + } + if inSysInfo { + if line == "" { + inSysInfo = false + continue + } + if i := strings.Index(line, ":"); i > 0 { + k := strings.TrimSpace(line[:i]) + v := strings.TrimSpace(line[i+1:]) + if report.SysInfo == nil { + report.SysInfo = map[string]string{} + } + report.SysInfo[k] = v } continue } @@ -187,12 +247,24 @@ func (r *Runner) parseResults(report *Report, output string) { currentDuration = fmt.Sprintf("%dms", ms) continue } - if strings.HasPrefix(line, "output:") { - currentOutput = append(currentOutput, strings.TrimPrefix(line, "output:")) + if strings.HasPrefix(line, "score:") { + currentScore, _ = strconv.ParseFloat(strings.TrimPrefix(line, "score:"), 64) continue } - if strings.HasPrefix(line, "samples:") || strings.HasPrefix(line, "max:") { - currentOutput = append(currentOutput, line) + if strings.HasPrefix(line, "metric:") { + kv := strings.TrimPrefix(line, "metric:") + if i := strings.Index(kv, "="); i > 0 { + name := strings.TrimSpace(kv[:i]) + val, _ := strconv.ParseFloat(strings.TrimSpace(kv[i+1:]), 64) + if currentMetrics == nil { + currentMetrics = map[string]float64{} + } + currentMetrics[name] = val + } + continue + } + if strings.HasPrefix(line, "output:") { + currentOutput = append(currentOutput, strings.TrimPrefix(line, "output:")) continue } @@ -202,13 +274,6 @@ func (r *Runner) parseResults(report *Report, output string) { } } flush() - - // 从系统信息输出解析主机信息 - for _, line := range lines { - if strings.HasPrefix(line, "hostname:") { - // 可扩展：存入 report 的系统信息字段 - } - } } // assembleScript 根据 metrics 读取对应的独立脚本文件，拼装成完整脚本 @@ -248,9 +313,24 @@ func (r *Runner) assembleScript() string { sb.WriteString("\n") } + // 按"先单项、后综合"排序测试脚本 + ordered := make([]Metric, len(r.metrics)) + copy(ordered, r.metrics) + sort.SliceStable(ordered, func(i, j int) bool { + return testPriority(ordered[i]) < testPriority(ordered[j]) + }) + // 逐个测试脚本 - for _, m := range r.metrics { - if m.IsMonitor || !m.Enabled { + for _, m := range ordered { + if m.IsMonitor { + continue + } + if !m.Enabled { + // 缺少依赖时仍输出一个 skip 结果，让报告完整展示原因 + sb.WriteString(fmt.Sprintf("echo '===TEST:%s==='\n", m.Name)) + sb.WriteString("echo 'status:skip'\n") + sb.WriteString(fmt.Sprintf("echo 'output:%s'\n", m.DisableReason)) + sb.WriteString("echo ''\n") continue } data := r.readScript(m.Name) @@ -286,6 +366,27 @@ func (r *Runner) readScript(name string) []byte { return data } +// testPriority 测试执行顺序：单项测试在前，综合测试(full)最后，监控项最后 +func testPriority(m Metric) int { + if m.IsMonitor { + return 1000 + } + switch TestType(m.Name) { + case TestCPU: + return 0 + case TestMemory: + return 1 + case TestDiskIO: + return 2 + case TestMemNative: + return 3 + case TestFull: + return 100 + default: + return 50 + } +} + // ============================ // 业务入口（workflow 调用） // ============================ @@ -293,6 +394,7 @@ func (r *Runner) readScript(name string) []byte { // TestDevice 对单台设备执行 SSH 登录 + 压力测试（业务入口） func TestDevice(ip string, cfg config.Config) *Report { sshCfg := sshclient.NewConfig(cfg.SSH.User, cfg.SSH.Password, cfg.SSH.KeyFile, cfg.SSH.Port, cfg.Scan.Timeout) + sshCfg.SudoPwd = cfg.SSH.SudoPwd conn, err := sshclient.Connect(ip, sshCfg) if err != nil { @@ -300,6 +402,19 @@ func TestDevice(ip string, cfg config.Config) *Report { } defer conn.Close() + // fnOS 的 admin 用户 HOME 目录可能不存在，登录会输出 + // "Could not chdir to home directory" 横幅污染命令输出，先建好 HOME + sshclient.EnsureHome(conn) + + // 根据配置决定是否在线安装时使用国内镜像源（默认 false，先试系统自带源） + SetMirror(cfg.Stress.UseMirror) + + // sudo 密码：优先用专属 sudo_password；为空时退回 SSH 登录密码（飞牛 admin 同密码） + sudoPwd := cfg.SSH.SudoPwd + if sudoPwd == "" { + sudoPwd = cfg.SSH.Password + } + // SSH 登录成功后回填远端 MAC（发现阶段只做了 ping，暂无 MAC） if mac := queryRemoteMAC(conn); mac != "" { if dev, ok := discovery.Devices[ip]; ok { @@ -314,8 +429,9 @@ func TestDevice(ip string, cfg config.Config) *Report { Threads: cfg.Stress.Threads, DiskSizeMB: 1024, TempLogInt: 10 * time.Second, + TempLimit: cfg.Stress.TempLimit, } - return NewRunner(stressCfg, ParseTypes(cfg.Stress.Types), conn).Run(ip) + return NewRunner(stressCfg, ParseTypes(cfg.Stress.Types), conn, cfg.Stress.AutoInstall, sudoPwd).Run(ip) } // NewSSHFailReport 构造 SSH 连接失败报告 @@ -365,6 +481,15 @@ func IsPassed(rpt *Report) bool { return rpt != nil && rpt.Failed == 0 && rpt.Errors == 0 } +// IsSSHFail 报告是否为 SSH 连接失败（连不上，无任何有效测试项） +func IsSSHFail(rpt *Report) bool { + if rpt == nil || len(rpt.Results) != 1 { + return false + } + r := rpt.Results[0] + return r.Type == "ssh" && (r.Status == StatusFail || r.Status == StatusError) +} + // Status 报告状态字符串 func Status(rpt *Report) string { if rpt == nil { diff --git a/auto-check/pkg/stress/tools.go b/auto-check/pkg/stress/tools.go index 39722d9..f6d56d5 100644 --- a/auto-check/pkg/stress/tools.go +++ b/auto-check/pkg/stress/tools.go @@ -1,6 +1,7 @@ package stress import ( + "fmt" "strings" "auto-check/pkg/sshclient" @@ -8,10 +9,82 @@ import ( "golang.org/x/crypto/ssh" ) +// ============================ +// 国内镜像源 +// ============================ + +// useMirror 控制是否把远端包管理器切到国内镜像。默认关闭——先用系统自带源 +// 试装（飞牛/群晖等定制系统常常已预置可用源），失败或超时再开启清华镜像。 +var useMirror = false + +// SetMirror 设置是否使用国内镜像源（由配置注入） +func SetMirror(on bool) { + useMirror = on +} + +// mirrorName 根据发行版返回镜像别名（用于日志） +const mirrorLabel = "清华 TUNA 镜像" + +// setupMirror 在安装前把对应包管理器的源切换为国内镜像（仅 apt 系需要改文件， +// 其它包管理器通过参数/变量临时指向镜像）。失败不致命，仅打印警告并继续用原源。 +func setupMirror(client *ssh.Client, pm string, sudoPwd string) { + if !useMirror { + return + } + switch pm { + case "apt-get": + // 备份并替换为清华镜像（Debian/Ubuntu 通用，按 lsb_release 选版本） + script := `set -e +OS_ID=$(. /etc/os-release 2>/dev/null; echo "$ID") +OS_VER=$(. /etc/os-release 2>/dev/null; echo "$VERSION_CODENAME") +[ -z "$OS_VER" ] && OS_VER=$(. /etc/os-release 2>/dev/null; echo "$VERSION_ID") +MIRROR="https://mirrors.tuna.tsinghua.edu.cn" +if [ "$OS_ID" = "ubuntu" ]; then + LIST="$MIRROR/ubuntu" +elif [ "$OS_ID" = "debian" ]; then + LIST="$MIRROR/debian" +elif [ "$OS_ID" = "raspbian" ]; then + LIST="$MIRROR/raspbian" +else + LIST="$MIRROR/debian" +fi +BAK=/etc/apt/sources.list.d/auto-check-mirror.bak +if [ ! -f "$BAK" ]; then + cp /etc/apt/sources.list "$BAK" 2>/dev/null || true + cp -r /etc/apt/sources.list.d "$BAK.d" 2>/dev/null || true +fi +: > /etc/apt/sources.list +echo "deb $LIST $OS_VER main contrib non-free" > /etc/apt/sources.list +echo "deb $LIST $OS_VER-updates main contrib non-free" >> /etc/apt/sources.list +echo "deb $LIST $OS_VER-backports main contrib non-free" >> /etc/apt/sources.list +echo "deb $LIST-security $OS_VER-security main contrib non-free" >> /etc/apt/sources.list +echo OK` + out, err := runPrivileged(client, "sh -c "+shellQuote(script), sudoPwd) + if err != nil || !strings.Contains(out, "OK") { + fmt.Printf(" [镜像] apt 切换国内源失败，将使用官方源: %v\n", err) + } else { + fmt.Printf(" [镜像] apt 已切换至%s\n", mirrorLabel) + } + case "dnf", "yum": + // 清华镜像站对 dnf/yum 提供 repo 文件；这里用变量临时指向（持久化需改 repo） + cmd := `echo "腾讯/清华镜像需手动配置 repo；尝试临时使用 mirrors.tuna.tsinghua.edu.cn 变量"` + runPrivileged(client, cmd, sudoPwd) + // 通过 baseurl 变量临时覆盖（仅对部分仓库有效，作为加速兜底） + fmt.Printf(" [镜像] %s 使用系统默认源（建议手动配置%s repo）\n", pm, mirrorLabel) + case "apk": + runPrivileged(client, "sed -i 's#https\\?://[^/]*alpinelinux.org#https://mirrors.tuna.tsinghua.edu.cn/alpine#g' /etc/apk/repositories", sudoPwd) + fmt.Printf(" [镜像] apk 已切换至%s\n", mirrorLabel) + } +} + // ============================ // 远程工具检测 // ============================ +// 压测核心依赖工具（缺则按需在线安装）。lm-sensors 依赖较多，仅单文件 +// 二进制 stress-ng 与 fio 为必需；sensors 缺失仅影响温度曲线，不阻断压测。 +var requiredTools = []string{"stress-ng", "fio"} + // ToolInfo 远程工具信息 type ToolInfo struct { Name string @@ -47,31 +120,185 @@ func DetectTools(client *ssh.Client) ToolSet { return ts } -// detectOne 检测单个工具 +// EnsureTools 检测工具；缺失且 autoInstall=true 时按发行版自动安装。 +// sudoPwd 为非 root 用户执行安装时提供（为空则尝试免密 sudo）。 +// 返回安装后的工具集合，以及最终仍缺失的工具名列表。 +func EnsureTools(client *ssh.Client, names []string, autoInstall bool, sudoPwd string) (ToolSet, []string) { + ts := DetectTools(client) + var missing []string + for _, n := range names { + if !ts.Has(n) { + missing = append(missing, n) + } + } + if len(missing) == 0 { + return ts, nil + } + + var failed []string + if autoInstall { + pm := detectPkgManager(client) + if pm == "" { + return ts, missing // 无法识别包管理器，跳过安装 + } + // 安装前先把源切到国内镜像（使用官方源在网络差的环境会超时） + setupMirror(client, pm, sudoPwd) + // 仅 apt 系需要先刷新元数据 + if pm == "apt-get" { + if out, err := runPrivileged(client, "timeout 180 apt-get update 2>&1", sudoPwd); err != nil || strings.Contains(out, "Err") || strings.Contains(out, "Failed") { + fmt.Printf(" [安装] apt-get update 异常(可能无可用源/无权限): %v\n%s\n", err, truncate(out, 400)) + } + } + for _, n := range missing { + pkg := pkgName(pm, n) + if pkg == "" { + failed = append(failed, n) + continue + } + installCmd := pkgInstallCmd(pm, pkg) + // runPrivileged 统一处理 sudo 与引号，这里直接给裸安装命令（带 timeout 兜底） + out, err := runPrivileged(client, fmt.Sprintf("timeout 300 %s 2>&1", installCmd), sudoPwd) + if err != nil || !strings.Contains(out, "Setting up "+pkg) && !strings.Contains(out, "已安装") && !strings.Contains(out, "already newest") { + fmt.Printf(" [安装] %s 安装失败: %v\n%s\n", pkg, err, truncate(out, 500)) + } + // 安装后重新检测该工具是否到位 + if info, ok := detectOne(client, n); ok { + ts.Tools[n] = info + } else { + failed = append(failed, n) + } + } + if len(failed) == 0 { + return ts, nil + } + return ts, failed + } + return ts, missing +} + +// runPrivileged 以特权执行命令：root 直接执行；非 root 用 sudo， +// 若提供了 sudo 密码则通过 stdin 喂入（无需 NOPASSWD 配置）。 +func runPrivileged(client *ssh.Client, cmd, sudoPwd string) (string, error) { + // 已是 root 则直接执行 + if out, err := sshclient.RunCommand(client, "id -u"); err == nil && strings.TrimSpace(out) == "0" { + return sshclient.RunCommand(client, cmd) + } + // 非 root：尝试 sudo + if sudoPwd != "" { + // echo 密码 | sudo -S 执行；整条命令用双引号包裹，避免命令内部单引号嵌套冲突。 + // 仅对命令内的双引号转义（安装命令均不含双引号，安全）。 + quoted := strings.ReplaceAll(singleLine(cmd), `"`, `\"`) + wrapped := fmt.Sprintf("echo %s | sudo -S -- sh -c \"%s\"", shellQuote(sudoPwd), quoted) + return sshclient.RunCommand(client, wrapped) + } + // 无密码：先尝试免密 sudo (-n)，失败则回退普通 sudo（会提示需要密码） + quoted := strings.ReplaceAll(singleLine(cmd), `"`, `\"`) + if out, err := sshclient.RunCommand(client, "sudo -n -- sh -c \""+quoted+"\""); err == nil { + return out, nil + } + return sshclient.RunCommand(client, "sudo -- sh -c \""+quoted+"\"") +} + +// shellQuote 转义单引号，避免密码破坏命令结构 +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + +// singleLine 去掉换行，保证能塞进 sh -c '...' +func singleLine(s string) string { + return strings.ReplaceAll(strings.ReplaceAll(s, "\n", " "), "\r", "") +} + +// truncate 截断过长的输出，便于日志展示 +func truncate(s string, n int) string { + // 去掉 ANSI/多余空行，只保留前 n 个字符 + s = strings.TrimSpace(s) + if len([]rune(s)) > n { + return string([]rune(s)[:n]) + "...(truncated)" + } + return s +} + +// detectPkgManager 识别远程发行版的包管理器 +func detectPkgManager(client *ssh.Client) string { + for _, pm := range []string{"apt-get", "dnf", "yum", "apk", "zypper"} { + if _, err := sshclient.RunCommand(client, fmt.Sprintf("command -v %s", pm)); err == nil { + return pm + } + } + return "" +} + +// pkgName 工具名 → 对应发行版的包名 +func pkgName(pm, tool string) string { + m := map[string]string{ + "stress-ng": "stress-ng", + "fio": "fio", + "lm-sensors": "lm-sensors", + "stressapptest": "stressapptest", + "iperf3": "iperf3", + } + if pm == "dnf" || pm == "yum" { + // RHEL 系 lm-sensors 包名为 lm_sensors + if tool == "lm-sensors" { + return "lm_sensors" + } + } + if pm == "zypper" { + if tool == "lm-sensors" { + return "sensors" + } + } + if v, ok := m[tool]; ok { + return v + } + return tool +} + +// pkgInstallCmd 生成安装命令 +func pkgInstallCmd(pm, pkg string) string { + switch pm { + case "apt-get": + return fmt.Sprintf("apt-get install -y %s", pkg) + case "dnf": + return fmt.Sprintf("dnf install -y %s", pkg) + case "yum": + return fmt.Sprintf("yum install -y %s", pkg) + case "apk": + return fmt.Sprintf("apk add %s", pkg) + case "zypper": + return fmt.Sprintf("zypper install -y %s", pkg) + } + return "" +} + +// detectOne 检测单个工具（查系统 PATH；缺失由 EnsureTools 在线安装补全） func detectOne(client *ssh.Client, name string) (ToolInfo, bool) { - out, err := sshclient.RunCommand(client, "which "+name+" 2>/dev/null") - path := strings.TrimSpace(out) - if err != nil || path == "" { + path := "" + if out, err := sshclient.RunCommand(client, "which "+name+" 2>/dev/null"); err == nil { + path = strings.TrimSpace(out) + } + if path == "" { return ToolInfo{}, false } var version string switch name { case "stress-ng": - out, _ = sshclient.RunCommand(client, path+" --version 2>&1 | head -1") - version = strings.TrimSpace(out) + v, _ := sshclient.RunCommand(client, path+" --version 2>&1 | head -1") + version = strings.TrimSpace(v) case "stressapptest": - out, _ = sshclient.RunCommand(client, path+" --help 2>&1 | head -1") - version = strings.TrimSpace(out) + v, _ := sshclient.RunCommand(client, path+" --help 2>&1 | head -1") + version = strings.TrimSpace(v) case "iperf3": - out, _ = sshclient.RunCommand(client, path+" --version 2>&1 | head -1") - version = strings.TrimSpace(out) + v, _ := sshclient.RunCommand(client, path+" --version 2>&1 | head -1") + version = strings.TrimSpace(v) case "fio": - out, _ = sshclient.RunCommand(client, path+" --version 2>&1 | head -1") - version = strings.TrimSpace(out) + v, _ := sshclient.RunCommand(client, path+" --version 2>&1 | head -1") + version = strings.TrimSpace(v) case "lm-sensors": - out, _ = sshclient.RunCommand(client, path+" -v 2>&1 | head -1") - version = strings.TrimSpace(out) + v, _ := sshclient.RunCommand(client, path+" -v 2>&1 | head -1") + version = strings.TrimSpace(v) } return ToolInfo{Name: name, Path: path, Version: version}, true diff --git a/auto-check/pkg/stress/types.go b/auto-check/pkg/stress/types.go index 79ffc06..addc246 100644 --- a/auto-check/pkg/stress/types.go +++ b/auto-check/pkg/stress/types.go @@ -35,14 +35,128 @@ type Result struct { Output string Error string Duration time.Duration - Samples []Sample // 时序采样数据（cpu 等测试采集） + Samples []Sample // 时序采样数据（用于绘制曲线） + Score float64 // 性能跑分（越大越好，单位见 MetricUnit） + Metrics map[string]float64 // 关键指标（最高温度、IOPS、吞吐等明细） } -// Sample 单次采样点 +// Sample 单次采样点（通用多指标：时间 + 各指标值） +// 指标名由脚本 SAMPLES 块首行表头决定，例如 cpu/temp/mem_used_pct/iops_total/bw_total type Sample struct { - Time string // HH:MM:SS - CPU float64 // CPU 使用率 % - Temp float64 // 温度 °C + Time string // 时间（HH:MM:SS 或相对秒） + Values map[string]float64 // 指标名 → 值 +} + +// Get 读取某个指标在该采样点的值（不存在返回 0） +func (s Sample) Get(name string) float64 { + return s.Values[name] +} + +// HasMetric 采样点是否包含某指标 +func (s Sample) HasMetric(name string) bool { + _, ok := s.Values[name] + return ok +} + +// ScoreUnit 返回某类测试跑分的单位 +func ScoreUnit(t TestType) string { + switch t { + case TestDiskIO: + return "MB/s" + default: + return "ops/s" + } +} + +// metricLabels 关键指标中文名映射 +var metricLabels = map[string]string{ + "max_temp": "最高温度", + "avg_temp": "平均温度", + "avg_cpu": "平均CPU利用率", + "mem_used_pct": "峰值内存使用率", + "avg_mem": "平均内存使用率", + "read_iops": "读IOPS", + "write_iops": "写IOPS", + "iops_total": "总IOPS", + "read_bw": "读吞吐", + "write_bw": "写吞吐", + "bw_total": "总吞吐", +} + +// MetricLabel 返回指标显示名，未知指标原样返回 +func MetricLabel(name string) string { + if l, ok := metricLabels[name]; ok { + return l + } + return name +} + +// MetricUnit 返回指标单位，未知指标返回空 +func MetricUnit(name string) string { + switch name { + case "max_temp", "avg_temp": + return "°C" + case "read_bw", "write_bw", "bw_total": + return "MB/s" + case "avg_cpu", "mem_used_pct", "avg_mem": + return "%" + default: + return "" + } +} + +// sortedMetricNames 按固定顺序返回指标名（保证报告输出稳定） +func sortedMetricNames(m map[string]float64) []string { + if len(m) == 0 { + return nil + } + order := []string{"max_temp", "avg_temp", "avg_cpu", "mem_used_pct", "avg_mem", + "read_iops", "write_iops", "iops_total", "read_bw", "write_bw", "bw_total"} + names := make([]string, 0, len(m)) + for _, k := range order { + if _, ok := m[k]; ok { + names = append(names, k) + } + } + // 兜底：未在 order 里的键追加 + for k := range m { + found := false + for _, n := range names { + if n == k { + found = true + break + } + } + if !found { + names = append(names, k) + } + } + return names +} + +// MetricItem 一条指标明细（已格式化，供报告/打印使用） +type MetricItem struct { + Name string `json:"name" yaml:"name"` + Label string `json:"label" yaml:"label"` + Value float64 `json:"value" yaml:"value"` + Unit string `json:"unit,omitempty" yaml:"unit,omitempty"` +} + +// OrderedMetrics 返回按固定顺序排列的指标明细列表 +func (r Result) OrderedMetrics() []MetricItem { + if len(r.Metrics) == 0 { + return nil + } + items := make([]MetricItem, 0, len(r.Metrics)) + for _, name := range sortedMetricNames(r.Metrics) { + items = append(items, MetricItem{ + Name: name, + Label: MetricLabel(name), + Value: r.Metrics[name], + Unit: MetricUnit(name), + }) + } + return items } // Report 完整压测报告 @@ -52,6 +166,7 @@ type Report struct { EndTime time.Time Duration time.Duration Results []Result + SysInfo map[string]string // 系统信息（hostname/cpu/cores/memory/kernel/disk） Passed int Failed int Skipped int @@ -96,6 +211,13 @@ func (r *Report) ToText() string { icon = "⚠" } sb.WriteString(fmt.Sprintf(" %s %-12s %s\n", icon, res.Type, res.Status)) + if res.Score > 0 { + sb.WriteString(fmt.Sprintf(" 跑分: %.1f %s\n", res.Score, ScoreUnit(res.Type))) + } + for _, name := range sortedMetricNames(res.Metrics) { + unit := MetricUnit(name) + sb.WriteString(fmt.Sprintf(" %s: %.1f%s\n", MetricLabel(name), res.Metrics[name], unit)) + } for _, line := range strings.Split(res.Output, "\n") { if strings.TrimSpace(line) != "" { sb.WriteString(fmt.Sprintf(" %s\n", line)) diff --git a/auto-check/pkg/workflow/workflow.go b/auto-check/pkg/workflow/workflow.go index af63604..374d6ab 100644 --- a/auto-check/pkg/workflow/workflow.go +++ b/auto-check/pkg/workflow/workflow.go @@ -71,6 +71,10 @@ func (w *Workflow) round() { fmt.Printf(" [测试] %s (%s) 开始压测...\n", ip, dev.MAC) rpt := stress.TestDevice(ip, w.cfg) stress.Results[ip] = rpt + if stress.IsSSHFail(rpt) { + fmt.Printf(" [测试] %s SSH 连接失败，跳过报告输出: %s\n", ip, rpt.Results[0].Error) + continue + } report.SaveAndPrintDeviceReport(&dev, rpt, cfg.Report.Path, cfg.Report.Print) } diff --git a/auto-check/run.err b/auto-check/run.err new file mode 100644 index 0000000..e69de29 diff --git a/auto-check/run.log b/auto-check/run.log new file mode 100644 index 0000000..fc52167 --- /dev/null +++ b/auto-check/run.log @@ -0,0 +1,142 @@ + [配置] 加载完成: 网段=10.0.3.0/24 间隔=3m0s 压测=cpu,memory,disk,full +══ 工作流启动（间隔 3m0s，设备以 IP 为 key）══ + +════ 轮次开始 [13:42:10] ════ +[扫描] 网段 10.0.3.0/24，共 254 个IP，10 并发 ping 探测... +[扫描] 完成，发现 11 台存活设备 + [扫描] 设备 11 台 + [测试] 10.0.3.1 () 开始压测... + [测试] 10.0.3.1 SSH 连接失败，跳过报告输出: SSH 连接 10.0.3.1:22 失败: dial tcp 10.0.3.1:22: connectex: No connection could be made because the target machine actively refused it. + [测试] 10.0.3.104 () 开始压测... + [测试] 10.0.3.104 SSH 连接失败，跳过报告输出: SSH 连接 10.0.3.104:22 失败: dial tcp 10.0.3.104:22: connectex: No connection could be made because the target machine actively refused it. + [测试] 10.0.3.123 () 开始压测... + [测试] 10.0.3.123 SSH 连接失败，跳过报告输出: SSH 连接 10.0.3.123:22 失败: dial tcp 10.0.3.123:22: connectex: No connection could be made because the target machine actively refused it. + [测试] 10.0.3.152 () 开始压测... + [测试] 10.0.3.152 SSH 连接失败，跳过报告输出: SSH 连接 10.0.3.152:22 失败: dial tcp 10.0.3.152:22: connectex: No connection could be made because the target machine actively refused it. + [测试] 10.0.3.147 () 开始压测... + [测试] 10.0.3.147 SSH 连接失败，跳过报告输出: SSH 连接 10.0.3.147:22 失败: dial tcp 10.0.3.147:22: connectex: No connection could be made because the target machine actively refused it. + [测试] 10.0.3.148 () 开始压测... + [MAC] 10.0.3.148 -> fe:49:6a:ab:e1:79 + +════════ [10.0.3.148] 压力测试开始 ════════ + 工具: stress-ng(stress-ng, version 0.15.06 (gcc 12.2.0, x86_64 Linux 6.18.18.c877-trim)) iperf3(iperf 3.12 (cJSON 1.7.15)) fio(fio-3.33) + 指标: cpu memory disk full dmesg + [脚本] 拼装完成，10334 字节 + [报告] reports/report-10.0.3.148.html + +════════ [10.0.3.148] 压力测试完成 ════════ +══ [10.0.3.148] 压力测试报告 ══ + 耗时: 2m18.501s + + ✓ cpu pass + 跑分: 1285.6 ops/s + 最高温度: 57.0°C + 平均CPU利用率: 99.1% + stress-ng: info: [3652909] Working directory / is not read/writeable, some I/O tests may fail + stress-ng: info: [3652909] setting to a 30 second run per stressor + stress-ng: info: [3652909] dispatching hogs: 4 cpu + stress-ng: metrc: [3652909] stressor bogo ops real time usr time sys time bogo ops/s bogo ops/s + stress-ng: metrc: [3652909] (secs) (secs) (secs) (real time) (usr+sys time) + stress-ng: metrc: [3652909] cpu 38577 30.01 90.66 0.43 1285.60 423.51 + stress-ng: info: [3652909] successful run completed in 30.02s + ✓ memory pass + 跑分: 33472.4 ops/s + 最高温度: 58.0°C + 峰值内存使用率: 57.1% + stress-ng: info: [3653312] Working directory / is not read/writeable, some I/O tests may fail + stress-ng: info: [3653312] setting to a 30 second run per stressor + stress-ng: info: [3653312] dispatching hogs: 4 vm + stress-ng: metrc: [3653312] stressor bogo ops real time usr time sys time bogo ops/s bogo ops/s + stress-ng: metrc: [3653312] (secs) (secs) (secs) (real time) (usr+sys time) + stress-ng: metrc: [3653312] vm 1006646 30.07 47.26 5.75 33472.37 18990.03 + stress-ng: info: [3653312] successful run completed in 30.28s + ✓ disk pass + 跑分: 445.0 MB/s + 读IOPS: 77.0 + 写IOPS: 34.0 + 总IOPS: 111.0 + 读吞吐: 309.0MB/s + 写吞吐: 136.0MB/s + 总吞吐: 445.0MB/s + autocheck: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=psync, iodepth=32 + fio-3.33 + Starting 1 process + autocheck: Laying out IO file (1 file / 1024MiB) + note: both iodepth >= 1 and synchronous I/O engine are selected, queue depth will be capped at 1 + autocheck: (groupid=0, jobs=1): err= 0: pid=3653771: Thu Aug 13 13:44:37 2026 + read: IOPS=77, BW=309KiB/s (317kB/s)(9276KiB/30005msec) + clat (usec): min=253, max=830522, avg=11740.11, stdev=28335.80 + lat (usec): min=255, max=830525, avg=11742.04, stdev=28335.81 + clat percentiles (msec): + | 1.00th=[ 4], 5.00th=[ 5], 10.00th=[ 6], 20.00th=[ 7], + | 30.00th=[ 8], 40.00th=[ 9], 50.00th=[ 11], 60.00th=[ 12], + | 70.00th=[ 13], 80.00th=[ 14], 90.00th=[ 15], 95.00th=[ 16], + | 99.00th=[ 59], 99.50th=[ 65], 99.90th=[ 676], 99.95th=[ 802], + | 99.99th=[ 835] + bw ( KiB/s): min= 28, max= 420, per=98.98%, avg=306.62, stdev=113.30, samples=29 + iops : min= 7, max= 105, avg=76.55, stdev=28.40, samples=29 + write: IOPS=34, BW=136KiB/s (140kB/s)(4088KiB/30005msec); 0 zone resets + clat (usec): min=131, max=841470, avg=2638.71, stdev=36403.27 + lat (usec): min=131, max=841473, avg=2641.68, stdev=36403.26 + clat percentiles (usec): + | 1.00th=[ 188], 5.00th=[ 269], 10.00th=[ 318], 20.00th=[ 383], + | 30.00th=[ 449], 40.00th=[ 506], 50.00th=[ 586], 60.00th=[ 685], + | 70.00th=[ 824], 80.00th=[ 979], 90.00th=[ 1254], 95.00th=[ 1483], + | 99.00th=[ 9372], 99.50th=[ 15270], 99.90th=[767558], 99.95th=[843056], + | 99.99th=[843056] + bw ( KiB/s): min= 8, max= 211, per=99.09%, avg=135.72, stdev=55.82, samples=29 + iops : min= 2, max= 52, avg=33.83, stdev=14.03, samples=29 + lat (usec) : 250=1.11%, 500=10.72%, 750=8.20%, 1000=5.00% + lat (msec) : 2=4.88%, 4=1.65%, 10=33.49%, 20=33.10%, 50=0.57% + lat (msec) : 100=1.11%, 250=0.03%, 750=0.03%, 1000=0.12% + cpu : usr=0.46%, sys=2.25%, ctx=3357, majf=0, minf=13 + IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% + submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% + complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% + issued rwts: total=2319,1022,0,0 short=0,0,0,0 dropped=0,0,0,0 + latency : target=0, window=0, percentile=100.00%, depth=32 + Run status group 0 (all jobs): + READ: bw=309KiB/s (317kB/s), 309KiB/s-309KiB/s (317kB/s-317kB/s), io=9276KiB (9499kB), run=30005-30005msec + WRITE: bw=136KiB/s (140kB/s), 136KiB/s-136KiB/s (140kB/s-140kB/s), io=4088KiB (4186kB), run=30005-30005msec + Disk stats (read/write): + sda: ios=2319/1057, merge=0/81, ticks=26764/7589, in_queue=38611, util=97.50% + ✓ full pass + 跑分: 40708.6 ops/s + 最高温度: 58.0°C + 平均CPU利用率: 95.8% + 峰值内存使用率: 51.2% + stress-ng: info: [3653901] Working directory / is not read/writeable, some I/O tests may fail + stress-ng: info: [3653901] setting to a 30 second run per stressor + stress-ng: info: [3653901] dispatching hogs: 4 cpu, 4 vm, 2 iomix + stress-ng: metrc: [3653901] stressor bogo ops real time usr time sys time bogo ops/s bogo ops/s + stress-ng: metrc: [3653901] (secs) (secs) (secs) (real time) (usr+sys time) + stress-ng: metrc: [3653901] cpu 22552 30.02 54.07 0.35 751.21 414.42 + stress-ng: metrc: [3653901] vm 1193352 30.01 45.94 3.66 39767.94 24057.59 + stress-ng: metrc: [3653901] iomix 5692 30.04 2.13 4.02 189.48 924.72 + stress-ng: info: [3653901] successful run completed in 30.08s + ✓ dmesg pass + 无硬件相关内核报错 + + 共 5 项: 5 通过, 0 失败, 0 跳过, 0 错误 + + [测试] 10.0.3.162 () 开始压测... + [测试] 10.0.3.162 SSH 连接失败，跳过报告输出: SSH 连接 10.0.3.162:22 失败: dial tcp 10.0.3.162:22: connectex: No connection could be made because the target machine actively refused it. + [测试] 10.0.3.119 () 开始压测... + [测试] 10.0.3.119 SSH 连接失败，跳过报告输出: SSH 连接 10.0.3.119:22 失败: dial tcp 10.0.3.119:22: connectex: No connection could be made because the target machine actively refused it. + [测试] 10.0.3.160 () 开始压测... + [测试] 10.0.3.160 SSH 连接失败，跳过报告输出: SSH 连接 10.0.3.160:22 失败: dial tcp 10.0.3.160:22: connectex: No connection could be made because the target machine actively refused it. + [测试] 10.0.3.171 () 开始压测... + [测试] 10.0.3.171 SSH 连接失败，跳过报告输出: SSH 连接 10.0.3.171:22 失败: dial tcp 10.0.3.171:22: connectex: No connection could be made because the target machine actively refused it. + [测试] 10.0.3.236 () 开始压测... + [测试] 10.0.3.236 SSH 连接失败，跳过报告输出: SSH 连接 10.0.3.236:22 失败: dial tcp 10.0.3.236:22: connectex: No connection could be made because the target machine actively refused it. + [状态] 10.0.3.160 ✗ → fail + [状态] 10.0.3.236 ✗ → fail + [状态] 10.0.3.1 ✗ → fail + [状态] 10.0.3.104 ✗ → fail + [状态] 10.0.3.147 ✗ → fail + [状态] 10.0.3.119 ✗ → fail + [状态] 10.0.3.171 ✗ → fail + [状态] 10.0.3.123 ✗ → fail + [状态] 10.0.3.152 ✗ → fail + [状态] 10.0.3.148 (fe:49:6a:ab:e1:79) ✓ → pass + [状态] 10.0.3.162 ✗ → fail diff --git a/auto-check/scripts/cpu.sh b/auto-check/scripts/cpu.sh index 00acf23..f7b57e6 100644 --- a/auto-check/scripts/cpu.sh +++ b/auto-check/scripts/cpu.sh @@ -1,13 +1,18 @@ echo "===TEST:cpu===" +# 读取温度（sensors 不可用时输出空） +read_temp() { + sensors 2>/dev/null | grep -oP '\+?\d+(\.\d+)?(?=°C)' | head -1 +} + # 后台采样 CPU 使用率 + 温度 SAMPLE_LOG=/tmp/auto-check-cpu-sample.log -> "$SAMPLE_LOG" +: > "$SAMPLE_LOG" ( while true; do TS=$(date +%H:%M:%S) - CPU=$(top -bn1 2>/dev/null | grep "Cpu(s)" | sed 's/.*,\s*\([0-9.]*\)%*\s*id.*/\1/' | awk '{printf "%.1f", 100-$1}' || echo 0) - TEMP=$(sensors 2>/dev/null | grep -iE 'Package|Core\s*0|temp1|CPU' | head -1 | grep -oP '\d+\.?\d*' | head -1 || echo 0) + CPU=$(top -bn1 2>/dev/null | grep "Cpu(s)" | sed -n 's/.*,\s*\([0-9.]*\)%*\s*id.*/\1/p' | awk '{printf "%.1f", 100-$1}' 2>/dev/null) + TEMP=$(read_temp) echo "${TS},${CPU},${TEMP}" >> "$SAMPLE_LOG" sleep "${AUTOCHECK_SAMPLE_INTERVAL:-2}" done @@ -16,7 +21,7 @@ SAMPLE_PID=$! START_TIME=$(date +%s%N) set +e -OUTPUT=$(stress-ng --cpu "${AUTOCHECK_THREADS:-4}" --cpu-method all --timeout "${AUTOCHECK_DURATION:-30}s" --metrics-brief --temp-path /tmp 2>&1) +OUTPUT=$(stress-ng --cpu "${AUTOCHECK_THREADS:-4}" --cpu-method all --timeout "${AUTOCHECK_DURATION:-30}s" --temp-path /tmp --metrics-brief 2>&1) EXIT_CODE=$? set -e END_TIME=$(date +%s%N) @@ -27,10 +32,27 @@ wait $SAMPLE_PID 2>/dev/null || true DURATION_MS=$(( (END_TIME - START_TIME) / 1000000 )) if [ "$EXIT_CODE" -eq 0 ]; then echo 'status:pass'; else echo 'status:fail'; fi echo "duration:${DURATION_MS}ms" + +# 跑分：各 stressor 的 bogo ops/s 之和 +SCORE=$(echo "$OUTPUT" | awk ' +/^stress-ng:/ && ($4=="cpu" || $4=="vm" || $4=="iomix" || $4=="memrate") { + v=$(NF-1)+0; if (v>0) s+=v +} +END { printf "%.1f", s+0 }') +echo "score:${SCORE}" + +# 关键指标 +AVG_CPU=$(awk -F',' '$2!="" {s+=$2; n++} END{printf "%.1f", (n?s/n:0)}' "$SAMPLE_LOG") +echo "metric:avg_cpu=${AVG_CPU}" +MAX_TEMP=$(awk -F',' '$3!="" && $3+0>m {m=$3} END{printf "%.1f", m+0}' "$SAMPLE_LOG") +HAS_TEMP=$(awk -F',' '$3!="" && $3+0>0{f=1} END{print f+0}' "$SAMPLE_LOG") +if [ "$HAS_TEMP" = "1" ]; then echo "metric:max_temp=${MAX_TEMP}"; fi + echo "output:${OUTPUT}" -# 输出采样数据 +# 采样数据（首行表头） echo "===SAMPLES===" +echo "time,cpu,temp" cat "$SAMPLE_LOG" echo "===END_SAMPLES===" diff --git a/auto-check/scripts/disk.sh b/auto-check/scripts/disk.sh index 472ee6b..a07239f 100644 --- a/auto-check/scripts/disk.sh +++ b/auto-check/scripts/disk.sh @@ -1,14 +1,77 @@ echo "===TEST:disk===" -TEST_DIR=${AUTOCHECK_DISK_DIR:-/tmp/stress-disk-test} -DISK_SIZE_MB=${AUTOCHECK_DISK_SIZE_MB:-1024} + +TEST_DIR=${AUTOCHECK_DISK_DIR:-/tmp/auto-check-disk} +mkdir -p "$TEST_DIR" + START_TIME=$(date +%s%N) -set +e -OUTPUT=$(mkdir -p "${TEST_DIR}" && stress-ng --iomix 2 --iomix-bytes "${DISK_SIZE_MB}M" --timeout "${AUTOCHECK_DURATION:-30}s" --metrics-brief 2>&1; rm -rf "${TEST_DIR}") -EXIT_CODE=$? -set -e -END_TIME=$(date +%s%N) -DURATION_MS=$(( (END_TIME - START_TIME) / 1000000 )) -if [ "$EXIT_CODE" -eq 0 ]; then echo 'status:pass'; else echo 'status:fail'; fi -echo "duration:${DURATION_MS}ms" -echo "output:${OUTPUT}" + +if command -v fio >/dev/null 2>&1; then + FIO_IMG="$TEST_DIR/fio.img" + BW_LOG="$TEST_DIR/ac-bw" + IOPS_LOG="$TEST_DIR/ac-iops" + rm -f "$BW_LOG"*.log "$IOPS_LOG"*.log "$FIO_IMG" + + set +e + OUTPUT=$(fio --name=autocheck --rw=randrw --rwmixread=70 --bs=4k \ + --size="${AUTOCHECK_DISK_SIZE_MB:-512}M" --iodepth=32 --numjobs=1 \ + --runtime="${AUTOCHECK_DURATION:-30}" --time_based --direct=1 \ + --group_reporting --output-format=normal \ + --write_bw_log="$BW_LOG" --write_iops_log="$IOPS_LOG" \ + --log_avg_msec=1000 --filename="$FIO_IMG" 2>&1) + EXIT_CODE=$? + set -e + + READ_LINE=$(echo "$OUTPUT" | grep -E '^[[:space:]]*read:' | head -1) + WRITE_LINE=$(echo "$OUTPUT" | grep -E '^[[:space:]]*write:' | head -1) + READ_IOPS=$(echo "$READ_LINE" | grep -oP 'IOPS=\K[0-9.]+' | head -1) + READ_BW=$(echo "$READ_LINE" | grep -oP 'BW=\K[0-9.]+' | head -1) + WRITE_IOPS=$(echo "$WRITE_LINE" | grep -oP 'IOPS=\K[0-9.]+' | head -1) + WRITE_BW=$(echo "$WRITE_LINE" | grep -oP 'BW=\K[0-9.]+' | head -1) + SCORE=$(awk -v r="${READ_BW:-0}" -v w="${WRITE_BW:-0}" 'BEGIN{printf "%.1f", r+w}') + IOPS_TOTAL=$(awk -v r="${READ_IOPS:-0}" -v w="${WRITE_IOPS:-0}" 'BEGIN{printf "%.1f", r+w}') + + END_TIME=$(date +%s%N) + DURATION_MS=$(( (END_TIME - START_TIME) / 1000000 )) + if [ "$EXIT_CODE" -eq 0 ]; then echo 'status:pass'; else echo 'status:fail'; fi + echo "duration:${DURATION_MS}ms" + echo "score:${SCORE}" + echo "metric:iops_total=${IOPS_TOTAL}" + [ -n "$READ_IOPS" ] && echo "metric:read_iops=${READ_IOPS}" + [ -n "$WRITE_IOPS" ] && echo "metric:write_iops=${WRITE_IOPS}" + [ -n "$READ_BW" ] && echo "metric:read_bw=${READ_BW}" + [ -n "$WRITE_BW" ] && echo "metric:write_bw=${WRITE_BW}" + [ -n "$READ_BW" ] && [ -n "$WRITE_BW" ] && echo "metric:bw_total=${SCORE}" + echo "output:${OUTPUT}" + + # IOPS / 吞吐 曲线（按秒聚合） + BW_LOG_FILE=$(ls "$BW_LOG"*.log 2>/dev/null | head -1) + IOPS_LOG_FILE=$(ls "$IOPS_LOG"*.log 2>/dev/null | head -1) + echo "===SAMPLES===" + echo "time,iops_total,bw_total" + if [ -n "$BW_LOG_FILE" ] && [ -n "$IOPS_LOG_FILE" ]; then + awk -F',' ' + FNR==NR { t=int($1/1000); if($3==0) ri[t]=$2; else wi[t]=$2; seen[t]=1; next } + { t=int($1/1000); if($3==0) rb[t]=$2; else wb[t]=$2; seen[t]=1 } + END { for (k in seen) printf "%d,%.0f,%.1f\n", k, ri[k]+wi[k], (rb[k]+wb[k])/1024 } + ' "$IOPS_LOG_FILE" "$BW_LOG_FILE" | sort -n + fi + echo "===END_SAMPLES===" + + rm -f "$FIO_IMG" "$BW_LOG"*.log "$IOPS_LOG"*.log +else + # 无 fio，降级为 stress-ng 磁盘 IO 压测（无 IOPS/吞吐曲线） + set +e + OUTPUT=$(stress-ng --iomix "${AUTOCHECK_THREADS:-2}" --iomix-bytes "${AUTOCHECK_DISK_MB:-512}M" --timeout "${AUTOCHECK_DURATION:-30}s" --temp-path /tmp --metrics-brief 2>&1) + EXIT_CODE=$? + set -e + SCORE=$(echo "$OUTPUT" | awk '/^stress-ng:/ && $4=="iomix" { v=$(NF-1)+0; if(v>0) s+=v } END{printf "%.1f", s+0}') + + END_TIME=$(date +%s%N) + DURATION_MS=$(( (END_TIME - START_TIME) / 1000000 )) + if [ "$EXIT_CODE" -eq 0 ]; then echo 'status:pass'; else echo 'status:fail'; fi + echo "duration:${DURATION_MS}ms" + echo "score:${SCORE}" + echo "output:${OUTPUT}" +fi + echo '' diff --git a/auto-check/scripts/full.sh b/auto-check/scripts/full.sh index d33a950..4aeb54b 100644 --- a/auto-check/scripts/full.sh +++ b/auto-check/scripts/full.sh @@ -1,12 +1,71 @@ echo "===TEST:full===" + +# 读取温度（sensors 不可用时输出空） +read_temp() { + sensors 2>/dev/null | grep -oP '\+?\d+(\.\d+)?(?=°C)' | head -1 +} + +# 后台采样 CPU + 内存 + 温度 +SAMPLE_LOG=/tmp/auto-check-full-sample.log +: > "$SAMPLE_LOG" +( + while true; do + TS=$(date +%H:%M:%S) + CPU=$(top -bn1 2>/dev/null | grep "Cpu(s)" | sed -n 's/.*,\s*\([0-9.]*\)%*\s*id.*/\1/p' | awk '{printf "%.1f", 100-$1}' 2>/dev/null) + MEM_TOTAL=$(awk '/^MemTotal:/{print $2}' /proc/meminfo) + MEM_AVAIL=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo) + if [ -z "$MEM_AVAIL" ] || [ "$MEM_AVAIL" = "0" ]; then + MEM_AVAIL=$(awk '/^MemFree:/{print $2}' /proc/meminfo) + fi + MEM_PCT=$(awk -v t="$MEM_TOTAL" -v a="$MEM_AVAIL" 'BEGIN{if(t>0) printf "%.1f", (t-a)/t*100}') + TEMP=$(read_temp) + echo "${TS},${CPU},${MEM_PCT},${TEMP}" >> "$SAMPLE_LOG" + sleep "${AUTOCHECK_SAMPLE_INTERVAL:-2}" + done +) & +SAMPLE_PID=$! + START_TIME=$(date +%s%N) set +e -OUTPUT=$(stress-ng --cpu "${AUTOCHECK_THREADS:-4}" --vm 2 --vm-bytes 128M --iomix 1 --iomix-bytes 256M --timeout "${AUTOCHECK_DURATION:-30}s" --metrics-brief 2>&1) +OUTPUT=$(stress-ng \ + --cpu "${AUTOCHECK_THREADS:-4}" --cpu-method all \ + --vm "${AUTOCHECK_THREADS:-4}" --vm-bytes "${AUTOCHECK_MEM_MB:-256}M" --vm-method all \ + --iomix 2 --iomix-bytes "${AUTOCHECK_DISK_SIZE_MB:-512}M" \ + --timeout "${AUTOCHECK_DURATION:-30}s" --temp-path /tmp --metrics-brief 2>&1) EXIT_CODE=$? set -e END_TIME=$(date +%s%N) + +kill $SAMPLE_PID 2>/dev/null || true +wait $SAMPLE_PID 2>/dev/null || true + DURATION_MS=$(( (END_TIME - START_TIME) / 1000000 )) if [ "$EXIT_CODE" -eq 0 ]; then echo 'status:pass'; else echo 'status:fail'; fi echo "duration:${DURATION_MS}ms" + +# 跑分：各 stressor 的 bogo ops/s 之和 +SCORE=$(echo "$OUTPUT" | awk ' +/^stress-ng:/ && ($4=="cpu" || $4=="vm" || $4=="iomix" || $4=="memrate") { + v=$(NF-1)+0; if (v>0) s+=v +} +END { printf "%.1f", s+0 }') +echo "score:${SCORE}" + +# 关键指标 +AVG_CPU=$(awk -F',' '$2!="" {s+=$2; n++} END{printf "%.1f", (n?s/n:0)}' "$SAMPLE_LOG") +echo "metric:avg_cpu=${AVG_CPU}" +MAX_MEM=$(awk -F',' '$3!="" && $3+0>m {m=$3} END{printf "%.1f", m+0}' "$SAMPLE_LOG") +echo "metric:mem_used_pct=${MAX_MEM}" +MAX_TEMP=$(awk -F',' '$4!="" && $4+0>m {m=$4} END{printf "%.1f", m+0}' "$SAMPLE_LOG") +HAS_TEMP=$(awk -F',' '$4!="" && $4+0>0{f=1} END{print f+0}' "$SAMPLE_LOG") +if [ "$HAS_TEMP" = "1" ]; then echo "metric:max_temp=${MAX_TEMP}"; fi + echo "output:${OUTPUT}" + +# 采样数据（首行表头） +echo "===SAMPLES===" +echo "time,cpu,mem_used_pct,temp" +cat "$SAMPLE_LOG" +echo "===END_SAMPLES===" + echo '' diff --git a/auto-check/scripts/memory.sh b/auto-check/scripts/memory.sh index bd3db6a..479d02b 100644 --- a/auto-check/scripts/memory.sh +++ b/auto-check/scripts/memory.sh @@ -1,14 +1,64 @@ echo "===TEST:memory===" -VM_WORKERS=${AUTOCHECK_THREADS:-4} -if [ "$VM_WORKERS" -gt 4 ]; then VM_WORKERS=4; fi + +# 读取温度（sensors 不可用时输出空） +read_temp() { + sensors 2>/dev/null | grep -oP '\+?\d+(\.\d+)?(?=°C)' | head -1 +} + +# 后台采样内存使用率 + 温度 +SAMPLE_LOG=/tmp/auto-check-memory-sample.log +: > "$SAMPLE_LOG" +( + while true; do + TS=$(date +%H:%M:%S) + MEM_TOTAL=$(awk '/^MemTotal:/{print $2}' /proc/meminfo) + MEM_AVAIL=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo) + if [ -z "$MEM_AVAIL" ] || [ "$MEM_AVAIL" = "0" ]; then + MEM_AVAIL=$(awk '/^MemFree:/{print $2}' /proc/meminfo) + fi + MEM_PCT=$(awk -v t="$MEM_TOTAL" -v a="$MEM_AVAIL" 'BEGIN{if(t>0) printf "%.1f", (t-a)/t*100}') + TEMP=$(read_temp) + echo "${TS},${MEM_PCT},${TEMP}" >> "$SAMPLE_LOG" + sleep "${AUTOCHECK_SAMPLE_INTERVAL:-2}" + done +) & +SAMPLE_PID=$! + START_TIME=$(date +%s%N) set +e -OUTPUT=$(stress-ng --vm "${VM_WORKERS}" --vm-bytes 256M --vm-method all --timeout "${AUTOCHECK_DURATION:-30}s" --metrics-brief 2>&1) +OUTPUT=$(stress-ng --vm "${AUTOCHECK_THREADS:-4}" --vm-bytes "${AUTOCHECK_MEM_MB:-256}M" --vm-method all --timeout "${AUTOCHECK_DURATION:-30}s" --temp-path /tmp --metrics-brief 2>&1) EXIT_CODE=$? set -e END_TIME=$(date +%s%N) + +kill $SAMPLE_PID 2>/dev/null || true +wait $SAMPLE_PID 2>/dev/null || true + DURATION_MS=$(( (END_TIME - START_TIME) / 1000000 )) if [ "$EXIT_CODE" -eq 0 ]; then echo 'status:pass'; else echo 'status:fail'; fi echo "duration:${DURATION_MS}ms" + +# 跑分：各 stressor 的 bogo ops/s 之和 +SCORE=$(echo "$OUTPUT" | awk ' +/^stress-ng:/ && ($4=="cpu" || $4=="vm" || $4=="iomix" || $4=="memrate") { + v=$(NF-1)+0; if (v>0) s+=v +} +END { printf "%.1f", s+0 }') +echo "score:${SCORE}" + +# 关键指标 +MAX_MEM=$(awk -F',' '$2!="" && $2+0>m {m=$2} END{printf "%.1f", m+0}' "$SAMPLE_LOG") +echo "metric:mem_used_pct=${MAX_MEM}" +MAX_TEMP=$(awk -F',' '$3!="" && $3+0>m {m=$3} END{printf "%.1f", m+0}' "$SAMPLE_LOG") +HAS_TEMP=$(awk -F',' '$3!="" && $3+0>0{f=1} END{print f+0}' "$SAMPLE_LOG") +if [ "$HAS_TEMP" = "1" ]; then echo "metric:max_temp=${MAX_TEMP}"; fi + echo "output:${OUTPUT}" + +# 采样数据（首行表头） +echo "===SAMPLES===" +echo "time,mem_used_pct,temp" +cat "$SAMPLE_LOG" +echo "===END_SAMPLES===" + echo '' diff --git a/auto-check/scripts/temp_end.sh b/auto-check/scripts/temp_end.sh index 7b216ef..a510e5f 100644 --- a/auto-check/scripts/temp_end.sh +++ b/auto-check/scripts/temp_end.sh @@ -1,12 +1,15 @@ echo '===MONITOR:temp===' if [ -f "$TEMP_LOG" ] && [ -s "$TEMP_LOG" ]; then + MAX_T=$(sort -n "$TEMP_LOG" | tail -1) + AVG_T=$(awk '{s+=$1; n++} END{printf "%.1f", (n?s/n:0)}' "$TEMP_LOG") + CNT=$(wc -l < "$TEMP_LOG") echo 'status:pass' - TEMP_MAX=$(grep -oP '\d+\.?\d*°C' "$TEMP_LOG" | sort -t. -k1 -n | tail -1 || echo 'unknown') - echo "samples:$(wc -l < "$TEMP_LOG")" - echo "max:${TEMP_MAX}" - echo "output:$(tail -10 "$TEMP_LOG")" + echo "metric:max_temp=${MAX_T}" + echo "metric:avg_temp=${AVG_T}" + echo "output:采样 ${CNT} 次, 最高 ${MAX_T}°C, 平均 ${AVG_T}°C" else echo 'status:skip' + echo 'output:未采集到温度数据' fi echo '' kill $TEMP_PID 2>/dev/null || true diff --git a/auto-check/scripts/temp_start.sh b/auto-check/scripts/temp_start.sh index 1dba5e7..ebcd309 100644 --- a/auto-check/scripts/temp_start.sh +++ b/auto-check/scripts/temp_start.sh @@ -1,4 +1,8 @@ TEMP_LOG=/tmp/auto-check-temp.log > "$TEMP_LOG" -(while true; do sensors 2>/dev/null | grep -i 'temp\|core\|cpu' | head -5 >> "$TEMP_LOG"; sleep "${AUTOCHECK_TEMP_INTERVAL:-10}"; done) & +(while true; do + T=$(sensors 2>/dev/null | grep -oP '\+?\d+(\.\d+)?(?=°C)' | head -1) + [ -n "$T" ] && echo "$T" >> "$TEMP_LOG" + sleep "${AUTOCHECK_TEMP_INTERVAL:-10}" +done) & TEMP_PID=$!