|
| 1 | +// compare.go — 关键指标对比：人话名称，当前值与 1h/6h/12h/1d/3d/7d 前的值并排，附变化百分比。 |
| 2 | +// |
| 3 | +// 首屏不再要求读者先懂 σ：先回答"现在和平时比变了什么"，z 值热力图与诊断链在下面给出"变化是否异常、谁干的"。 |
| 4 | +// 历史值用 Series.Lookup 精确取：24 小时内来自原始层，更早来自 5 分钟长期层。 |
| 5 | +// 没有历史时返回 null（页面显示"—"），绝不用 0 冒充。 |
| 6 | +package main |
| 7 | + |
| 8 | +import ( |
| 9 | + "math" |
| 10 | + "runtime" |
| 11 | + "sort" |
| 12 | + "strings" |
| 13 | + "time" |
| 14 | +) |
| 15 | + |
| 16 | +var compareCols = []struct { |
| 17 | + Name string |
| 18 | + Ago time.Duration |
| 19 | +}{ |
| 20 | + {"1h", time.Hour}, {"6h", 6 * time.Hour}, {"12h", 12 * time.Hour}, |
| 21 | + {"1d", 24 * time.Hour}, {"3d", 72 * time.Hour}, {"7d", 7 * 24 * time.Hour}, |
| 22 | +} |
| 23 | + |
| 24 | +type cmpRow struct { |
| 25 | + Label string `json:"label"` |
| 26 | + Unit string `json:"unit"` |
| 27 | + Now *float64 `json:"now"` |
| 28 | + Past []*float64 `json:"past"` |
| 29 | + Bad int `json:"bad"` // +1 往上是坏，-1 往下是坏，0 无所谓好坏 |
| 30 | + ID string `json:"id"` |
| 31 | +} |
| 32 | + |
| 33 | +type cmpGroup struct { |
| 34 | + Name string `json:"name"` |
| 35 | + Rows []cmpRow `json:"rows"` |
| 36 | +} |
| 37 | + |
| 38 | +type CompareJSON struct { |
| 39 | + At int64 `json:"at"` |
| 40 | + Cols []string `json:"cols"` |
| 41 | + Groups []cmpGroup `json:"groups"` |
| 42 | +} |
| 43 | + |
| 44 | +// cmpDef：一行的定义。ids 多于一个时按 f 合成（如 CPU 忙碌 = user+sys+softirq，按核数归一）。 |
| 45 | +type cmpDef struct { |
| 46 | + label, unit string |
| 47 | + bad int |
| 48 | + ids []string |
| 49 | + f func(v []float64) float64 |
| 50 | +} |
| 51 | + |
| 52 | +func one(label, unit string, bad int, id string) cmpDef { |
| 53 | + return cmpDef{label: label, unit: unit, bad: bad, ids: []string{id}} |
| 54 | +} |
| 55 | + |
| 56 | +func perCPU(label string, ids ...string) cmpDef { |
| 57 | + n := float64(runtime.NumCPU()) |
| 58 | + return cmpDef{label: label, unit: "percent", bad: 1, ids: ids, f: func(v []float64) float64 { |
| 59 | + s := 0.0 |
| 60 | + for _, x := range v { |
| 61 | + s += x |
| 62 | + } |
| 63 | + return s / n // cpu.* 是"占一个核的百分比"之和，除以核数得整机百分比 |
| 64 | + }} |
| 65 | +} |
| 66 | + |
| 67 | +var compareDefs = []struct { |
| 68 | + name string |
| 69 | + rows []cmpDef |
| 70 | +}{ |
| 71 | + {"CPU", []cmpDef{ |
| 72 | + perCPU("CPU 忙碌 %", "cpu.user", "cpu.sys", "cpu.softirq"), |
| 73 | + perCPU("iowait %", "cpu.iowait"), |
| 74 | + perCPU("steal %（虚机被宿主机抢占）", "cpu.steal"), |
| 75 | + one("1 分钟负载", "load", 1, "loadavg.1m"), |
| 76 | + one("运行队列", "count", 1, "procs_running"), |
| 77 | + one("CPU 压力 PSI %", "percent", 1, "psi.cpu.some10"), |
| 78 | + one("上下文切换/秒", "/s", 0, "ctxt"), |
| 79 | + one("中断/秒", "/s", 0, "intr"), |
| 80 | + }}, |
| 81 | + {"内存", []cmpDef{ |
| 82 | + one("可用内存", "bytes", -1, "mem.available"), |
| 83 | + one("页缓存", "bytes", 0, "mem.cached"), |
| 84 | + one("脏页", "bytes", 1, "mem.dirty"), |
| 85 | + one("内核 slab", "bytes", 1, "slab"), |
| 86 | + one("swap 使用", "bytes", 1, "swap.used"), |
| 87 | + one("内存压力 PSI %", "percent", 1, "psi.mem.some10"), |
| 88 | + one("主缺页/秒", "/s", 1, "pgmajfault"), |
| 89 | + }}, |
| 90 | + {"磁盘", []cmpDef{ |
| 91 | + one("最忙盘 util %", "percent", 1, "disk.util"), |
| 92 | + one("读延迟", "ms", 1, "disk.await_r"), |
| 93 | + one("写延迟", "ms", 1, "disk.await_w"), |
| 94 | + one("读 IOPS", "/s", 0, "disk.riops"), |
| 95 | + one("写 IOPS", "/s", 0, "disk.wiops"), |
| 96 | + one("读吞吐", "bytes/s", 0, "disk.rbytes"), |
| 97 | + one("写吞吐", "bytes/s", 0, "disk.wbytes"), |
| 98 | + one("IO 压力 PSI %", "percent", 1, "psi.io.some10"), |
| 99 | + one("D 状态进程", "count", 1, "procs_blocked"), |
| 100 | + }}, |
| 101 | + {"网络", []cmpDef{ |
| 102 | + one("网卡入向", "bytes/s", 0, "net.rx"), |
| 103 | + one("网卡出向", "bytes/s", 0, "net.tx"), |
| 104 | + one("收包/秒", "/s", 0, "net.rx_pps"), |
| 105 | + one("发包/秒", "/s", 0, "net.tx_pps"), |
| 106 | + one("收丢包/秒", "/s", 1, "net.rx_drop"), |
| 107 | + one("发丢包/秒", "/s", 1, "net.tx_drop"), |
| 108 | + one("TCP 重传/秒", "/s", 1, "tcp.retrans"), |
| 109 | + one("conntrack 条目", "count", 1, "conntrack"), |
| 110 | + }}, |
| 111 | +} |
| 112 | + |
| 113 | +// lookupTol：历史值的时间容差，约为回看时长的 1%，夹在 30 秒到 10 分钟之间。 |
| 114 | +func lookupTol(ago time.Duration) time.Duration { |
| 115 | + t := ago / 100 |
| 116 | + if t < 30*time.Second { |
| 117 | + t = 30 * time.Second |
| 118 | + } |
| 119 | + if t > 10*time.Minute { |
| 120 | + t = 10 * time.Minute |
| 121 | + } |
| 122 | + return t |
| 123 | +} |
| 124 | + |
| 125 | +func (b *HeatmapBuilder) valueAt(d cmpDef, at time.Time, tol time.Duration, current bool) *float64 { |
| 126 | + vals := make([]float64, 0, len(d.ids)) |
| 127 | + for _, id := range d.ids { |
| 128 | + var v float64 |
| 129 | + var ok bool |
| 130 | + if current { |
| 131 | + var p point |
| 132 | + if p, ok = b.series.Last(id); ok && at.Sub(p.TS) > 2*time.Minute { |
| 133 | + ok = false // 最后一个点太旧：该指标已停止上报 |
| 134 | + } |
| 135 | + v = p.V |
| 136 | + } else { |
| 137 | + v, ok = b.series.Lookup(id, at, tol) |
| 138 | + } |
| 139 | + if !ok || math.IsNaN(v) || math.IsInf(v, 0) { |
| 140 | + return nil |
| 141 | + } |
| 142 | + vals = append(vals, v) |
| 143 | + } |
| 144 | + x := vals[0] |
| 145 | + if d.f != nil { |
| 146 | + x = d.f(vals) |
| 147 | + } |
| 148 | + return &x |
| 149 | +} |
| 150 | + |
| 151 | +// Compare 生成对比表。多块盘 / 多张网卡时追加逐设备的行。 |
| 152 | +func (b *HeatmapBuilder) Compare(now time.Time) *CompareJSON { |
| 153 | + out := &CompareJSON{At: now.Unix()} |
| 154 | + for _, c := range compareCols { |
| 155 | + out.Cols = append(out.Cols, c.Name) |
| 156 | + } |
| 157 | + ids := b.series.MetricIDs() |
| 158 | + perDev := func(prefix, label, unit string, bad int) []cmpDef { |
| 159 | + var devs []string |
| 160 | + for _, id := range ids { |
| 161 | + if strings.HasPrefix(id, prefix) { |
| 162 | + devs = append(devs, strings.TrimPrefix(id, prefix)) |
| 163 | + } |
| 164 | + } |
| 165 | + if len(devs) < 2 { // 只有一块盘/一张网卡时整机行已经说明一切 |
| 166 | + return nil |
| 167 | + } |
| 168 | + sort.Strings(devs) |
| 169 | + var rows []cmpDef |
| 170 | + for _, d := range devs { |
| 171 | + rows = append(rows, one(label+" · "+d, unit, bad, prefix+d)) |
| 172 | + } |
| 173 | + return rows |
| 174 | + } |
| 175 | + for _, g := range compareDefs { |
| 176 | + defs := g.rows |
| 177 | + switch g.name { |
| 178 | + case "磁盘": |
| 179 | + defs = append(append([]cmpDef(nil), defs...), perDev("disk.util@", "util %", "percent", 1)...) |
| 180 | + defs = append(defs, perDev("disk.await_w@", "写延迟", "ms", 1)...) |
| 181 | + case "网络": |
| 182 | + defs = append(append([]cmpDef(nil), defs...), perDev("net.rx@", "入向", "bytes/s", 0)...) |
| 183 | + defs = append(defs, perDev("net.tx@", "出向", "bytes/s", 0)...) |
| 184 | + defs = append(defs, perDev("net.rx_drop@", "收丢包/秒", "/s", 1)...) |
| 185 | + } |
| 186 | + grp := cmpGroup{Name: g.name} |
| 187 | + for _, d := range defs { |
| 188 | + row := cmpRow{Label: d.label, Unit: d.unit, Bad: d.bad, ID: strings.Join(d.ids, "+")} |
| 189 | + if row.Now = b.valueAt(d, now, 0, true); row.Now == nil { |
| 190 | + continue // 这台机器没有这项（比如没有 PSI、没有 conntrack）：整行不显示 |
| 191 | + } |
| 192 | + for _, c := range compareCols { |
| 193 | + row.Past = append(row.Past, b.valueAt(d, now.Add(-c.Ago), lookupTol(c.Ago), false)) |
| 194 | + } |
| 195 | + grp.Rows = append(grp.Rows, row) |
| 196 | + } |
| 197 | + if len(grp.Rows) > 0 { |
| 198 | + out.Groups = append(out.Groups, grp) |
| 199 | + } |
| 200 | + } |
| 201 | + return out |
| 202 | +} |
0 commit comments