v2.9: two-tier storage with per-round downsampling, canvas smoke layer, 300-day window

Storage: day files older than hot_days (30) are rewritten in place with each round
collapsed to [min, median, p90, max]; files past retention (300) are deleted. The
round itself survives — same timestamp, sent/recv counts, burst flag and z-score — so
cold data flows through the same Round struct, the same /api/series, and the same
render code as hot data. No second data structure, no second render path: the tiering
is invisible above the storage layer, which is what kept breaking before.

Rendering: the smoke scatter moved off ECharts onto a plain canvas overlay written
directly into an ImageData buffer. ECharts still owns the axes, median line, loss bars
and tooltip; the canvas only paints samples, which needs no per-point hit testing.
This is what makes long windows drawable at all — ECharts kept per-point state and
died in the hundreds of thousands; a pixel buffer does not care.

Verified end to end in a browser: 6h -> 24h -> 7d -> 30d -> 70d -> 300d -> 1h -> 30d
-> 6h, canvas fingerprint changes on every switch, returning to a window reproduces
its earlier fingerprint, zero JS errors. Unit tests cover downsampleRound (metadata
preserved, idempotent) and Tier (old days downsampled, recent days untouched, round
count unchanged). Race detector clean.

Author: githubflyideas

config.go

@@ -19,7 +19,8 @@ type Config struct {
 	TargetsDir    string
 	Targets       []TargetCfg
 	Probe         ProbeCfg
-	RetentionDays int
+	HotDays       int // full samples kept this long
+	RetentionDays int // downsampled data kept this long
 }
 
 func defaultConfig() *Config {
@@ -28,7 +29,8 @@ func defaultConfig() *Config {
 		DataDir:       "./data",
 		TargetsDir:    "./targets",
 		Probe:         ProbeCfg{IntervalSec: 60, Packets: 20, GapMs: 50, TimeoutMs: 1000},
-		RetentionDays: 70,
+		HotDays:       30,
+		RetentionDays: 300,
 	}
 }
 

main.go

@@ -222,7 +222,7 @@ func housekeeping(cfg *Config, store *Store, stop chan struct{}) {
 		day := now.Format("2006-01-02")
 		if now.Hour() == 0 && now.Minute() == 5 && last != day {
 			last = day
-			store.Retention(cfg.RetentionDays)
+			store.Tier(cfg.HotDays, cfg.RetentionDays)
 		}
 	}
 }

pingping_test.go

@@ -1,6 +1,11 @@
 package main
 
-import "testing"
+import (
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+)
 
 func TestParseListLine(t *testing.T) {
 	tt, err := parseListLine("59.43.247.1 HK CN2 pace=fast", "icmp")
@@ -95,3 +100,61 @@ func FuzzRobustZ(f *testing.F) {
 		}
 	})
 }
+
+func TestDownsampleRound(t *testing.T) {
+	ms := make([]float64, 20)
+	for i := range ms {
+		ms[i] = float64(i + 1) // 1..20
+	}
+	in := Round{T: 1234567890, S: 20, R: 18, MS: ms, B: true, Z: 3.42}
+	out := downsampleRound(in)
+
+	if len(out.MS) != 4 {
+		t.Fatalf("want 4 samples, got %d", len(out.MS))
+	}
+	// min / median / p90 / max preserved in order
+	if out.MS[0] != 1 || out.MS[3] != 20 {
+		t.Fatalf("min/max wrong: %v", out.MS)
+	}
+	if out.MS[1] < out.MS[0] || out.MS[2] < out.MS[1] || out.MS[3] < out.MS[2] {
+		t.Fatalf("not sorted: %v", out.MS)
+	}
+	// everything else must survive untouched — this is what keeps one code path
+	if out.T != in.T || out.S != in.S || out.R != in.R || out.B != in.B || out.Z != in.Z {
+		t.Fatalf("metadata lost: %+v", out)
+	}
+	// idempotent: downsampling twice changes nothing
+	if again := downsampleRound(out); len(again.MS) != 4 || again.MS[0] != out.MS[0] {
+		t.Fatalf("not idempotent: %v", again.MS)
+	}
+}
+
+func TestTierDownsamplesOldDays(t *testing.T) {
+	dir := t.TempDir()
+	s, err := NewStore(dir, []TargetCfg{{Name: "X", Type: "icmp", Host: "1.1.1.1", dir: "X"}})
+	if err != nil { t.Fatal(err) }
+	old := time.Now().AddDate(0, 0, -40).Format("2006-01-02")
+	recent := time.Now().AddDate(0, 0, -5).Format("2006-01-02")
+	ms := make([]float64, 20)
+	for i := range ms { ms[i] = float64(i + 30) }
+	for _, day := range []string{old, recent} {
+		f, _ := os.Create(dir + "/X/" + day + ".jsonl")
+		for i := 0; i < 10; i++ {
+			line, _ := json.Marshal(Round{T: time.Now().Unix() - int64(i*60), S: 20, R: 20, MS: ms})
+			f.Write(append(line, byte(10)))
+		}
+		f.Close()
+	}
+	s.Tier(30, 300)
+	oldRounds, _ := s.readDay("X", old)
+	newRounds, _ := s.readDay("X", recent)
+	if len(oldRounds) == 0 || len(oldRounds[0].MS) != 4 {
+		t.Fatalf("old day should be downsampled to 4 samples, got %d", len(oldRounds[0].MS))
+	}
+	if len(newRounds) == 0 || len(newRounds[0].MS) != 20 {
+		t.Fatalf("recent day must keep all samples, got %d", len(newRounds[0].MS))
+	}
+	if len(oldRounds) != 10 {
+		t.Fatalf("round count must survive downsampling: %d", len(oldRounds))
+	}
+}

static/index.html

@@ -97,7 +97,7 @@
 </div>
 
 <div class="scope">
-  <div class="crt"><div id="chart"></div></div>
+  <div class="crt"><div id="chart"></div><canvas id="smokeCv"></canvas></div>
   <div class="readouts">
     <div class="ro"><div class="v" id="p50">–</div><div class="l">P50 MS</div></div>
     <div class="ro"><div class="v" id="p90">–</div><div class="l">P90 MS</div></div>
@@ -114,6 +114,7 @@
       <button data-m="10080">7d</button>
       <button data-m="43200">30d</button>
       <button data-m="100800">70d</button>
+      <button data-m="432000">300d</button>
     </div>
     <div class="rng custom">
       <input type="datetime-local" id="customFrom" step="60">
@@ -166,6 +167,83 @@
 
 const chart = echarts.init($('chart'));
 
+
+// 自绘烟雾层:ECharts 的散点图在几十万点就吃不消,因为它给每个点维护状态。
+// 烟雾不需要单点命中(tooltip 按轮显示),所以直接写像素缓冲 —— 百万点也是一次遍历。
+function drawSmoke(rounds) {
+  const cv = $('smokeCv'), box = $('chart');
+  const W = box.clientWidth, H = box.clientHeight;
+  const dpr = window.devicePixelRatio || 1;
+  cv.width = W * dpr; cv.height = H * dpr;
+  cv.style.width = W + 'px'; cv.style.height = H + 'px';
+  const ctx = cv.getContext('2d');
+  ctx.clearRect(0, 0, cv.width, cv.height);
+  if (!rounds.length) return;
+
+  // 与 ECharts 网格对齐(grid: left 52, right 46, top 18, bottom 42)
+  const gl = 52 * dpr, gr = 46 * dpr, gt = 18 * dpr, gb = 42 * dpr;
+  const plotW = cv.width - gl - gr, plotH = cv.height - gt - gb;
+  if (plotW <= 0 || plotH <= 0) return;
+
+  // 用 ECharts 当前的坐标范围,保证两层严丝合缝
+  const opt = chart.getOption();
+  const yAxis = opt.yAxis && opt.yAxis[0];
+  const t0 = rounds[0].t * 1000, t1 = rounds[rounds.length - 1].t * 1000;
+  const yMin = yAxis && yAxis.min != null ? yAxis.min : 0;
+  let yMax = yAxis && yAxis.max != null ? yAxis.max : 0;
+  if (!yMax) { // 回退:自己算上界
+    for (const r of rounds) for (const v of (r.ms || [])) if (v > yMax) yMax = v;
+    yMax *= 1.05;
+  }
+  const tSpan = Math.max(1, t1 - t0), ySpan = Math.max(1e-6, yMax - yMin);
+
+  // 逐点写 ImageData:比 fillRect 快一个数量级
+  const img = ctx.createImageData(cv.width, cv.height);
+  const buf = img.data;
+  const hot = hexToRgb(cssv('--smoke-hot')), mid = hexToRgb(cssv('--smoke-mid')), cool = hexToRgb(cssv('--smoke-cool'));
+
+  for (const r of rounds) {
+    const ms = r.ms;
+    if (!ms || !ms.length) continue;
+    const x = gl + (r.t * 1000 - t0) / tSpan * plotW;
+    if (x < 0 || x >= cv.width) continue;
+    // 该轮中位数用于着色:贴线的亮,散开的冷
+    const srt = ms.length > 2 ? [...ms].sort((a, b) => a - b) : ms;
+    const med = srt[Math.floor(srt.length / 2)];
+    for (const v of ms) {
+      const y = gt + plotH - (v - yMin) / ySpan * plotH;
+      if (y < 0 || y >= cv.height) continue;
+      const dev = med ? Math.min(1, Math.abs(v - med) / med / 0.5) : 0;
+      const c = dev < 0.5 ? lerp(hot, mid, dev * 2) : lerp(mid, cool, (dev - 0.5) * 2);
+      const a = Math.round((0.5 - dev * 0.4) * 255);
+      const xi = x | 0, yi = y | 0;
+      for (let dx = 0; dx < 2; dx++) for (let dy = 0; dy < 2; dy++) { // 2x2 让点可见
+        const px = xi + dx, py = yi + dy;
+        if (px >= cv.width || py >= cv.height) continue;
+        const idx = (py * cv.width + px) * 4;
+        // 叠加混合:重叠越多越亮,这正是烟雾的质感
+        buf[idx] = Math.min(255, buf[idx] + c[0] * a / 255);
+        buf[idx + 1] = Math.min(255, buf[idx + 1] + c[1] * a / 255);
+        buf[idx + 2] = Math.min(255, buf[idx + 2] + c[2] * a / 255);
+        buf[idx + 3] = Math.min(255, buf[idx + 3] + a);
+      }
+    }
+  }
+  ctx.putImageData(img, 0, 0);
+}
+
+function hexToRgb(h) {
+  h = (h || '#7ce38b').trim();
+  if (h[0] === '#') h = h.slice(1);
+  if (h.length === 3) h = h.split('').map(c => c + c).join('');
+  const n = parseInt(h, 16);
+  return [(n >> 16) & 255, (n >> 8) & 255, n & 255];
+}
+function lerp(a, b, t) {
+  t = Math.max(0, Math.min(1, t));
+  return [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t];
+}
+
 async function loadTargets() {
   targets = await (await fetch('/api/targets')).json() || [];
   const kinds = new Set(targets.map(t => t.type));
@@ -183,6 +261,7 @@
 }
 
 let sel = null; // {from,to} 自定义查询区间(unix 秒)
+let lastRounds = [];
 let renderSeq = 0;
 let rendering = false; // 一次渲染(含慢速 band 查询)是否仍在进行
 async function render() {
@@ -291,13 +370,7 @@
         if (i.b) html += ` <span style="color:${cssv('--alert')}">· ◆ burst z=${i.z ? i.z.toFixed(2) : '—'}</span>`;
         return html;
       } },
-    // 荧光束渐变:贴线的样本 = 炽亮核心,散开的样本渐冷渐暗 —— 渐变即信息
-    visualMap: { show: false, type: 'continuous', dimension: 2, min: 0, max: 0.5,
-      seriesIndex: 0,
-      inRange: { color: [cssv('--smoke-hot'), cssv('--smoke-mid'), cssv('--smoke-cool')],
-                 opacity: [0.5, 0.10] } },
     series: [
-      { type: 'scatter', data: smoke, symbolSize: 2.2, silent: true, large: true },
       { type: 'line', data: median, showSymbol: false, silent: true, z: 4,
         lineStyle: { color: phos, width: 7, opacity: 0.10 } },
       { type: 'line', data: median, showSymbol: false,
@@ -310,6 +383,8 @@
         tooltip: { formatter: p => `◆ z-score ${p.value[2] ? p.value[2].toFixed(2) : '—'} · loss ${p.value[1].toFixed(0)}% (${p.value[3]}/${p.value[4]})` } }
     ]
   }, true);
+  drawSmoke(rounds); // ECharts 画完坐标轴后叠加烟雾层
+  lastRounds = rounds;
 }
 
 
@@ -379,7 +454,7 @@
   applyTheme();
   render();
 };
-window.addEventListener('resize', () => chart && chart.resize());
+window.addEventListener('resize', () => { if (chart) { chart.resize(); drawSmoke(lastRounds); } });
 </script>
 </body>
 </html>

store.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 )
@@ -259,6 +260,104 @@ func pct(sorted []float64, p float64) float64 {
 }
 
 // Retention:保留期就是 rm。按天分文件让清理不需要任何压缩整理逻辑。
+// downsampleRound collapses a round's samples to [min, median, P90, max].
+// The round itself is preserved — same timestamp, same sent/recv counts, same burst
+// flag and z-score — so cold data flows through the exact same code path as hot data.
+// Only the within-round redundancy is dropped: on a month-wide axis those 20-30
+// samples land in the same pixel column anyway.
+func downsampleRound(r Round) Round {
+	if len(r.MS) <= 4 {
+		return r // already small enough
+	}
+	sorted := append([]float64(nil), r.MS...)
+	sort.Float64s(sorted)
+	n := len(sorted)
+	r.MS = []float64{
+		round2(sorted[0]),
+		round2(sorted[n/2]),
+		round2(sorted[int(float64(n)*0.9)]),
+		round2(sorted[n-1]),
+	}
+	return r
+}
+
+// Downsample rewrites one day file in place with per-round downsampling. Files are
+// rewritten atomically via a temp file so a crash cannot leave a half-written day.
+func (s *Store) Downsample(name, day string) error {
+	rounds, err := s.readDay(name, day)
+	if err != nil || len(rounds) == 0 {
+		return err
+	}
+	// already downsampled? cheap check on the first round
+	if len(rounds[0].MS) <= 4 {
+		return nil
+	}
+	path := s.dayFile(name, day)
+	tmp := path + ".tmp"
+	f, err := os.Create(tmp)
+	if err != nil {
+		return err
+	}
+	for _, r := range rounds {
+		line, err := json.Marshal(downsampleRound(r))
+		if err != nil {
+			f.Close()
+			os.Remove(tmp)
+			return err
+		}
+		if _, err := f.Write(append(line, byte(10))); err != nil {
+			f.Close()
+			os.Remove(tmp)
+			return err
+		}
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	return os.Rename(tmp, path)
+}
+
+// Tier runs nightly: day files older than hotDays get downsampled in place, files
+// older than keepDays are deleted. One pass, no separate cold directory — the
+// tiering is invisible above the storage layer.
+func (s *Store) Tier(hotDays, keepDays int) {
+	hotCut := time.Now().AddDate(0, 0, -hotDays).Format("2006-01-02")
+	keepCut := time.Now().AddDate(0, 0, -keepDays).Format("2006-01-02")
+	s.mu.RLock()
+	names := make([]string, 0, len(s.dirs))
+	for n := range s.dirs {
+		names = append(names, n)
+	}
+	s.mu.RUnlock()
+
+	for _, name := range names {
+		s.mu.RLock()
+		dir := s.dirs[name]
+		s.mu.RUnlock()
+		entries, err := os.ReadDir(filepath.Join(s.dir, dir))
+		if err != nil {
+			continue
+		}
+		for _, e := range entries {
+			fn := e.Name()
+			if len(fn) < 10 || strings.HasSuffix(fn, ".tmp") {
+				continue
+			}
+			day := fn[:10]
+			switch {
+			case day < keepCut:
+				os.Remove(filepath.Join(s.dir, dir, fn))
+				log.Printf("retention: removed %s/%s", dir, fn)
+			case day < hotCut:
+				if err := s.Downsample(name, day); err != nil {
+					log.Printf("downsample %s/%s failed: %v", dir, day, err)
+				}
+			}
+		}
+	}
+}
+
 func (s *Store) Retention(days int) {
 	if days <= 0 {
 		return

web.go

@@ -164,7 +164,7 @@ func serveWeb(cfg *Config, store *Store, users map[string]string) error {
 		to, _ := strconv.ParseInt(q.Get("to"), 10, 64)
 		if from == 0 || to == 0 {
 			minutes, _ := strconv.Atoi(q.Get("minutes"))
-			if minutes <= 0 || minutes > 100800 { // up to 70 days
+			if minutes <= 0 || minutes > 432000 { // up to 300 days
 				minutes = 360
 			}
 			to = time.Now().Unix()