hot reload sandbox and improve web fetch

Author: idoubi

cmd/fastclaw/main.go

@@ -64,6 +64,10 @@ func (a *apiResolver) EnsureAgent(ctx context.Context, userID, agentID string) e
 // chat turn.
 func (a *apiResolver) ReloadAgents() error { return a.gw.ReloadAgents() }
 
+// ReloadSandbox rebuilds the gateway-owned sandbox executor pool after
+// system-scope sandbox settings change, avoiding a manual process restart.
+func (a *apiResolver) ReloadSandbox() error { return a.gw.ReloadSandbox() }
+
 // RegisterChannelFromConfig hot-starts a freshly-saved channel row.
 // Called by setup handlers after they persist a new bot config so the
 // adapter starts polling without a process restart.

internal/agent/tools/web_fetch.go

@@ -8,11 +8,11 @@ import (
 	"net"
 	"net/http"
 	"net/url"
-	"regexp"
 	"strings"
 	"time"
 
 	"github.com/fastclaw-ai/fastclaw/internal/toolproviders"
+	webfetchprovider "github.com/fastclaw-ai/fastclaw/internal/toolproviders/webfetch"
 )
 
 type webFetchArgs struct {
@@ -26,8 +26,6 @@ const (
 	fetchUserAgent = "FastClaw/1.0 (AI Agent Web Fetcher)"
 )
 
-var htmlTagRe = regexp.MustCompile(`<[^>]*>`)
-
 // safeFetchClient is an http.Client whose dialer rejects private,
 // loopback, link-local, multicast, and CGNAT addresses — the SSRF
 // defense for web_fetch. The check runs at DIAL time, after DNS has
@@ -275,15 +273,17 @@ func webFetchTool(ctx context.Context, r *Registry, rawArgs json.RawMessage) (st
 		return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status)
 	}
 
-	// Read body with a limit to prevent memory issues
-	limitReader := io.LimitReader(resp.Body, int64(maxLen*3)) // read more than needed since HTML is verbose
+	// Read body with a limit to prevent memory issues. WeChat articles
+	// need a bounded larger window because the readable #js_content node
+	// can appear after a long script/config prelude.
+	limitReader := io.LimitReader(resp.Body, webfetchprovider.FetchReadLimit(args.URL, maxLen))
 	body, err := io.ReadAll(limitReader)
 	if err != nil {
 		return "", fmt.Errorf("read body: %w", err)
 	}
 
-	// Strip HTML tags
-	text := stripHTML(string(body))
+	// Strip HTML tags, using site-specific article extraction when needed.
+	text := webfetchprovider.HTMLToText(args.URL, string(body))
 
 	// Truncate to max length (UTF-8 safe: back up to a valid rune boundary).
 	if len(text) > maxLen {
@@ -312,33 +312,3 @@ func assertHTTPScheme(rawURL string) error {
 	}
 	return nil
 }
-
-// stripHTML removes HTML tags and cleans up whitespace.
-func stripHTML(html string) string {
-	// Remove script and style elements entirely
-	scriptRe := regexp.MustCompile(`(?is)<script[^>]*>.*?</script>`)
-	html = scriptRe.ReplaceAllString(html, "")
-	styleRe := regexp.MustCompile(`(?is)<style[^>]*>.*?</style>`)
-	html = styleRe.ReplaceAllString(html, "")
-
-	// Remove HTML tags
-	text := htmlTagRe.ReplaceAllString(html, " ")
-
-	// Decode common HTML entities
-	text = strings.ReplaceAll(text, "&amp;", "&")
-	text = strings.ReplaceAll(text, "&lt;", "<")
-	text = strings.ReplaceAll(text, "&gt;", ">")
-	text = strings.ReplaceAll(text, "&quot;", "\"")
-	text = strings.ReplaceAll(text, "&#39;", "'")
-	text = strings.ReplaceAll(text, "&nbsp;", " ")
-
-	// Collapse whitespace
-	spaceRe := regexp.MustCompile(`[ \t]+`)
-	text = spaceRe.ReplaceAllString(text, " ")
-
-	// Collapse multiple newlines
-	nlRe := regexp.MustCompile(`\n{3,}`)
-	text = nlRe.ReplaceAllString(text, "\n\n")
-
-	return strings.TrimSpace(text)
-}

internal/gateway/reload.go

@@ -6,6 +6,7 @@ import (
 	"log/slog"
 
 	"github.com/fastclaw-ai/fastclaw/internal/channels"
+	"github.com/fastclaw-ai/fastclaw/internal/sandbox"
 	"github.com/fastclaw-ai/fastclaw/internal/store"
 )
 
@@ -55,6 +56,45 @@ func (g *Gateway) ReloadAgents() error {
 	return nil
 }
 
+// ReloadSandbox rebuilds the gateway-wide sandbox executor pool from the
+// current system-scope sandbox config. It is called after the admin/settings
+// UI saves sandbox changes so exec tools can pick them up without a process
+// restart.
+func (g *Gateway) ReloadSandbox() error {
+	if g.store == nil {
+		return nil
+	}
+	cfg := readSystemSandboxCfg(g.store)
+	next := buildSystemSandboxPool(cfg, g.workspace)
+
+	g.mu.Lock()
+	prev := g.sandboxPool
+	g.sandboxPool = next
+	if g.projectRuntime != nil {
+		g.projectRuntime.SetSandboxPool(cfg.Backend, next)
+	}
+	evicted := 0
+	if g.users != nil {
+		evicted = g.users.setSystemSandboxPool(next)
+	}
+	g.mu.Unlock()
+
+	if prev != nil && prev != next {
+		prev.CloseAll()
+	}
+	slog.Info("hot-reload: sandbox executor pool rebuilt",
+		"backend", sandboxPoolBackend(next),
+		"evictedUserSpaces", evicted)
+	return nil
+}
+
+func sandboxPoolBackend(p sandbox.ExecutorPool) string {
+	if p == nil {
+		return ""
+	}
+	return p.Backend()
+}
+
 // reloadAgentForUser is a finer-grained invalidate used by setup handlers
 // after a single user mutates their own agents.
 func (g *Gateway) reloadAgentForUser(_ context.Context, userID string) {

internal/gateway/userspace.go

@@ -959,6 +959,18 @@ func (r *userSpaceRegistry) setProjectRuntime(m *coderuntime.Manager) {
 	r.mu.Unlock()
 }
 
+// setSystemSandboxPool swaps the gateway-owned pool reference used for
+// future UserSpace loads, then drops already-loaded spaces so active agents
+// reattach to the new pool on their next request.
+func (r *userSpaceRegistry) setSystemSandboxPool(p sandbox.ExecutorPool) int {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.systemSandboxPool = p
+	evicted := len(r.spaces)
+	r.spaces = make(map[string]*userSpaceEntry)
+	return evicted
+}
+
 type userSpaceEntry struct {
 	space    *UserSpace
 	lastUsed time.Time

internal/runtime/runtime.go

@@ -179,6 +179,16 @@ func NewManager(st store.Store, workspaceRoot, image string, policy *sandbox.Pol
 	}
 }
 
+// SetSandboxPool updates the borrowed turn-sandbox pool after system
+// sandbox settings are saved. Existing live runtimes keep running; new
+// pooled preview starts use the refreshed backend/pool.
+func (m *Manager) SetSandboxPool(backend string, pool sandbox.ExecutorPool) {
+	m.mu.Lock()
+	m.backend = backend
+	m.pool = pool
+	m.mu.Unlock()
+}
+
 // usesPool reports whether the preview should run through the shared
 // turn-sandbox pool (non-docker backend) rather than a dedicated docker
 // container. Requires a pool to borrow; without one we fall back to the

internal/setup/handlers.go

@@ -540,8 +540,9 @@ func (s *Server) handleUpdateConfig(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	var raw struct {
-		Prefs  *config.PrefsCfg `json:"prefs"`
-		Skills *struct {
+		Prefs   *config.PrefsCfg `json:"prefs"`
+		Sandbox *json.RawMessage `json:"sandbox"`
+		Skills  *struct {
 			AgentEntries map[string]map[string]config.SkillEntryCfg `json:"agentEntries"`
 		} `json:"skills"`
 	}
@@ -610,10 +611,27 @@ func (s *Server) handleUpdateConfig(w http.ResponseWriter, r *http.Request) {
 	// agent loaded before the change keeps seeing the stale model and
 	// surfaces "no usable LLM provider" in chat.
 	sc, scopeID := s.scopeForSave(r)
+	if sc == scope.System && raw.Sandbox != nil {
+		if err := s.reloadSystemSandbox(); err != nil {
+			jsonResponse(w, http.StatusInternalServerError, map[string]any{"ok": false, "error": err.Error()})
+			return
+		}
+	}
 	s.invalidateScope(sc, scopeID)
 	jsonResponse(w, http.StatusOK, map[string]any{"ok": true})
 }
 
+func (s *Server) reloadSystemSandbox() error {
+	type sandboxReloader interface{ ReloadSandbox() error }
+	if s.userResolver == nil {
+		return nil
+	}
+	if r, ok := s.userResolver.(sandboxReloader); ok {
+		return r.ReloadSandbox()
+	}
+	return nil
+}
+
 // scopeForSave mirrors the scope-resolution logic in saveUserConfig so
 // callers can invalidate exactly the UserSpaces that were just touched.
 func (s *Server) scopeForSave(r *http.Request) (string, string) {

internal/setup/handlers_admin.go

@@ -187,12 +187,12 @@ type onboardRequest struct {
 	Password    string `json:"password"`
 	DisplayName string `json:"displayName,omitempty"`
 
-	Provider  string `json:"provider"`
-	APIBase   string `json:"apiBase"`
-	APIKey    string `json:"apiKey"`
-	APIType   string `json:"apiType,omitempty"`
-	AuthType  string `json:"authType,omitempty"`
-	Model     string `json:"model"`
+	Provider string `json:"provider"`
+	APIBase  string `json:"apiBase"`
+	APIKey   string `json:"apiKey"`
+	APIType  string `json:"apiType,omitempty"`
+	AuthType string `json:"authType,omitempty"`
+	Model    string `json:"model"`
 
 	AgentName string `json:"agentName,omitempty"`
 
@@ -317,6 +317,10 @@ func (s *Server) handleOnboard(w http.ResponseWriter, r *http.Request) {
 			jsonResponse(w, http.StatusInternalServerError, map[string]any{"ok": false, "error": err.Error()})
 			return
 		}
+		if err := s.reloadSystemSandbox(); err != nil {
+			jsonResponse(w, http.StatusInternalServerError, map[string]any{"ok": false, "error": err.Error()})
+			return
+		}
 	}
 	cookie, err := s.authResolver.IssueSession(r.Context(), acct.ID)
 	if err == nil {

internal/toolproviders/webfetch/direct.go

@@ -16,9 +16,9 @@ import (
 // pick any other provider.
 type Direct struct{}
 
-func (Direct) Category() string       { return Category }
-func (Direct) Name() string           { return "direct" }
-func (Direct) CredentialFree() bool   { return true }
+func (Direct) Category() string     { return Category }
+func (Direct) Name() string         { return "direct" }
+func (Direct) CredentialFree() bool { return true }
 
 const (
 	directTimeout   = 30 * time.Second
@@ -57,11 +57,13 @@ func (d *Direct) Execute(ctx context.Context, req toolproviders.Request) (toolpr
 	}
 
 	// Read 3× the cap because HTML is verbose and stripping tags shrinks
-	// it substantially — same heuristic the legacy direct fetcher used.
-	body, err := io.ReadAll(io.LimitReader(resp.Body, int64(a.MaxLen*3)))
+	// it substantially. WeChat articles are much larger before the
+	// #js_content article body, so give those a bounded larger window and
+	// extract the article node before truncating.
+	body, err := io.ReadAll(io.LimitReader(resp.Body, FetchReadLimit(a.URL, a.MaxLen)))
 	if err != nil {
 		return toolproviders.Response{}, toolproviders.Retry(fmt.Errorf("direct read: %w", err))
 	}
-	text := truncate(stripHTML(string(body)), a.MaxLen)
+	text := truncate(HTMLToText(a.URL, string(body)), a.MaxLen)
 	return toolproviders.Response{Text: text}, nil
 }

internal/toolproviders/webfetch/webfetch.go

@@ -6,6 +6,8 @@ package webfetch
 
 import (
 	"fmt"
+	stdhtml "html"
+	"net/url"
 	"regexp"
 	"strings"
 
@@ -62,6 +64,61 @@ func truncate(text string, maxLen int) string {
 	return text[:maxLen] + "\n[...truncated]"
 }
 
+func isWeChatArticle(rawURL string) bool {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return false
+	}
+	host := strings.ToLower(u.Hostname())
+	return host == "mp.weixin.qq.com" || strings.HasSuffix(host, ".mp.weixin.qq.com")
+}
+
+func FetchReadLimit(rawURL string, maxLen int) int64 {
+	if isWeChatArticle(rawURL) {
+		return 2 << 20
+	}
+	return int64(maxLen * 3)
+}
+
+func HTMLToText(rawURL, htmlBody string) string {
+	if isWeChatArticle(rawURL) {
+		if article := extractElementByID(htmlBody, "js_content"); article != "" {
+			return stripHTML(article)
+		}
+	}
+	return stripHTML(htmlBody)
+}
+
+func extractElementByID(htmlBody, id string) string {
+	re := regexp.MustCompile(`(?is)<([a-z0-9]+)\b[^>]*\bid\s*=\s*['"]` + regexp.QuoteMeta(id) + `['"][^>]*>`)
+	loc := re.FindStringSubmatchIndex(htmlBody)
+	if loc == nil {
+		return ""
+	}
+	tagName := strings.ToLower(htmlBody[loc[2]:loc[3]])
+	start := loc[0]
+	searchFrom := loc[1]
+	tagRe := regexp.MustCompile(`(?is)<\s*(/?)\s*` + regexp.QuoteMeta(tagName) + `\b[^>]*>`)
+	depth := 1
+	for _, m := range tagRe.FindAllStringSubmatchIndex(htmlBody[searchFrom:], -1) {
+		tagStart := searchFrom + m[0]
+		tagEnd := searchFrom + m[1]
+		closing := m[2] >= 0 && htmlBody[searchFrom+m[2]:searchFrom+m[3]] == "/"
+		selfClosing := strings.HasSuffix(strings.TrimSpace(htmlBody[tagStart:tagEnd]), "/>")
+		if closing {
+			depth--
+			if depth == 0 {
+				return htmlBody[start:tagEnd]
+			}
+			continue
+		}
+		if !selfClosing {
+			depth++
+		}
+	}
+	return ""
+}
+
 var htmlTagRe = regexp.MustCompile(`<[^>]*>`)
 
 // stripHTML removes script/style blocks, drops remaining HTML tags, and
@@ -81,6 +138,7 @@ func stripHTML(html string) string {
 	text = strings.ReplaceAll(text, "&quot;", "\"")
 	text = strings.ReplaceAll(text, "&#39;", "'")
 	text = strings.ReplaceAll(text, "&nbsp;", " ")
+	text = stdhtml.UnescapeString(text)
 
 	spaceRe := regexp.MustCompile(`[ \t]+`)
 	text = spaceRe.ReplaceAllString(text, " ")

internal/toolproviders/webfetch/webfetch_test.go

@@ -0,0 +1,33 @@
+package webfetch
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestHTMLToTextExtractsWeChatArticleBody(t *testing.T) {
+	html := `<html><head><script>window.cgiData = { noisy: "` + strings.Repeat("x", 1024) + `" }</script></head>
+<body>
+<div id="js_content" class="rich_media_content">
+  <section><p>第一段&nbsp;内容</p><div><p>第二段 &amp; 更多</p></div></section>
+</div>
+<script>window.after = "ignore me"</script>
+</body></html>`
+
+	got := HTMLToText("https://mp.weixin.qq.com/s/example", html)
+	if !strings.Contains(got, "第一段 内容") || !strings.Contains(got, "第二段 & 更多") {
+		t.Fatalf("expected article text, got %q", got)
+	}
+	if strings.Contains(got, "window.cgiData") || strings.Contains(got, "ignore me") {
+		t.Fatalf("expected scripts to be ignored, got %q", got)
+	}
+}
+
+func TestFetchReadLimitUsesLargerWindowForWeChat(t *testing.T) {
+	if got := FetchReadLimit("https://mp.weixin.qq.com/s/example", 10000); got < 1<<20 {
+		t.Fatalf("FetchReadLimit for WeChat = %d, want a large bounded window", got)
+	}
+	if got := FetchReadLimit("https://example.com/post", 10000); got != 30000 {
+		t.Fatalf("FetchReadLimit for regular URL = %d, want 30000", got)
+	}
+}