hardening: jsonschema auto, fuzz additions, stricter schemas

Author: jaine

ao/access/process.lua

@@ -6,6 +6,7 @@ local ids = require("ao.shared.ids")
 local auth = require("ao.shared.auth")
 local idem = require("ao.shared.idempotency")
 local audit = require("ao.shared.audit")
+local schema = require("ao.shared.schema")
 local metrics = require("ao.shared.metrics")
 
 local handlers = {}
@@ -78,6 +79,8 @@ function handlers.GrantEntitlement(msg)
   if not ok_len_asset then return codec.error("INVALID_INPUT", err_asset, { field = "Asset" }) end
   local ok_len_policy, err_policy = validation.check_length(msg.Policy, 64, "Policy")
   if not ok_len_policy then return codec.error("INVALID_INPUT", err_policy, { field = "Policy" }) end
+  local ok_schema, schema_err = schema.validate("entitlement", { subject = msg.Subject, asset = msg.Asset, policy = msg.Policy })
+  if not ok_schema then return codec.error("INVALID_INPUT", "Policy failed schema", { errors = schema_err }) end
   local policy_size = validation.estimate_json_length(msg.Policy)
   local ok_size, err_size = validation.check_size(policy_size, MAX_POLICY_BYTES, "Policy")
   if not ok_size then return codec.error("INVALID_INPUT", err_size, { field = "Policy" }) end

ao/catalog/process.lua

@@ -145,11 +145,11 @@ function handlers.UpsertProduct(msg)
   if msg.Payload.sku ~= msg.Sku then
     return codec.error("INVALID_INPUT", "Payload sku must match Sku field", { field = "Sku" })
   end
-  local ok_schema, schema_err = schema.validate("product", msg.Payload)
-  if not ok_schema then return codec.error("INVALID_INPUT", "Payload failed schema", { errors = schema_err }) end
   local payload_len = validation.estimate_json_length(msg.Payload)
   local ok_size, err_size = validation.check_size(payload_len, MAX_PAYLOAD_BYTES, "Payload")
   if not ok_size then return codec.error("INVALID_INPUT", err_size, { field = "Payload" }) end
+  local ok_schema, schema_err = schema.validate("product", msg.Payload)
+  if not ok_schema then return codec.error("INVALID_INPUT", "Payload failed schema", { errors = schema_err }) end
   local key = ids.product_key(msg["Site-Id"], msg.Sku)
   state.products[key] = { payload = msg.Payload, version = msg.Version }
   audit.record("catalog", "UpsertProduct", msg, nil, { sku = msg.Sku })

ao/registry/process.lua

@@ -8,6 +8,7 @@ local idem = require("ao.shared.idempotency")
 local ids = require("ao.shared.ids")
 local audit = require("ao.shared.audit")
 local metrics = require("ao.shared.metrics")
+local schema = require("ao.shared.schema")
 
 local handlers = {}
 local allowed_actions = {
@@ -93,6 +94,8 @@ function handlers.RegisterSite(msg)
   if msg.Config ~= nil then
     local ok_type_cfg, err_type_cfg = validation.assert_type(msg.Config, "table", "Config")
     if not ok_type_cfg then return codec.error("INVALID_INPUT", err_type_cfg, { field = "Config" }) end
+    local ok_schema, schema_err = schema.validate("registryConfig", msg.Config)
+    if not ok_schema then return codec.error("INVALID_INPUT", "Config failed schema", { errors = schema_err }) end
   end
   local config_len = validation.estimate_json_length(config)
   local ok_size, err_size = validation.check_size(config_len, MAX_CONFIG_BYTES, "Config")

ao/shared/arweave.lua

@@ -19,6 +19,9 @@ local HTTP_SIGNER_HEADER = os.getenv("ARWEAVE_HTTP_SIGNER_HEADER") or "X-Arweave
 local HTTP_RETRIES = tonumber(os.getenv("ARWEAVE_HTTP_RETRIES") or "3")
 local HTTP_BACKOFF_MS = tonumber(os.getenv("ARWEAVE_HTTP_BACKOFF_MS") or "200")
 local MAX_MANIFEST_BYTES = tonumber(os.getenv("ARWEAVE_MAX_MANIFEST_BYTES") or "262144") -- 256 KiB
+local HTTP_MAX_BODY = tonumber(os.getenv("ARWEAVE_HTTP_MAX_BODY") or "1048576") -- 1 MiB
+local EXPECT_RESPONSE_HASH = os.getenv("ARWEAVE_EXPECT_RESPONSE_HASH")
+local FORCE_ERROR = os.getenv("ARWEAVE_FORCE_ERROR") == "1"
 
 local function next_tx()
   counter = counter + 1
@@ -206,7 +209,9 @@ if MODE == "http" then
     end
     local hash = sha256(serialized) or fallback_checksum(serialized)
     local httpStatus, response_path
-    if HTTP_REAL and ENDPOINT and has_curl() and not (os.getenv("ARWEAVE_HTTP_DRYRUN") == "1") then
+    if FORCE_ERROR then
+      httpStatus = 500
+    elseif HTTP_REAL and ENDPOINT and has_curl() and not (os.getenv("ARWEAVE_HTTP_DRYRUN") == "1") then
       if not signer_exists() then
         log_request(tx, {
           endpoint = ENDPOINT or "<missing-endpoint>",
@@ -233,12 +238,18 @@ if MODE == "http" then
         f:close()
         if #body == 0 then
           log_request(tx, { warning = "empty_response" })
+        elseif HTTP_MAX_BODY and #body > HTTP_MAX_BODY then
+          log_request(tx, { error = "response_too_large", size = #body })
+          return nil, "http_response_too_large"
         else
           local resp_hash = sha256(body)
           if not resp_hash then
             log_request(tx, { warning = "response_hash_failed" })
           else
             log_request(tx, { responseHash = resp_hash })
+            if EXPECT_RESPONSE_HASH and resp_hash ~= EXPECT_RESPONSE_HASH then
+              return nil, "response_hash_mismatch"
+            end
           end
         end
       end

ao/shared/schema.lua

@@ -3,7 +3,7 @@
 -- uses that; otherwise falls back to the embedded validator below.
 
 local Schema = {}
-local USE_PY = os.getenv("SCHEMA_VALIDATOR") == "python"
+local SCHEMA_MODE = os.getenv("SCHEMA_VALIDATOR") or "auto" -- auto|python|embedded
 
 -- Schemas embedded as Lua tables (converted from schemas/*.json)
 local SCHEMAS = {
@@ -55,9 +55,27 @@ local SCHEMAS = {
     type = "object",
     required = { "subject", "asset" },
     properties = {
-      subject = { type = "string" },
-      asset = { type = "string" },
-      policy = { type = "string" },
+      subject = { type = "string", minLength = 1, maxLength = 128 },
+      asset = { type = "string", minLength = 1, maxLength = 256 },
+      policy = { type = "string", minLength = 1, maxLength = 128 },
+    },
+  },
+  accessAsset = {
+    type = "object",
+    required = { "asset", "ref" },
+    properties = {
+      asset = { type = "string", minLength = 1, maxLength = 256 },
+      ref = { type = "string", minLength = 1, maxLength = 2048 },
+      visibility = { type = "string", enum = { "protected", "public" } }
+    },
+  },
+  registryConfig = {
+    type = "object",
+    required = {},
+    properties = {
+      version = { type = "string", minLength = 1, maxLength = 128 },
+      metadata = { type = "object" },
+      flags = { type = "object" },
     },
   },
 }
@@ -149,7 +167,7 @@ local function validate_against(schema, value, path, errors)
 end
 
 function Schema.validate(schema_name, value)
-  if USE_PY then
+  if SCHEMA_MODE ~= "embedded" then
     local ok, err = Schema.validate_python(schema_name, value)
     if ok ~= nil then return ok, err end -- nil means fallback to embedded
   end
@@ -167,6 +185,10 @@ end
 
 -- Python/jsonschema validator (optional). Returns nil if not usable.
 function Schema.validate_python(schema_name, value)
+  local has_py = os.execute("python3 -c \"import jsonschema\" >/dev/null 2>&1")
+  if has_py ~= true and has_py ~= 0 then
+    return nil, "python_jsonschema_missing"
+  end
   local schema_path = "schemas/" .. schema_name .. ".schema.json"
   local f = io.open(schema_path, "r")
   if not f then return nil, "schema_not_found" end
@@ -202,12 +224,15 @@ function Schema.validate_python(schema_name, value)
   end
   jf:write(json_encode(value))
   jf:close()
-  local cmd = string.format("python3 - <<'PY'\nimport json,sys\ntry:\n import jsonschema\nexcept ImportError:\n sys.exit(2)\nwith open(%q) as f: schema=json.load(f)\nwith open(%q) as f: inst=json.load(f)\ntry:\n jsonschema.validate(inst, schema)\n sys.exit(0)\nexcept jsonschema.ValidationError as e:\n print(e.message)\n sys.exit(1)\nPY", schema_path, tmp)
+  local cmd = string.format("python3 - <<'PY'\nimport json,sys,jsonschema\nwith open(%q) as f: schema=json.load(f)\nwith open(%q) as f: inst=json.load(f)\ntry:\n jsonschema.validate(inst, schema)\n sys.exit(0)\nexcept jsonschema.ValidationError:\n sys.exit(1)\nPY", schema_path, tmp)
   local ok = os.execute(cmd)
   os.remove(tmp)
-  if ok == 0 then return true end
-  if ok == 2 then return nil, "python_jsonschema_missing" end
-  return false, { "python_validator_failed" }
+  if ok == 0 or ok == true then return true end
+  -- If validation fails, treat as schema error; otherwise fallback
+  if ok == 256 or ok == false then
+    return false, { "python_validator_failed" }
+  end
+  return nil, "python_validator_unavailable"
 end
 
 return Schema

ao/site/process.lua

@@ -156,11 +156,11 @@ function handlers.PutDraft(msg)
   -- normalize content against schema expectations
   if not msg.Content.id then msg.Content.id = msg["Page-Id"] end
   if not msg.Content.blocks then msg.Content.blocks = {} end
-  local ok_schema, schema_err = schema.validate("page", msg.Content)
-  if not ok_schema then return codec.error("INVALID_INPUT", "Content failed schema", { errors = schema_err }) end
   local content_len = validation.estimate_json_length(msg.Content)
   local ok_size, err_size = validation.check_size(content_len, MAX_CONTENT_BYTES, "Content")
   if not ok_size then return codec.error("INVALID_INPUT", err_size, { field = "Content" }) end
+  local ok_schema, schema_err = schema.validate("page", msg.Content)
+  if not ok_schema then return codec.error("INVALID_INPUT", "Content failed schema", { errors = schema_err }) end
   local key = ids.page_key(msg["Site-Id"], msg["Page-Id"], "draft")
   state.drafts[key] = { content = msg.Content, updatedAt = os.date("!%Y-%m-%dT%H:%M:%SZ") }
   return codec.ok({ draftId = key })

arweave/snapshots/mock-tx-000001.json

@@ -1 +1 @@
-{"siteId":"site-1","version":"v2","pages":[{"content":{"id":"home","blocks":[],"title":"Hello"},"pageId":"home"}]}
+{"pages":[{"pageId":"home","content":{"blocks":[],"id":"home","title":"Hello"}}],"version":"v2","siteId":"site-1"}

arweave/snapshots/mock-tx-000002.json

@@ -1 +1 @@
-{"siteId":"site-2","version":"v1","pages":[{"content":{"id":"old","blocks":[],"title":"Old"},"pageId":"old"}]}
+{"pages":[{"pageId":"old","content":{"blocks":[],"id":"old","title":"Old"}}],"version":"v1","siteId":"site-2"}

scripts/verify/fuzz.lua

@@ -6,6 +6,7 @@ local catalog = require("ao.catalog.process")
 local site = require("ao.site.process")
 local ar = require("ao.shared.arweave")
 local audit = require("ao.shared.audit")
+local auth = require("ao.shared.auth")
 
 local function with_req(fields)
   fields["Request-Id"] = fields["Request-Id"] or tostring(math.random())
@@ -46,6 +47,55 @@ do
   end
 end
 
+-- Force Arweave HTTP error via env flag
+do
+  package.loaded["ao.shared.arweave"] = nil
+  os.setenv("ARWEAVE_MODE", "http")
+  os.setenv("ARWEAVE_HTTP_REAL", "1")
+  os.setenv("ARWEAVE_FORCE_ERROR", "1")
+  local ar2 = require("ao.shared.arweave")
+  local tx, err = ar2.put_snapshot({ dummy = "ok" })
+  if err ~= "http_error" then
+    error("expected http_error with force flag")
+  end
+  os.setenv("ARWEAVE_FORCE_ERROR", nil)
+  package.loaded["ao.shared.arweave"] = nil
+end
+
+-- Auth ed25519 verification round-trip
+do
+  -- generate keypair
+  os.execute("openssl genpkey -algorithm ed25519 -out /tmp/ao-ed.key >/dev/null 2>&1")
+  os.execute("openssl pkey -in /tmp/ao-ed.key -pubout -out /tmp/ao-ed.pub >/dev/null 2>&1")
+  local target = "PublishVersion|site-x|rid-x"
+  os.execute(string.format("printf %%s %q > /tmp/ao-msg", target))
+  os.execute("openssl pkeyutl -sign -inkey /tmp/ao-ed.key -rawin -in /tmp/ao-msg -out /tmp/ao-sig >/dev/null 2>&1")
+  local sig_hex = io.popen("xxd -p /tmp/ao-sig"):read("*l")
+  os.setenv("AUTH_SIGNATURE_TYPE", "ed25519")
+  os.setenv("AUTH_SIGNATURE_PUBLIC", "/tmp/ao-ed.pub")
+  os.setenv("AUTH_REQUIRE_SIGNATURE", "1")
+  package.loaded["ao.shared.auth"] = nil
+  local auth2 = require("ao.shared.auth")
+  local ok, err = auth2.require_signature({ Action = "PublishVersion", ["Site-Id"] = "site-x", ["Request-Id"] = "rid-x", Signature = sig_hex })
+  if not ok then error("ed25519 signature should verify: " .. tostring(err)) end
+  local ok2, err2 = auth2.require_signature({ Action = "PublishVersion", ["Site-Id"] = "site-x", ["Request-Id"] = "rid-x", Signature = "deadbeef" })
+  if ok2 then error("bad signature should fail") end
+  os.setenv("AUTH_REQUIRE_SIGNATURE", nil)
+  os.setenv("AUTH_SIGNATURE_TYPE", nil)
+  os.setenv("AUTH_SIGNATURE_PUBLIC", nil)
+  package.loaded["ao.shared.auth"] = nil
+end
+
+-- Concurrent publish/version set simulation
+do
+  local siteId = "conc-site"
+  local site = require("ao.site.process")
+  site.route(with_req({ Action = "PutDraft", ["Site-Id"] = siteId, ["Page-Id"] = "p1", Content = { title = "T" }, ["Actor-Role"] = "editor" }))
+  local ok1 = site.route(with_req({ Action = "PublishVersion", ["Site-Id"] = siteId, Version = "v1", ["Actor-Role"] = "publisher" }))
+  local conflict = site.route(with_req({ Action = "PublishVersion", ["Site-Id"] = siteId, Version = "v2", ExpectedVersion = "old", ["Actor-Role"] = "publisher" }))
+  if conflict.status ~= "ERROR" then error("Expected VERSION_CONFLICT on second publish") end
+end
+
 -- Audit rotation/prune: set tiny rotate and emit many records
 do
   os.setenv = os.setenv or function() end -- no-op if not available