From ed3bcba177bbf69cbf049370e75a52241c699136 Mon Sep 17 00:00:00 2001 From: Raymond Nicholas Date: Tue, 15 Sep 2026 12:23:32 +0100 Subject: [PATCH 01/10] refactor: pass router dependencies as a struct Tier 3's admin endpoints need store instances cryden.Engine keeps unexported, so they can only come from whoever built them. A Deps struct absorbs that without growing every call site positionally. Co-Authored-By: Claude Code --- httpapi/oauth_health_test.go | 2 +- httpapi/router.go | 25 ++++++++++++++++++++++--- main.go | 6 +++++- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/httpapi/oauth_health_test.go b/httpapi/oauth_health_test.go index f55e6b8..47f8306 100644 --- a/httpapi/oauth_health_test.go +++ b/httpapi/oauth_health_test.go @@ -193,7 +193,7 @@ func TestAdminOAuthHealthRouteIsGatedByRequireAdmin(t *testing.T) { t.Fatalf("login (regular user): %v", err) } - router := NewRouter(engine, nil, config.Config{}) + router := NewRouter(Deps{Engine: engine, Config: config.Config{}}) const path = "/v1/admin/oauth/health" call := func(token string) *httptest.ResponseRecorder { diff --git a/httpapi/router.go b/httpapi/router.go index aac5ccc..6a99819 100644 --- a/httpapi/router.go +++ b/httpapi/router.go @@ -9,14 +9,33 @@ import ( "github.com/crydensync/api/config" ) +// Deps is everything the route table needs to build its handlers. It is a +// struct rather than a parameter list because the admin surface keeps +// growing: every endpoint added under /v1/admin needs something the route +// table does not have today, and a struct absorbs that without every +// existing call site growing a positional argument. It also lets a test +// build a router with exactly the dependencies the endpoint under test +// needs and leave the rest nil. +type Deps struct { + Engine *cryden.Engine + + // DB is used only by the health handler, which pings it. Nil is fine + // for a router built without a database (tests); /v1/health then + // reports the database as unconfigured rather than panicking. + DB *sql.DB + Config config.Config +} + // NewRouter builds the full route table. Called once from main.go. -func NewRouter(engine *cryden.Engine, db *sql.DB, cfg config.Config) http.Handler { +func NewRouter(d Deps) http.Handler { + engine := d.Engine + auth := &AuthHandlers{Engine: engine} sessions := &SessionHandlers{Engine: engine} account := &AccountHandlers{Engine: engine} email := &EmailHandlers{Engine: engine} - health := &HealthHandler{DB: db} - oauth := &OAuthHandlers{Engine: engine, Config: cfg} + health := &HealthHandler{DB: d.DB} + oauth := &OAuthHandlers{Engine: engine, Config: d.Config} oauthHealth := NewOAuthHealthHandlers(oauth) totp := &TOTPHandlers{Engine: engine} passkeys := &PasskeyHandlers{Engine: engine} diff --git a/main.go b/main.go index 9cfba3e..58b8cc8 100644 --- a/main.go +++ b/main.go @@ -132,7 +132,11 @@ func main() { log.Fatalf("failed to construct cryden engine: %v", err) } - router := httpapi.NewRouter(engine, db, cfg) + router := httpapi.NewRouter(httpapi.Deps{ + Engine: engine, + DB: db, + Config: cfg, + }) limiter := httpapi.NewEdgeRateLimiter(cfg.EdgeRateLimit, cfg.EdgeRateLimitWindow) handler := httpapi.WithCORS(cfg.CORSOrigins, httpapi.WithEdgeRateLimit(limiter, router)) From 62aa746258b73b6e60fe5a36f48a16164dc8d934 Mon Sep 17 00:00:00 2001 From: Raymond Nicholas Date: Tue, 15 Sep 2026 12:27:47 +0100 Subject: [PATCH 02/10] feat: add Argon2id, cloud-logger and email-template config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PASSWORD_HASHER selects argon2id; ARGON2ID_* each override one field of cryden's defaults, since the engine treats a partial struct as a real configuration rather than defaults-plus-overrides. LOG_LEVEL goes through logger.ParseLevel so a typo fails startup instead of silently falling back to debug. EMAIL_TEMPLATE_DIR adds templates/, this repo's own copy — cryden owns no message copy on purpose. Co-Authored-By: Claude Code --- .env.example | 54 +++++++++ config/config.go | 214 ++++++++++++++++++++++++++++++++++++ config/config_test.go | 140 ++++++++++++++++++++++- email_sender.go | 47 +++++++- main.go | 40 ++++++- templates/templates.go | 170 ++++++++++++++++++++++++++++ templates/templates_test.go | 149 +++++++++++++++++++++++++ 7 files changed, 806 insertions(+), 8 deletions(-) create mode 100644 templates/templates.go create mode 100644 templates/templates_test.go diff --git a/.env.example b/.env.example index 2aa250e..8ed9c5d 100644 --- a/.env.example +++ b/.env.example @@ -66,3 +66,57 @@ CREDENTIAL_STUFFING_COOLDOWN_MINUTES= REDIS_URL= RATE_LIMIT_ATTEMPTS= RATE_LIMIT_WINDOW_SECONDS= + +# Password hashing. bcrypt is the engine's default; argon2id is the +# current recommendation for new deployments (memory-hard, and the knob a +# GPU attacker cannot parallelize around). Switching is safe at any time +# and needs no migration — existing bcrypt hashes keep verifying and are +# rewritten one successful login at a time, which is what +# GET /v1/admin/security/hash-migration reports on. +# +# The ARGON2ID_* vars default to RFC 9106's second recommended option +# (64 MiB, t=3, p=4) and only need setting to tune them to your hardware. +# Raise ARGON2ID_MEMORY_KIB before ARGON2ID_ITERATIONS if you have +# headroom: memory hardness is the whole reason to pick Argon2id. Note +# that each var replaces exactly one field — leaving the rest at cryden's +# defaults, never at zero. +PASSWORD_HASHER= +ARGON2ID_MEMORY_KIB= +ARGON2ID_ITERATIONS= +ARGON2ID_PARALLELISM= +ARGON2ID_SALT_LENGTH= +ARGON2ID_KEY_LENGTH= + +# The non-secret label every generated API key starts with ("ck_9f3a1c02…"). +# Set it to something recognisable as yours so a key leaked into a commit +# is greppable by your own secret scanners. +API_KEY_PREFIX= + +# Cloud logging — a second, redacted, filtered copy of the engine's log +# records, alongside the full-detail JSON line on stdout. Off unless +# CLOUD_LOGGING is set. LOG_LEVEL is the threshold the shipped copy +# drops below and must be debug/info/warn/error: an unrecognized value is +# a startup failure rather than a silent fallback, because defaulting a +# typo to debug multiplies a vendor bill and defaulting it to error +# throws away the records you were trying to keep. +# +# CLOUD_LOG_REDACTION is "mask" (value replaced with [redacted]) or +# "hash" (keyed HMAC digest, so the same address still reads as the same +# address across records — "one IP, forty accounts" is the shape +# credential stuffing has, and a mask destroys it). "hash" requires +# CLOUD_LOG_HASH_KEY, which must be identical on every replica and should +# be a value of its own rather than a reuse of JWT_SECRET. +CLOUD_LOGGING= +LOG_LEVEL= +CLOUD_LOG_REDACTION= +CLOUD_LOG_HASH_KEY= + +# A directory holding message templates, rendered instead of the console +# senders' built-in lines. Optional — unset keeps today's behaviour. +# Recognised files are verification.txt and magic_link.txt; supply either +# or both. Available fields: {{.To}}, {{.Token}}, {{.URL}} (empty for a +# verification message, and for a magic link when BASE_URL is unset). +# A directory that is set but holds neither file, or a file that does not +# parse, is a startup failure — a template directory that silently did +# nothing is worse than one that refused to start. +EMAIL_TEMPLATE_DIR= diff --git a/config/config.go b/config/config.go index 1eeecac..10ab761 100644 --- a/config/config.go +++ b/config/config.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/crydensync/cryden/v2/logger" "github.com/crydensync/cryden/v2/security" ) @@ -113,8 +114,99 @@ type Config struct { // to need company. RateLimitAttempts int RateLimitWindow time.Duration + + // PasswordHasher selects which algorithm NEW password hashes are + // written with — PasswordHasherBcrypt (the engine's default) or + // PasswordHasherArgon2id. Switching is safe at any time and needs no + // migration: cryden wraps whichever hasher it holds in a MultiHasher + // that picks the verifier from each stored hash's own format, so + // existing bcrypt hashes keep verifying and are rewritten one + // successful login at a time. That gradual rewrite is what + // GET /v1/admin/security/hash-migration reports on. + // + // An unrecognized value is a startup failure rather than a silent + // fall back to bcrypt: someone who typed "argon" meant to turn + // Argon2id on, and quietly leaving them on the weaker algorithm is + // the one failure mode this setting must not have. + PasswordHasher string + + // Argon2idParams is the cost configuration for that hasher. Like the + // anomaly thresholds above, it starts as cryden's own + // security.DefaultArgon2idParams and each ARGON2ID_* env var replaces + // only the field it names — cryden reads a partially-filled params + // struct as a real custom configuration used as-is, so a struct + // assembled from only the env vars that happened to be set would + // silently zero the rest and fail validation on a deployment that + // meant to change one knob. + // + // Populated whether or not PasswordHasher selects argon2id, so + // GET /v1/admin/security/hash-migration can report what the + // deployment would write without reconstructing it separately. + Argon2idParams security.Argon2idParams + + // APIKeyPrefix is the non-secret label every generated API key + // starts with, as in "ck_9f3a1c02...". The point of the convention is + // that a key leaked into a commit is greppable, so set it to + // something recognisable as yours. cryden rejects whitespace and + // underscores (the underscore separates the label from the secret), + // and ignores it entirely unless the API key store is wired — which + // main.go always does. + APIKeyPrefix string + + // LogLevel is the threshold the cloud sink drops records below, + // leaving the local copy untouched. Parsed by logger.ParseLevel, + // whose error is the point: a typo defaulted to debug quietly + // multiplies a vendor's bill. + LogLevel logger.Level + + // CloudLogging turns the cloud sink on. Off by default, and off means + // Config.Logger stays nil and the engine keeps its own console + // default — there is nothing to configure for a deployment that ships + // no logs anywhere. + CloudLogging bool + + // CloudLogRedaction picks how personal data is stripped from the copy + // leaving the building: CloudLogRedactionMask replaces it with a + // fixed marker, CloudLogRedactionHash replaces it with a keyed digest + // so the same address still reads as the same address across records + // — "one IP, forty accounts" is exactly the shape credential + // stuffing has, which a mask destroys. Hash mode needs + // CloudLogHashKey. + CloudLogRedaction string + + // CloudLogHashKey is the HMAC key for hash-mode redaction. It must be + // the same on every replica or one address hashes two ways and the + // correlation the mode exists for is gone — and it should be a value + // of its own rather than a reuse of JWT_SECRET or ENCRYPTION_KEY. + // cryden's NewHashingRedactor asks for that separation explicitly: + // this key is handed to the component whose entire job is to hand its + // output to a third party. + CloudLogHashKey string + + // EmailTemplateDir is a directory holding message templates this repo + // renders instead of its console senders' built-in lines. cryden + // deliberately owns no template configuration at all — a message body + // is a host app's copy, not the engine's — so this is entirely this + // repo's. Empty keeps the console senders' hard-coded text + // byte-for-byte; a dir that is set but unreadable or missing a + // template is a startup failure, the same class of typo as an + // unparseable REDIS_URL. + EmailTemplateDir string } +// PasswordHasher values. Bcrypt is the engine's own default and what an +// unset PASSWORD_HASHER leaves in place. +const ( + PasswordHasherBcrypt = "bcrypt" + PasswordHasherArgon2id = "argon2id" +) + +// CloudLogRedaction values, matching cryden's two Redactor constructors. +const ( + CloudLogRedactionMask = "mask" + CloudLogRedactionHash = "hash" +) + // Load reads .env (if present, filling only gaps — real env vars // always win) then reads the actual environment. No external // dependency for .env parsing — same minimal-loader approach as csax. @@ -265,9 +357,131 @@ func Load() (Config, error) { return cfg, err } + // Password hashing. Bcrypt is the engine's default, so the only thing + // this repo has to do for it is not pass a hasher — but the argon2id + // parameters are assembled either way, because the hash-migration + // report describes what the deployment is configured to write and + // should not have to rebuild that answer from a second place. + cfg.PasswordHasher = os.Getenv("PASSWORD_HASHER") + switch cfg.PasswordHasher { + case "": + cfg.PasswordHasher = PasswordHasherBcrypt + case PasswordHasherBcrypt, PasswordHasherArgon2id: + default: + return cfg, fmt.Errorf("PASSWORD_HASHER must be %q or %q, got %q", + PasswordHasherBcrypt, PasswordHasherArgon2id, cfg.PasswordHasher) + } + + // Defaults first, then one override per env var — see the field + // comment for why a partially-filled struct would be a real problem + // here rather than a harmless one. + cfg.Argon2idParams = security.DefaultArgon2idParams + if cfg.Argon2idParams.Memory, err = envUint32("ARGON2ID_MEMORY_KIB", cfg.Argon2idParams.Memory); err != nil { + return cfg, err + } + if cfg.Argon2idParams.Iterations, err = envUint32("ARGON2ID_ITERATIONS", cfg.Argon2idParams.Iterations); err != nil { + return cfg, err + } + if cfg.Argon2idParams.Parallelism, err = envUint8("ARGON2ID_PARALLELISM", cfg.Argon2idParams.Parallelism); err != nil { + return cfg, err + } + if cfg.Argon2idParams.SaltLength, err = envUint32("ARGON2ID_SALT_LENGTH", cfg.Argon2idParams.SaltLength); err != nil { + return cfg, err + } + if cfg.Argon2idParams.KeyLength, err = envUint32("ARGON2ID_KEY_LENGTH", cfg.Argon2idParams.KeyLength); err != nil { + return cfg, err + } + + // The engine applies "ck" as its own default, so writing it here too + // is not redundant: the hash-migration report and the console both + // want to show the prefix actually in force, and reading it back off + // a config struct is the only way to get that without duplicating + // cryden's default in a second place. + cfg.APIKeyPrefix = os.Getenv("API_KEY_PREFIX") + if cfg.APIKeyPrefix == "" { + cfg.APIKeyPrefix = "ck" + } + + // Cloud logging. LOG_LEVEL is parsed rather than defaulted on error: + // see ParseLevel's own doc comment — a typo silently filed at debug + // multiplies a vendor bill, and one silently filed at error throws + // away the records someone was trying to keep. + if cfg.LogLevel, err = logger.ParseLevel(envString("LOG_LEVEL", "info")); err != nil { + return cfg, err + } + if cfg.CloudLogging, err = envBool("CLOUD_LOGGING", false); err != nil { + return cfg, err + } + cfg.CloudLogRedaction = envString("CLOUD_LOG_REDACTION", CloudLogRedactionMask) + switch cfg.CloudLogRedaction { + case CloudLogRedactionMask, CloudLogRedactionHash: + default: + return cfg, fmt.Errorf("CLOUD_LOG_REDACTION must be %q or %q, got %q", + CloudLogRedactionMask, CloudLogRedactionHash, cfg.CloudLogRedaction) + } + cfg.CloudLogHashKey = os.Getenv("CLOUD_LOG_HASH_KEY") + // Required only when it would actually be used. Asking every + // deployment for a second secret it has no purpose for is how a + // required-when-unused setting ends up copy-pasted from JWT_SECRET, + // which is the specific thing the key separation exists to prevent. + if cfg.CloudLogRedaction == CloudLogRedactionHash && cfg.CloudLogHashKey == "" { + return cfg, fmt.Errorf("CLOUD_LOG_HASH_KEY is required when CLOUD_LOG_REDACTION is %q", CloudLogRedactionHash) + } + + // Email templates. Optional: unset keeps the console senders' own + // text. main.go is what reports a directory that is set but broken. + cfg.EmailTemplateDir = os.Getenv("EMAIL_TEMPLATE_DIR") + return cfg, nil } +// envString reads an optional string env var, falling back to def when it +// is unset or empty. An empty value counts as unset rather than as a +// setting of its own: every string knob here has a working default, so +// "set it to nothing" is never how a deployment means to say something. +func envString(name, def string) string { + if v := os.Getenv(name); v != "" { + return v + } + return def +} + +// envUint32 and envUint8 read an optional unsigned env var, falling back +// to def when it is unset or empty. They exist rather than an int-and-cast +// because Argon2id's cost parameters are unsigned all the way down: a +// negative value cast to uint8 does not fail, it wraps to 255 lanes, and +// the hasher would then take a deployment's typo as a configuration. So a +// minus sign is a startup failure here, which is the only place it can +// still be caught saying what it meant. +// +// The width is ParseUint's bitSize, which rejects an out-of-range value +// with its own "value out of range" — so the error names the variable and +// then says precisely what was wrong with it, without a second copy of +// each bound to keep in step. +func envUint32(name string, def uint32) (uint32, error) { + v := os.Getenv(name) + if v == "" { + return def, nil + } + n, err := strconv.ParseUint(v, 10, 32) + if err != nil { + return 0, fmt.Errorf("%s must be a non-negative whole number: %w", name, err) + } + return uint32(n), nil +} + +func envUint8(name string, def uint8) (uint8, error) { + v := os.Getenv(name) + if v == "" { + return def, nil + } + n, err := strconv.ParseUint(v, 10, 8) + if err != nil { + return 0, fmt.Errorf("%s must be a non-negative whole number: %w", name, err) + } + return uint8(n), nil +} + // envInt reads an optional integer env var, falling back to def when it // is unset or empty. func envInt(name string, def int) (int, error) { diff --git a/config/config_test.go b/config/config_test.go index 002aca2..607b59b 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -5,15 +5,16 @@ import ( "testing" "time" + "github.com/crydensync/cryden/v2/logger" "github.com/crydensync/cryden/v2/security" ) -// tier2EnvVars are the vars these tests assert on, cleared before every +// tieredEnvVars are the vars these tests assert on, cleared before every // case so a value left in the developer's shell cannot make a // default-value assertion pass or fail for the wrong reason. Setting one // to "" is the same as leaving it unset: every loader in this package // treats empty as absent. -var tier2EnvVars = []string{ +var tieredEnvVars = []string{ "ANOMALY_DETECTION", "ANOMALY_WINDOW_MINUTES", "ANOMALY_HISTORY_SIZE", @@ -27,6 +28,18 @@ var tier2EnvVars = []string{ "REDIS_URL", "RATE_LIMIT_ATTEMPTS", "RATE_LIMIT_WINDOW_SECONDS", + "PASSWORD_HASHER", + "ARGON2ID_MEMORY_KIB", + "ARGON2ID_ITERATIONS", + "ARGON2ID_PARALLELISM", + "ARGON2ID_SALT_LENGTH", + "ARGON2ID_KEY_LENGTH", + "API_KEY_PREFIX", + "LOG_LEVEL", + "CLOUD_LOGGING", + "CLOUD_LOG_REDACTION", + "CLOUD_LOG_HASH_KEY", + "EMAIL_TEMPLATE_DIR", } func loadForTest(t *testing.T, env map[string]string) (Config, error) { @@ -34,7 +47,7 @@ func loadForTest(t *testing.T, env map[string]string) (Config, error) { t.Setenv("DATABASE_URL", "postgres://user:pw@localhost/db") t.Setenv("JWT_SECRET", "test-secret") t.Setenv("CORS_ORIGINS", "http://localhost:5173") - for _, name := range tier2EnvVars { + for _, name := range tieredEnvVars { t.Setenv(name, "") } for name, value := range env { @@ -137,6 +150,12 @@ func TestTier2MalformedValuesAreStartupErrors(t *testing.T) { {"non-numeric minutes", map[string]string{"ANOMALY_WINDOW_MINUTES": "soon"}, "ANOMALY_WINDOW_MINUTES must be a number of minutes"}, {"non-numeric seconds", map[string]string{"RATE_LIMIT_WINDOW_SECONDS": "1.5"}, "RATE_LIMIT_WINDOW_SECONDS must be a number of seconds"}, {"non-numeric count", map[string]string{"CREDENTIAL_STUFFING_TARGET_ACCOUNTS": "many"}, "CREDENTIAL_STUFFING_TARGET_ACCOUNTS must be a number"}, + {"unknown hasher", map[string]string{"PASSWORD_HASHER": "argon"}, "PASSWORD_HASHER must be"}, + {"negative argon2id cost", map[string]string{"ARGON2ID_PARALLELISM": "-1"}, "ARGON2ID_PARALLELISM must be a non-negative whole number"}, + {"oversized argon2id lane count", map[string]string{"ARGON2ID_PARALLELISM": "256"}, "ARGON2ID_PARALLELISM must be a non-negative whole number"}, + {"unknown log level", map[string]string{"LOG_LEVEL": "verbose"}, "unrecognized level name"}, + {"unknown redaction mode", map[string]string{"CLOUD_LOG_REDACTION": "encrypt"}, "CLOUD_LOG_REDACTION must be"}, + {"hash redaction without a key", map[string]string{"CLOUD_LOG_REDACTION": "hash"}, "CLOUD_LOG_HASH_KEY is required"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -150,3 +169,118 @@ func TestTier2MalformedValuesAreStartupErrors(t *testing.T) { }) } } + +// The Tier 3 defaults, all of which have to be the engine's own answers +// rather than this repo's guesses: bcrypt is what cryden runs with no +// hasher set, and the argon2id parameters are RFC 9106's option two. +func TestTier3DefaultsComeFromTheEngine(t *testing.T) { + cfg, err := loadForTest(t, nil) + if err != nil { + t.Fatalf("Load() failed with only the required vars set: %v", err) + } + + if cfg.PasswordHasher != PasswordHasherBcrypt { + t.Errorf("PasswordHasher = %q, want %q", cfg.PasswordHasher, PasswordHasherBcrypt) + } + // Assembled even when bcrypt is in force, so the hash-migration + // report can say what this deployment WOULD write. + if cfg.Argon2idParams != security.DefaultArgon2idParams { + t.Errorf("Argon2idParams = %+v, want the engine's defaults %+v", cfg.Argon2idParams, security.DefaultArgon2idParams) + } + if cfg.APIKeyPrefix != "ck" { + t.Errorf("APIKeyPrefix = %q, want the engine's own default \"ck\"", cfg.APIKeyPrefix) + } + if cfg.CloudLogging { + t.Error("cloud logging is on without CLOUD_LOGGING being set") + } + if cfg.LogLevel != logger.LevelInfo { + t.Errorf("LogLevel = %s, want info", cfg.LogLevel) + } + if cfg.CloudLogRedaction != CloudLogRedactionMask { + t.Errorf("CloudLogRedaction = %q, want %q", cfg.CloudLogRedaction, CloudLogRedactionMask) + } + if cfg.EmailTemplateDir != "" { + t.Errorf("EmailTemplateDir = %q, want empty (built-in sender text)", cfg.EmailTemplateDir) + } +} + +// One env var must move exactly one field. This is the same trap the +// anomaly thresholds have: cryden treats a partially-filled params +// struct as a complete custom configuration, so an assembly that +// started from zero would set four knobs to zero and fail validation. +func TestTier3Argon2idOverrideLeavesEveryOtherKnobDefaulted(t *testing.T) { + cfg, err := loadForTest(t, map[string]string{ + "PASSWORD_HASHER": PasswordHasherArgon2id, + "ARGON2ID_ITERATIONS": "5", + }) + if err != nil { + t.Fatalf("Load() failed: %v", err) + } + + if cfg.PasswordHasher != PasswordHasherArgon2id { + t.Errorf("PasswordHasher = %q, want %q", cfg.PasswordHasher, PasswordHasherArgon2id) + } + if cfg.Argon2idParams.Iterations != 5 { + t.Errorf("Iterations = %d, want 5", cfg.Argon2idParams.Iterations) + } + if cfg.Argon2idParams.Memory != security.DefaultArgon2idParams.Memory { + t.Errorf("Memory = %d, want the engine default %d", cfg.Argon2idParams.Memory, security.DefaultArgon2idParams.Memory) + } + if cfg.Argon2idParams.Parallelism != security.DefaultArgon2idParams.Parallelism { + t.Errorf("Parallelism = %d, want the engine default %d", cfg.Argon2idParams.Parallelism, security.DefaultArgon2idParams.Parallelism) + } + if cfg.Argon2idParams.SaltLength != security.DefaultArgon2idParams.SaltLength { + t.Errorf("SaltLength = %d, want the engine default %d", cfg.Argon2idParams.SaltLength, security.DefaultArgon2idParams.SaltLength) + } + if cfg.Argon2idParams.KeyLength != security.DefaultArgon2idParams.KeyLength { + t.Errorf("KeyLength = %d, want the engine default %d", cfg.Argon2idParams.KeyLength, security.DefaultArgon2idParams.KeyLength) + } + // And the assembled set is one cryden will actually accept — the + // assertion the field-by-field checks above cannot make between them. + if _, err := security.NewArgon2idHasher(cfg.Argon2idParams); err != nil { + t.Errorf("the assembled params were rejected by the engine: %v", err) + } +} + +// A wrong LOG_LEVEL must not fall back to anything. ParseLevel's own +// doc comment is explicit that both fallbacks are wrong in one +// direction: debug multiplies a vendor bill, error discards records +// someone was trying to keep. +func TestTier3LogLevelIsParsedNotDefaulted(t *testing.T) { + cfg, err := loadForTest(t, map[string]string{"LOG_LEVEL": "WARN"}) + if err != nil { + t.Fatalf("Load() failed: %v", err) + } + if cfg.LogLevel != logger.LevelWarn { + t.Errorf("LogLevel = %s, want warn (case-insensitive)", cfg.LogLevel) + } + + if _, err := loadForTest(t, map[string]string{"LOG_LEVEL": "verbose"}); err == nil { + t.Error("an unknown LOG_LEVEL was accepted, want a startup failure") + } +} + +// The hash key is required only when something would use it — asking +// every deployment for a secret it has no purpose for is how a second +// required key ends up copy-pasted from JWT_SECRET, which is the exact +// reuse cryden's NewHashingRedactor warns against. +func TestTier3CloudLogHashKeyIsRequiredOnlyForHashRedaction(t *testing.T) { + cfg, err := loadForTest(t, map[string]string{"CLOUD_LOG_REDACTION": CloudLogRedactionMask}) + if err != nil { + t.Fatalf("mask redaction without a hash key failed: %v", err) + } + if cfg.CloudLogHashKey != "" { + t.Errorf("CloudLogHashKey = %q, want empty", cfg.CloudLogHashKey) + } + + cfg, err = loadForTest(t, map[string]string{ + "CLOUD_LOG_REDACTION": CloudLogRedactionHash, + "CLOUD_LOG_HASH_KEY": "a-key-of-its-own", + }) + if err != nil { + t.Fatalf("hash redaction with a key failed: %v", err) + } + if cfg.CloudLogHashKey != "a-key-of-its-own" { + t.Errorf("CloudLogHashKey = %q, want the value that was set", cfg.CloudLogHashKey) + } +} diff --git a/email_sender.go b/email_sender.go index d3b9607..a23cc6a 100644 --- a/email_sender.go +++ b/email_sender.go @@ -4,14 +4,35 @@ import ( "context" "log" "net/url" + + "github.com/crydensync/api/templates" ) // consoleEmailSender is a dev stand-in implementing notify.EmailSender. // Real deployments must replace this with a real provider (SES, // SendGrid, Resend, Postmark) — see the api README. -type consoleEmailSender struct{} +// +// Templates is optional. Nil means no EMAIL_TEMPLATE_DIR was configured, +// and the built-in line below is printed exactly as it always was; a +// non-nil set renders the operator's own copy instead. The two share the +// "[EMAIL]" prefix either way, so tailing a log for one still finds the +// other. +type consoleEmailSender struct { + Templates *templates.Set +} func (s *consoleEmailSender) SendVerification(ctx context.Context, to string, rawToken string) error { + // The URL is deliberately empty: cryden hands over a token and no + // notion of where a browser should take it, and this repo owns no + // verification landing page to name. + body, ok, err := s.Templates.Verification(templates.Data{To: to, Token: rawToken}) + if err != nil { + return err + } + if ok { + log.Printf("[EMAIL] verification message for %s:\n%s", to, body) + return nil + } log.Printf("[EMAIL] Verification token for %s: %s", to, rawToken) return nil } @@ -31,13 +52,33 @@ type consoleMagicLinkSender struct { // /v1/magic-link/complete; the path below is a dev-time guess, not a // contract this repo owns. BaseURL string + // Templates is consoleEmailSender.Templates' counterpart, with the + // same nil-means-built-in behaviour. + Templates *templates.Set } func (s *consoleMagicLinkSender) SendMagicLink(ctx context.Context, to string, rawToken string) error { - if s.BaseURL == "" { + // The URL is assembled here rather than in the template on purpose: + // it is a fact about this deployment's routing, which the template + // author should not have to reconstruct from a token. + link := "" + if s.BaseURL != "" { + link = s.BaseURL + "/magic-link?token=" + url.QueryEscape(rawToken) + } + + body, ok, err := s.Templates.MagicLink(templates.Data{To: to, Token: rawToken, URL: link}) + if err != nil { + return err + } + if ok { + log.Printf("[MAGIC LINK] message for %s:\n%s", to, body) + return nil + } + + if link == "" { log.Printf("[MAGIC LINK] Token for %s: %s (BASE_URL unset, no clickable link)", to, rawToken) return nil } - log.Printf("[MAGIC LINK] For %s: %s/magic-link?token=%s", to, s.BaseURL, url.QueryEscape(rawToken)) + log.Printf("[MAGIC LINK] For %s: %s", to, link) return nil } diff --git a/main.go b/main.go index 58b8cc8..feda995 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ import ( "github.com/crydensync/api/config" "github.com/crydensync/api/httpapi" "github.com/crydensync/api/operator" + "github.com/crydensync/api/templates" ) func main() { @@ -36,17 +37,36 @@ func main() { operators := operator.NewStore(db) + // Email templates are optional and entirely this repo's: cryden owns + // no message copy. An unset EMAIL_TEMPLATE_DIR leaves both senders + // printing their own built-in line, byte for byte as before. + var emailTemplates *templates.Set + if cfg.EmailTemplateDir != "" { + emailTemplates, err = templates.Load(cfg.EmailTemplateDir) + if err != nil { + log.Fatalf("invalid EMAIL_TEMPLATE_DIR: %v", err) + } + log.Printf("email templates loaded from %s", cfg.EmailTemplateDir) + } + engineCfg := cryden.Config{ JWTSecret: cfg.JWTSecret, Users: postgres.NewUserStore(db), Sessions: postgres.NewSessionStore(db), Audit: postgres.NewAuditStore(db), Verifications: postgres.NewVerificationStore(db), - EmailSender: &consoleEmailSender{}, // dev stand-in — see email_sender.go - MagicLinkSender: &consoleMagicLinkSender{BaseURL: cfg.BaseURL}, // dev stand-in — see email_sender.go + EmailSender: &consoleEmailSender{Templates: emailTemplates}, // dev stand-in — see email_sender.go + MagicLinkSender: &consoleMagicLinkSender{BaseURL: cfg.BaseURL, Templates: emailTemplates}, // dev stand-in — see email_sender.go AccessTokenTTL: cfg.AccessTokenTTL, OAuth: postgres.NewOAuthStore(db), + // API keys are always wired: a machine credential is part of the + // API surface this repo offers, not a second factor a deployment + // opts into. The prefix is the non-secret label that makes a key + // leaked into a commit greppable. + APIKeys: postgres.NewAPIKeyStore(db), + APIKeyPrefix: cfg.APIKeyPrefix, + // Attaches a "role" claim for console operators only — an // ordinary end user's token gets no extra claims at all, not // even role="user". See operator/store.go for why this is a @@ -63,6 +83,22 @@ func main() { }), } + // Password hashing. Leaving Hasher unset is what selects bcrypt — the + // engine builds its own default from BcryptCost. Selecting argon2id + // here is the whole switch: cryden wraps whichever hasher it holds in + // a MultiHasher that reads the verifier off each stored hash, so + // changing this rewrites hashes one successful login at a time and + // never invalidates a credential. + if cfg.PasswordHasher == config.PasswordHasherArgon2id { + hasher, err := security.NewArgon2idHasher(cfg.Argon2idParams) + if err != nil { + log.Fatalf("invalid Argon2id parameters: %v", err) + } + engineCfg.Hasher = hasher + log.Printf("new password hashes are written with Argon2id (memory %d KiB, iterations %d, parallelism %d)", + cfg.Argon2idParams.Memory, cfg.Argon2idParams.Iterations, cfg.Argon2idParams.Parallelism) + } + // Second factors are all-or-nothing on ENCRYPTION_KEY: cryden refuses // to build an engine with a TOTP or WebAuthn store set and no // encryption key, and a half-configured deployment would be worse than diff --git a/templates/templates.go b/templates/templates.go new file mode 100644 index 0000000..07124dd --- /dev/null +++ b/templates/templates.go @@ -0,0 +1,170 @@ +// Package templates renders the message bodies this api sends, from +// plain-text template files on disk. +// +// It exists because cryden owns no template configuration at all, on +// purpose: a message body is a host app's copy — its wording, its brand, +// its legal boilerplate — and an auth engine that shipped default text +// would be putting words in its host's mouth. So the engine hands over a +// recipient and a raw token and nothing else, and what those become is +// decided here. +// +// Templates are deliberately plain text/template over .txt files rather +// than HTML. The senders this package feeds are console stand-ins (see +// email_sender.go in the repo root) whose entire purpose is to show an +// operator what a real provider would send, and a template language with +// no auto-escaping into a message a human reads in a log is one less way +// for the two to disagree. +package templates + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "text/template" +) + +// File names Load looks for, relative to the directory it is given. The +// two are separate files rather than one file with two blocks because +// they are different messages to a different purpose: "confirm this +// address is yours" and "click here to log in" have different copy, and +// cryden keeps their senders apart for the same reason (see its +// notify/magic_link_sender.go). +const ( + VerificationFile = "verification.txt" + MagicLinkFile = "magic_link.txt" +) + +// Data is what every template is rendered with. One struct for both +// messages rather than a type per message: the fields are the same three +// things in both cases, and a second type would be a second thing for a +// template author to look up. +type Data struct { + // To is the recipient's email address as the engine gave it. + To string + + // Token is the raw, single-use token. It is the entire secret of the + // message — anyone holding it can complete the action it names — so a + // template that logs it somewhere of its own is a template that + // leaked. + Token string + + // URL is a ready-to-click link, already escaped, for the messages + // that have one. Empty for a verification message (the token is the + // payload there, and this repo owns no verification landing page) and + // empty for a magic link on a deployment with no BASE_URL set. A + // template that wants to render it unconditionally should expect the + // empty string rather than a missing field. + URL string +} + +// Set is the loaded templates. A nil *Set means "no template directory +// configured", which every method here handles — so a caller can hold a +// nil Set and pass it straight through to its senders without a branch +// at each call site. A non-nil Set always has at least one template; see +// Load. +type Set struct { + verification *template.Template + magicLink *template.Template +} + +// Load reads a template directory. A directory holding neither file is +// an error: pointing this deployment at a directory of templates and +// getting today's built-in text instead is the kind of setting that +// looks applied and isn't. +// +// One file is enough, though — each message falls back independently, so +// a deployment that wants its own magic-link copy without touching the +// verification one supplies only that file. A file that is present but +// does not parse is always an error, never a fallback: it is a typo in +// copy someone wrote on purpose, and silently sending them the old text +// would hide it until a user complained. +func Load(dir string) (*Set, error) { + set := &Set{} + var found int + + verification, err := loadOne(dir, VerificationFile) + if err != nil { + return nil, err + } + if verification != nil { + set.verification = verification + found++ + } + + magicLink, err := loadOne(dir, MagicLinkFile) + if err != nil { + return nil, err + } + if magicLink != nil { + set.magicLink = magicLink + found++ + } + + if found == 0 { + return nil, fmt.Errorf("no templates found in %s — expected %s and/or %s", + dir, VerificationFile, MagicLinkFile) + } + return set, nil +} + +// Verification renders the verification-email body. ok is false when no +// verification template is configured, which is the caller's cue to use +// its own built-in text rather than to treat an empty body as a message. +func (s *Set) Verification(data Data) (body string, ok bool, err error) { + if s == nil || s.verification == nil { + return "", false, nil + } + body, err = render(s.verification, data) + if err != nil { + return "", false, err + } + return body, true, nil +} + +// MagicLink is Verification's counterpart for the magic-link message, +// with the same ok meaning. +func (s *Set) MagicLink(data Data) (body string, ok bool, err error) { + if s == nil || s.magicLink == nil { + return "", false, nil + } + body, err = render(s.magicLink, data) + if err != nil { + return "", false, err + } + return body, true, nil +} + +// loadOne reads and parses one template file, returning (nil, nil) when +// the file simply does not exist. That is the only "absent" case +// tolerated: a file that is there but unreadable — a permission problem, +// a directory where a file was expected — comes back as an error, since +// it is a deployment that meant to configure this and didn't. +func loadOne(dir, name string) (*template.Template, error) { + path := filepath.Join(dir, name) + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("reading template %s: %w", path, err) + } + tmpl, err := template.New(name).Parse(string(raw)) + if err != nil { + return nil, fmt.Errorf("parsing template %s: %w", path, err) + } + return tmpl, nil +} + +// render executes one template and trims the trailing newline most +// editors leave on a file. Without that trim, every rendered body ends +// in a blank line that only shows up in the message a user actually +// receives — the classic thing that is nobody's bug until it is +// everybody's. +func render(tmpl *template.Template, data Data) (string, error) { + var buf strings.Builder + if err := tmpl.Execute(&buf, data); err != nil { + return "", fmt.Errorf("rendering template %s: %w", tmpl.Name(), err) + } + return strings.TrimRight(buf.String(), "\n"), nil +} diff --git a/templates/templates_test.go b/templates/templates_test.go new file mode 100644 index 0000000..a4511d5 --- /dev/null +++ b/templates/templates_test.go @@ -0,0 +1,149 @@ +package templates + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// writeTemplate drops one template file into dir, creating dir if the +// caller passed the result of t.TempDir() straight through. +func writeTemplate(t *testing.T, dir, name, body string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { + t.Fatalf("writing %s: %v", name, err) + } +} + +func TestSetRendersConfiguredTemplates(t *testing.T) { + dir := t.TempDir() + writeTemplate(t, dir, VerificationFile, "Confirm {{.To}}: {{.Token}}\n") + writeTemplate(t, dir, MagicLinkFile, "Log in as {{.To}} at {{.URL}}\n") + + set, err := Load(dir) + if err != nil { + t.Fatalf("Load(%s): %v", dir, err) + } + + body, ok, err := set.Verification(Data{To: "user@example.com", Token: "tok_123"}) + if err != nil { + t.Fatalf("Verification: %v", err) + } + if !ok { + t.Fatal("Verification reported no template, want the one that was configured") + } + if want := "Confirm user@example.com: tok_123"; body != want { + t.Errorf("Verification body = %q, want %q", body, want) + } + + body, ok, err = set.MagicLink(Data{To: "user@example.com", Token: "tok_123", URL: "https://app.example.com/magic-link?token=tok_123"}) + if err != nil { + t.Fatalf("MagicLink: %v", err) + } + if !ok { + t.Fatal("MagicLink reported no template, want the one that was configured") + } + if want := "Log in as user@example.com at https://app.example.com/magic-link?token=tok_123"; body != want { + t.Errorf("MagicLink body = %q, want %q", body, want) + } +} + +// The trailing newline most editors leave on a file is trimmed, because +// the alternative is a blank line at the end of every message a user +// receives — invisible in the file, obvious in the inbox. +func TestRenderedBodyHasNoTrailingNewline(t *testing.T) { + dir := t.TempDir() + writeTemplate(t, dir, VerificationFile, "Confirm {{.To}}\n\n") + + set, err := Load(dir) + if err != nil { + t.Fatalf("Load(%s): %v", dir, err) + } + body, _, err := set.Verification(Data{To: "user@example.com"}) + if err != nil { + t.Fatalf("Verification: %v", err) + } + if body != "Confirm user@example.com" { + t.Errorf("body = %q, want no trailing newline", body) + } +} + +// A nil Set is the "no EMAIL_TEMPLATE_DIR configured" case, and every +// render must answer ok=false rather than an empty body: the senders +// distinguish those two, and an empty body that looked like a message +// would be a blank email. +func TestNilSetRendersNothing(t *testing.T) { + var set *Set + + if body, ok, err := set.Verification(Data{To: "user@example.com"}); ok || err != nil || body != "" { + t.Errorf("Verification on a nil Set = (%q, %v, %v), want (\"\", false, nil)", body, ok, err) + } + if body, ok, err := set.MagicLink(Data{To: "user@example.com"}); ok || err != nil || body != "" { + t.Errorf("MagicLink on a nil Set = (%q, %v, %v), want (\"\", false, nil)", body, ok, err) + } +} + +// Each message falls back on its own, so a deployment that only wants +// its own magic-link copy supplies only that file. +func TestOneTemplateIsEnough(t *testing.T) { + dir := t.TempDir() + writeTemplate(t, dir, MagicLinkFile, "Log in: {{.URL}}\n") + + set, err := Load(dir) + if err != nil { + t.Fatalf("Load(%s): %v", dir, err) + } + if _, ok, _ := set.Verification(Data{}); ok { + t.Error("Verification reported a template when none was configured") + } + if _, ok, err := set.MagicLink(Data{URL: "https://app.example.com/magic-link?token=x"}); err != nil || !ok { + t.Errorf("MagicLink = (ok %v, err %v), want the configured template", ok, err) + } +} + +// The two ways a configured directory can be wrong. Both are startup +// failures rather than silent fallbacks: a directory somebody pointed +// this deployment at and got the old built-in text from is a setting +// that looks applied and isn't. +func TestLoadRejectsABrokenDirectory(t *testing.T) { + t.Run("no templates at all", func(t *testing.T) { + _, err := Load(t.TempDir()) + if err == nil { + t.Fatal("an empty directory was accepted, want an error") + } + // The message names what was looked for, since the usual cause is + // a file spelled differently. + for _, name := range []string{VerificationFile, MagicLinkFile} { + if !strings.Contains(err.Error(), name) { + t.Errorf("error = %q, want it to name %s", err, name) + } + } + }) + + t.Run("unparseable template", func(t *testing.T) { + dir := t.TempDir() + writeTemplate(t, dir, VerificationFile, "Confirm {{.To") + _, err := Load(dir) + if err == nil { + t.Fatal("a template with an unclosed action was accepted, want an error") + } + if !strings.Contains(err.Error(), VerificationFile) { + t.Errorf("error = %q, want it to name %s", err, VerificationFile) + } + }) + + t.Run("field that does not exist", func(t *testing.T) { + // Parsing succeeds on this one — it is Execute that fails, so it + // exercises the render path rather than Load. + dir := t.TempDir() + writeTemplate(t, dir, VerificationFile, "Confirm {{.Tokne}}") + set, err := Load(dir) + if err != nil { + t.Fatalf("Load(%s): %v", dir, err) + } + if _, _, err := set.Verification(Data{To: "user@example.com", Token: "tok"}); err == nil { + t.Error("a misspelled field rendered without error, want the engine to report it") + } + }) +} From 54cf819ae360c6dc55beccbf1ba4b9785de9ea7b Mon Sep 17 00:00:00 2001 From: Raymond Nicholas Date: Tue, 15 Sep 2026 12:30:59 +0100 Subject: [PATCH 03/10] feat: add API key endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST/GET /v1/api-keys and DELETE /v1/api-keys/{keyID}, all behind RequireAuth. cryden derives the user ID from the verified token and scopes every store call by it, so a caller can never read or revoke another account's key. The raw key is returned exactly once, with a notice saying so — cryden stores only its hash. expires_in_days is bounded so a large value cannot overflow the duration into a negative TTL. Co-Authored-By: Claude Code --- httpapi/apikey_handlers.go | 174 ++++++++++++++++++ httpapi/apikey_handlers_test.go | 296 +++++++++++++++++++++++++++++++ httpapi/errors.go | 23 ++- httpapi/response.go | 24 +++ httpapi/router.go | 14 ++ httpapi/session_handlers_test.go | 6 + 6 files changed, 536 insertions(+), 1 deletion(-) create mode 100644 httpapi/apikey_handlers.go create mode 100644 httpapi/apikey_handlers_test.go diff --git a/httpapi/apikey_handlers.go b/httpapi/apikey_handlers.go new file mode 100644 index 0000000..35e9047 --- /dev/null +++ b/httpapi/apikey_handlers.go @@ -0,0 +1,174 @@ +package httpapi + +import ( + "net/http" + "strings" + "time" + + "github.com/crydensync/cryden/v2" +) + +type APIKeyHandlers struct { + Engine *cryden.Engine +} + +// apiKeyNotice is echoed in the creation response because the raw key +// below exists in exactly one place — that response. cryden stores only +// its hash (token.HashToken) and can never reproduce it, so a caller that +// loses it has to mint a new one. +const apiKeyNotice = "this key is shown only once and cannot be retrieved later — store it somewhere safe now" + +// apiKeyMaxExpiryDays bounds the lifetime a caller may request. Not a +// security limit but a parsing one: expires_in_days is multiplied by +// 24h, and an unbounded int from a request body could overflow that +// multiplication into a NEGATIVE duration, which cryden would then +// reject as an invalid TTL — a confusing 400 for what looks like a valid +// request. Ten years is past any rotation policy and far from the +// overflow point. +const apiKeyMaxExpiryDays = 3650 + +// apiKeyMaxNameLength bounds the label. It is presentational and cryden +// does not require it to be unique or non-empty, so the only reason to +// cap it is the column it lands in and the list it is rendered in. +const apiKeyMaxNameLength = 100 + +// apiKeyDTO is the public shape of one key. KeyHash has no field here at +// all — cryden's cryden.APIKey never carries it (see publicAPIKey), and +// this type is the second layer of the same guarantee rather than a +// duplicate of it: a field that does not exist cannot be marshalled by +// accident later. +type apiKeyDTO struct { + ID string `json:"id"` + Name string `json:"name"` + Prefix string `json:"prefix"` + + // Scopes is always an array, never null, so a client can iterate it + // without a nil check. + Scopes []string `json:"scopes"` + + // ExpiresAt is null for a key that never expires, which is the default. + ExpiresAt *string `json:"expires_at"` + + // Expired is the server's own answer to "is this still usable". + // Deliberately sent rather than left to the client to compute from + // ExpiresAt: it is the same comparison cryden makes when it + // authenticates the key, so the two can never disagree. + Expired bool `json:"expired"` + + CreatedAt string `json:"created_at"` + + // LastUsedAt is null until the key is first used. It is written at + // most once every five minutes (see cryden's apiKeyLastUsedGranularity), + // so it answers "is anything still using this?" and not "when exactly + // was the last request". + LastUsedAt *string `json:"last_used_at"` +} + +func toAPIKeyDTO(k cryden.APIKey) apiKeyDTO { + scopes := k.Scopes + if scopes == nil { + scopes = []string{} + } + return apiKeyDTO{ + ID: k.ID, + Name: k.Name, + Prefix: k.Prefix, + Scopes: scopes, + ExpiresAt: formatTimePtr(k.ExpiresAt), + Expired: k.Expired(), + CreatedAt: formatTime(k.CreatedAt), + LastUsedAt: formatTimePtr(k.LastUsedAt), + } +} + +// Create — auth required. Mints a machine-to-machine credential for the +// calling user and returns the raw key exactly once, alongside the stored +// record and a notice saying so. Same one-time-display contract as +// POST /v1/recovery-codes/generate, and the notice is the same device: a +// client that renders the key without it is the failure mode being +// guarded against. +// +// expires_in_days of 0 or absent means the key never expires. That is +// cryden's own default and the honest one for a credential living in a +// deploy pipeline's environment — revocation, not expiry, is what +// actually stops a key. +func (h *APIKeyHandlers) Create(w http.ResponseWriter, r *http.Request) { + userID := UserIDFromContext(r) + + var req struct { + Name string `json:"name"` + Scopes []string `json:"scopes"` + ExpiresInDays int `json:"expires_in_days"` + } + if err := decodeJSON(r, &req); err != nil { + writeBadRequest(w, "invalid request body") + return + } + + name := strings.TrimSpace(req.Name) + if len(name) > apiKeyMaxNameLength { + writeBadRequest(w, "name must be 100 characters or fewer") + return + } + if req.ExpiresInDays < 0 || req.ExpiresInDays > apiKeyMaxExpiryDays { + writeBadRequest(w, "expires_in_days must be between 0 (never expires) and 3650") + return + } + + var ttl time.Duration + if req.ExpiresInDays > 0 { + ttl = time.Duration(req.ExpiresInDays) * 24 * time.Hour + } + + rawKey, key, err := cryden.GenerateAPIKey(r.Context(), h.Engine, userID, name, req.Scopes, ttl) + if err != nil { + writeErr(w, err) + return + } + + writeData(w, http.StatusCreated, map[string]any{ + "key": rawKey, + "notice": apiKeyNotice, + "api_key": toAPIKeyDTO(key), + }) +} + +// List — auth required. Returns the calling user's live keys, newest +// first. Revoked keys are absent; expired-but-unrevoked ones are present +// and marked, because "your CI key expired on Tuesday" is exactly what +// someone needs to see to understand why a pipeline broke, whereas a +// revoked key has already been dealt with. +func (h *APIKeyHandlers) List(w http.ResponseWriter, r *http.Request) { + userID := UserIDFromContext(r) + keys, err := cryden.ListAPIKeys(r.Context(), h.Engine, userID) + if err != nil { + writeErr(w, err) + return + } + + out := make([]apiKeyDTO, 0, len(keys)) + for _, k := range keys { + out = append(out, toAPIKeyDTO(k)) + } + writeData(w, http.StatusOK, out) +} + +// Revoke — auth required. keyID comes from the URL path, wired in +// router.go. Ownership is enforced inside the store statement cryden +// calls (auth.RevokeAPIKey → APIKeyStore.Revoke(ctx, userID, keyID)), so +// a key belonging to another account, a key that does not exist, and an +// already-revoked key all answer the same 404 api_key_not_found — a +// caller can never learn whether somebody else's key exists. +// +// Irreversible on purpose: the reason a key gets revoked is that somebody +// else may have it, so mint a new one rather than offering an un-revoke. +func (h *APIKeyHandlers) Revoke(w http.ResponseWriter, r *http.Request, keyID string) { + userID := UserIDFromContext(r) + if err := cryden.RevokeAPIKey(r.Context(), h.Engine, userID, keyID); err != nil { + writeErr(w, err) + return + } + // Same 200-and-a-status-body shape DELETE /v1/sessions/{id} returns, + // rather than a 204 — this repo's DELETEs answer consistently. + writeData(w, http.StatusOK, map[string]string{"status": "api key revoked"}) +} diff --git a/httpapi/apikey_handlers_test.go b/httpapi/apikey_handlers_test.go new file mode 100644 index 0000000..3796a4a --- /dev/null +++ b/httpapi/apikey_handlers_test.go @@ -0,0 +1,296 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/crydensync/cryden/v2" + + "github.com/crydensync/api/config" +) + +// apiKeyFixture is one signed-up, logged-in user plus the router built on +// the engine they live in. Every test below works through the real route +// table rather than calling a handler directly, because the routes are +// half of what these endpoints are: RequireAuth derives the user ID and +// cryden scopes every store call from it, so a handler called in +// isolation would prove neither. +type apiKeyFixture struct { + engine *cryden.Engine + router http.Handler + userID string + token string +} + +func newAPIKeyFixture(t *testing.T, email string) apiKeyFixture { + t.Helper() + return newAPIKeyFixtureOn(t, newTestEngine(t), email) +} + +// newAPIKeyFixtureOn is newAPIKeyFixture against an engine the caller +// already holds. Two accounts sharing one engine is what makes the +// ownership assertion below a real one: a second store would answer 404 +// for a key that simply isn't there, which proves nothing about the +// user_id predicate the store call actually carries. +func newAPIKeyFixtureOn(t *testing.T, engine *cryden.Engine, email string) apiKeyFixture { + t.Helper() + ctx := context.Background() + + user, err := cryden.SignUp(ctx, engine, email, testPassword, "203.0.113.10") + if err != nil { + t.Fatalf("signup (%s): %v", email, err) + } + tokens, err := cryden.Login(ctx, engine, email, testPassword, "203.0.113.10", chromeOnMacOS) + if err != nil { + t.Fatalf("login (%s): %v", email, err) + } + + return apiKeyFixture{ + engine: engine, + router: NewRouter(Deps{Engine: engine, Config: config.Config{}}), + userID: user.ID, + token: tokens.AccessToken, + } +} + +func (f apiKeyFixture) call(t *testing.T, method, path, body, token string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + f.router.ServeHTTP(rec, req) + return rec +} + +// createResponse is the creation envelope: the raw key, the stored +// record, and the notice saying the raw key cannot be shown again. +type createResponse struct { + Data struct { + Key string `json:"key"` + Notice string `json:"notice"` + APIKey apiKeyDTO `json:"api_key"` + } `json:"data"` +} + +func decodeCreate(t *testing.T, rec *httptest.ResponseRecorder) createResponse { + t.Helper() + var resp createResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decoding %s: %v", rec.Body.String(), err) + } + return resp +} + +// The one-time-display contract. A caller that loses the raw key has to +// mint a new one, so the response has to say so — and the key itself has +// to be absent from every later read, which is the half the engine +// guarantees by storing only its hash. +func TestAPIKeyCreateReturnsTheRawKeyExactlyOnce(t *testing.T) { + f := newAPIKeyFixture(t, "dana@example.com") + + rec := f.call(t, http.MethodPost, "/v1/api-keys", `{"name":"ci deploy","scopes":["read"]}`, f.token) + if rec.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201 (body %s)", rec.Code, rec.Body.String()) + } + created := decodeCreate(t, rec) + + if created.Data.Key == "" { + t.Fatal("no raw key in the creation response — it is the only place it can ever appear") + } + if !strings.HasPrefix(created.Data.Key, "ck_") { + t.Errorf("raw key %q does not carry the configured prefix", created.Data.Key) + } + if created.Data.Notice == "" { + t.Error("no notice, so a client can render the key without telling the user it is unretrievable") + } + // The stored Prefix is the label plus a short fragment of the secret + // ("ck_9f3a1c02") — cryden derives it from the raw key so the two can + // never disagree, and it is what a list shows in place of the secret. + if !strings.HasPrefix(created.Data.APIKey.Prefix, "ck_") { + t.Errorf("stored prefix = %q, want a ck_ label", created.Data.APIKey.Prefix) + } + if created.Data.APIKey.ID == "" { + t.Error("no id on the stored record, so there is nothing to revoke by") + } + // Scopes is an array in the response even when it was not sent, so a + // client can iterate it without a nil check. + if created.Data.APIKey.Scopes == nil { + t.Error("scopes came back null, want an array") + } + + // The listing is where a raw key would leak if the engine ever kept + // one. It must not appear anywhere in the body — not as a field, not + // as a prefix, not at all. + listRec := f.call(t, http.MethodGet, "/v1/api-keys", "", f.token) + if listRec.Code != http.StatusOK { + t.Fatalf("list status = %d, want 200 (body %s)", listRec.Code, listRec.Body.String()) + } + if strings.Contains(listRec.Body.String(), created.Data.Key) { + t.Errorf("the raw key appears in the listing: %s", listRec.Body.String()) + } + + var list struct { + Data []map[string]any `json:"data"` + } + if err := json.Unmarshal(listRec.Body.Bytes(), &list); err != nil { + t.Fatalf("decoding list %s: %v", listRec.Body.String(), err) + } + if len(list.Data) != 1 { + t.Fatalf("got %d keys, want the one that was just created", len(list.Data)) + } + if _, present := list.Data[0]["key"]; present { + t.Error("the listing carries a \"key\" field; only the creation response may") + } + if _, present := list.Data[0]["key_hash"]; present { + t.Error("the listing carries a key_hash field") + } +} + +// Revocation is scoped by cryden itself: the store statement is +// WHERE id = $1 AND user_id = $2, so someone else's key and a key that +// does not exist answer identically. A caller must never be able to learn +// whether another account's key ID is real. +func TestAPIKeyRevokeIsScopedToTheCallingUser(t *testing.T) { + owner := newAPIKeyFixture(t, "owner@example.com") + // Both accounts live in ONE engine, so the 404 below is the store's + // own "WHERE id = $1 AND user_id = $2" coming back empty rather than a + // key that was never in a second store to begin with. + other := newAPIKeyFixtureOn(t, owner.engine, "other@example.com") + + created := decodeCreate(t, owner.call(t, http.MethodPost, "/v1/api-keys", `{"name":"deploy"}`, owner.token)) + keyID := created.Data.APIKey.ID + + rec := other.call(t, http.MethodDelete, "/v1/api-keys/"+keyID, "", other.token) + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404 (body %s)", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "api_key_not_found") { + t.Errorf("body = %s, want api_key_not_found", rec.Body.String()) + } + + // And the failed attempt changed nothing: the key is still live for + // its actual owner. + var list struct { + Data []apiKeyDTO `json:"data"` + } + listRec := owner.call(t, http.MethodGet, "/v1/api-keys", "", owner.token) + if err := json.Unmarshal(listRec.Body.Bytes(), &list); err != nil { + t.Fatalf("decoding owner's list %s: %v", listRec.Body.String(), err) + } + if len(list.Data) != 1 || list.Data[0].ID != keyID { + t.Errorf("owner's keys = %+v, want the key another account failed to revoke", list.Data) + } + + // The owner can, and afterwards it is gone from the listing rather + // than listed as revoked — a revoked key is dealt with, and the list + // is what is still live. + ownRec := owner.call(t, http.MethodDelete, "/v1/api-keys/"+keyID, "", owner.token) + if ownRec.Code != http.StatusOK { + t.Fatalf("owner revoke status = %d, want 200 (body %s)", ownRec.Code, ownRec.Body.String()) + } + if !strings.Contains(ownRec.Body.String(), "api key revoked") { + t.Errorf("body = %s, want the status body DELETE /v1/sessions/{id} also returns", ownRec.Body.String()) + } + + listRec = owner.call(t, http.MethodGet, "/v1/api-keys", "", owner.token) + if err := json.Unmarshal(listRec.Body.Bytes(), &list); err != nil { + t.Fatalf("decoding owner's list %s: %v", listRec.Body.String(), err) + } + if len(list.Data) != 0 { + t.Errorf("after revoking, the listing still has %d keys: %+v", len(list.Data), list.Data) + } + + // Revoking a second time is the same 404 — an already-revoked key is + // indistinguishable from one that never existed. + if again := owner.call(t, http.MethodDelete, "/v1/api-keys/"+keyID, "", owner.token); again.Code != http.StatusNotFound { + t.Errorf("re-revoking status = %d, want 404", again.Code) + } +} + +// expires_in_days is the one field a caller supplies that can be wrong in +// a way the engine would report confusingly: it is multiplied by 24h +// before it reaches cryden, so a large enough value overflows into a +// negative duration and comes back as invalid_api_key_ttl. Bounding it +// here means the 400 says what is actually wrong. +func TestAPIKeyExpiryBounds(t *testing.T) { + f := newAPIKeyFixture(t, "dana@example.com") + + t.Run("absent means never expires", func(t *testing.T) { + created := decodeCreate(t, f.call(t, http.MethodPost, "/v1/api-keys", `{"name":"forever"}`, f.token)) + if created.Data.APIKey.ExpiresAt != nil { + t.Errorf("expires_at = %v, want null", *created.Data.APIKey.ExpiresAt) + } + if created.Data.APIKey.Expired { + t.Error("a key with no expiry reported itself as expired") + } + }) + + t.Run("a positive value sets a future expiry", func(t *testing.T) { + created := decodeCreate(t, f.call(t, http.MethodPost, "/v1/api-keys", `{"name":"90 days","expires_in_days":90}`, f.token)) + if created.Data.APIKey.ExpiresAt == nil { + t.Fatal("expires_at = null, want the requested expiry") + } + if created.Data.APIKey.Expired { + t.Error("a key minted with a 90-day expiry is already expired") + } + }) + + for _, tc := range []struct { + name string + body string + }{ + {"negative", `{"name":"past","expires_in_days":-1}`}, + {"absurd", `{"name":"forever","expires_in_days":4000}`}, + {"not a number", `{"name":"soon","expires_in_days":"90"}`}, + } { + t.Run(tc.name, func(t *testing.T) { + rec := f.call(t, http.MethodPost, "/v1/api-keys", tc.body, f.token) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (body %s)", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "bad_request") { + t.Errorf("body = %s, want a bad_request error", rec.Body.String()) + } + }) + } + + t.Run("an over-long name is refused", func(t *testing.T) { + body := `{"name":"` + strings.Repeat("x", 101) + `"}` + if rec := f.call(t, http.MethodPost, "/v1/api-keys", body, f.token); rec.Code != http.StatusBadRequest { + t.Errorf("status = %d, want 400", rec.Code) + } + }) +} + +// Every route on this surface is behind RequireAuth, and the user ID the +// handlers act on comes from the token — never from a body or a path, so +// there is no parameter a caller could tamper with to reach another +// account's keys. +func TestAPIKeyRoutesRequireAuth(t *testing.T) { + f := newAPIKeyFixture(t, "dana@example.com") + + for _, tc := range []struct{ method, path, body string }{ + {http.MethodPost, "/v1/api-keys", `{"name":"x"}`}, + {http.MethodGet, "/v1/api-keys", ""}, + {http.MethodDelete, "/v1/api-keys/some-id", ""}, + } { + rec := f.call(t, tc.method, tc.path, tc.body, "") + if rec.Code != http.StatusUnauthorized { + t.Errorf("%s %s without a token: status = %d, want 401", tc.method, tc.path, rec.Code) + } + if !strings.Contains(rec.Body.String(), "missing_auth_header") { + t.Errorf("%s %s body = %s, want missing_auth_header", tc.method, tc.path, rec.Body.String()) + } + } + + rec := f.call(t, http.MethodGet, "/v1/api-keys", "", "not-a-real-token") + if rec.Code != http.StatusUnauthorized { + t.Errorf("a garbage token: status = %d, want 401 (body %s)", rec.Code, rec.Body.String()) + } +} diff --git a/httpapi/errors.go b/httpapi/errors.go index 47a5be9..d7b0c5d 100644 --- a/httpapi/errors.go +++ b/httpapi/errors.go @@ -115,7 +115,22 @@ func mapError(err error) apiError { return apiError{http.StatusBadRequest, "invalid_ceremony_token", "this passkey ceremony has expired — please start again"} case errors.Is(err, auth.ErrPasswordBreached): return apiError{http.StatusBadRequest, "password_breached", "this password has appeared in a known data breach and cannot be used"} - // The four "not configured" sentinels below mean this deployment has + // API key errors. ErrInvalidAPIKey is what a presented key fails with + // — unknown, revoked, expired, malformed, empty, all one error on + // purpose so a caller cannot probe which of the keys it holds are + // still live. No endpoint in this repo accepts an API key yet, so + // nothing returns it today; it is mapped here so the first one that + // does cannot ship without it, which is the entire reason mapError is + // a single site. + case errors.Is(err, auth.ErrInvalidAPIKey): + return apiError{http.StatusUnauthorized, "invalid_api_key", "this API key is invalid, revoked or expired"} + case errors.Is(err, auth.ErrAPIKeyNotFound): + return apiError{http.StatusNotFound, "api_key_not_found", "no such API key"} + case errors.Is(err, auth.ErrInvalidAPIKeyScope): + return apiError{http.StatusBadRequest, "invalid_api_key_scope", "a scope must be non-empty and contain no whitespace"} + case errors.Is(err, auth.ErrInvalidAPIKeyTTL): + return apiError{http.StatusBadRequest, "invalid_api_key_ttl", "expiry cannot be in the past"} + // The five "not configured" sentinels below mean this deployment has // not enabled that feature, not that the caller did anything wrong. // 404 rather than 500 so a client can hide the option instead of // reporting a server fault, matching oauth_provider_not_configured. @@ -127,6 +142,12 @@ func mapError(err error) apiError { return apiError{http.StatusNotFound, "magic_link_not_configured", "magic-link login is not enabled on this deployment"} case errors.Is(err, cryden.ErrRecoveryCodesNotConfigured): return apiError{http.StatusNotFound, "recovery_codes_not_configured", "recovery codes are not enabled on this deployment"} + // main.go always wires the API key store, so this is not reachable on + // this repo's own deployments — it is mapped so an engine built + // without Config.APIKeys answers a real shape rather than a 500, + // which is what a test or an embedding host would otherwise get. + case errors.Is(err, cryden.ErrAPIKeysNotConfigured): + return apiError{http.StatusNotFound, "api_keys_not_configured", "API keys are not enabled on this deployment"} default: // Struct-typed errors (not plain sentinels) need errors.As, // not errors.Is — ErrOAuthEmailConflict carries Email and diff --git a/httpapi/response.go b/httpapi/response.go index 86308f7..6030066 100644 --- a/httpapi/response.go +++ b/httpapi/response.go @@ -5,6 +5,7 @@ import ( "errors" "log" "net/http" + "time" "github.com/crydensync/cryden/v2/auth" ) @@ -51,3 +52,26 @@ func writeBadRequest(w http.ResponseWriter, message string) { func decodeJSON(r *http.Request, v any) error { return json.NewDecoder(r.Body).Decode(v) } + +// timeLayout is how every timestamp leaves this API — RFC 3339 with an +// offset, which is what time.Time.Format("2006-01-02T15:04:05Z07:00") +// produces and what openapi/spec.yaml documents. Named once so a new +// field cannot quietly pick a second spelling. +const timeLayout = "2006-01-02T15:04:05Z07:00" + +func formatTime(t time.Time) string { + return t.Format(timeLayout) +} + +// formatTimePtr is formatTime for the engine's optional timestamps — +// ExpiresAt and LastUsedAt on an API key, which are nil rather than zero +// until something sets them. A nil stays a JSON null so "never expires" +// and "never used" are distinguishable from a real instant, which a +// zero-valued time rendered as a string would not be. +func formatTimePtr(t *time.Time) *string { + if t == nil { + return nil + } + formatted := t.Format(timeLayout) + return &formatted +} diff --git a/httpapi/router.go b/httpapi/router.go index 6a99819..066b28b 100644 --- a/httpapi/router.go +++ b/httpapi/router.go @@ -41,6 +41,7 @@ func NewRouter(d Deps) http.Handler { passkeys := &PasskeyHandlers{Engine: engine} magicLink := &MagicLinkHandlers{Engine: engine} recovery := &RecoveryHandlers{Engine: engine} + apiKeys := &APIKeyHandlers{Engine: engine} mux := http.NewServeMux() @@ -116,6 +117,19 @@ func NewRouter(d Deps) http.Handler { mux.HandleFunc("POST /v1/recovery-codes/generate", RequireAuth(engine, recovery.Generate)) + // API keys — machine-to-machine credentials belonging to the calling + // user. Every handler here is scoped by cryden itself (auth.GenerateAPIKey/ + // ListAPIKeys/RevokeAPIKey all take the userID straight from the verified + // token and never from the request), so one account can never read or + // revoke another's key. Optional per deployment: unset Config.APIKeys + // answers 404 api_keys_not_configured, the same shape as every other + // unconfigured engine feature. + mux.HandleFunc("POST /v1/api-keys", RequireAuth(engine, apiKeys.Create)) + mux.HandleFunc("GET /v1/api-keys", RequireAuth(engine, apiKeys.List)) + mux.HandleFunc("DELETE /v1/api-keys/{keyID}", RequireAuth(engine, func(w http.ResponseWriter, r *http.Request) { + apiKeys.Revoke(w, r, r.PathValue("keyID")) + })) + // Admin endpoints — the first in this repo, hence the note. Everything // under /v1/admin goes through RequireAdmin (middleware.go), which // needs the `role` claim an operator's token carries. OAuth provider diff --git a/httpapi/session_handlers_test.go b/httpapi/session_handlers_test.go index d0175c0..262e71f 100644 --- a/httpapi/session_handlers_test.go +++ b/httpapi/session_handlers_test.go @@ -50,6 +50,12 @@ func newTestEngineWithClaims(t *testing.T, claims token.ClaimsProvider) *cryden. Verifications: memory.NewVerificationStore(), EmailSender: stubMailSender{}, MagicLinkSender: stubMailSender{}, + // API keys are always wired in production (see main.go), so the + // test engine wires them too — otherwise the endpoints built on + // them would answer 404 api_keys_not_configured here and every + // test of one would be a test of that instead. + APIKeys: memory.NewAPIKeyStore(), + APIKeyPrefix: "ck", } if claims != nil { cfg.AccessTokenClaims = claims From 93e57f0e4381be3e934ab4601db9595fa5c57943 Mon Sep 17 00:00:00 2001 From: Raymond Nicholas Date: Tue, 15 Sep 2026 12:34:14 +0100 Subject: [PATCH 04/10] feat: add GET /v1/admin/security/hash-migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read-only, behind RequireAdmin. Reports what the deployment is configured to write, the user total, and the all-time and 7-day counts of the engine's own password_hash_upgraded audit events. upgraded_events counts events, not users, so it can exceed total_users — estimated_remaining is floored at zero and named "estimated" for that reason. main.go now hoists its user and audit stores so the report counts the same instances the engine writes through. Co-Authored-By: Claude Code --- httpapi/errors.go | 9 + httpapi/query.go | 50 ++++ httpapi/router.go | 38 ++- httpapi/security_handlers.go | 161 ++++++++++++ httpapi/security_handlers_test.go | 410 ++++++++++++++++++++++++++++++ main.go | 15 +- 6 files changed, 671 insertions(+), 12 deletions(-) create mode 100644 httpapi/query.go create mode 100644 httpapi/security_handlers.go create mode 100644 httpapi/security_handlers_test.go diff --git a/httpapi/errors.go b/httpapi/errors.go index d7b0c5d..8782e0a 100644 --- a/httpapi/errors.go +++ b/httpapi/errors.go @@ -42,6 +42,13 @@ var errNotOperator = errors.New("this account does not have console operator acc // code to the client; the distinction only matters server-side. var errEdgeRateLimited = errors.New("too many requests") +// errAdminStoresUnavailable is returned by an admin handler whose backing +// stores were never wired — see httpapi.Deps, whose store fields a test +// may legitimately leave nil. That is a wiring fact and not a server +// fault, so it answers 404 like every other unconfigured feature in this +// API rather than a 500 an operator would read as a bug. +var errAdminStoresUnavailable = errors.New("this report requires stores that are not configured on this deployment") + // The following three are local, API-layer-only errors from the // OAuth redirect/callback flow itself — never returned by the engine, // which never touches HTTP or a specific provider. @@ -60,6 +67,8 @@ func mapError(err error) apiError { return apiError{http.StatusForbidden, "not_operator", "this account does not have console operator access"} case errors.Is(err, errEdgeRateLimited): return apiError{http.StatusTooManyRequests, "rate_limited", "too many requests, please slow down"} + case errors.Is(err, errAdminStoresUnavailable): + return apiError{http.StatusNotFound, "not_configured", "this report is not available on this deployment"} case errors.Is(err, errOAuthProviderNotConfigured): return apiError{http.StatusNotFound, "oauth_provider_not_configured", "this OAuth provider is not configured on this deployment"} case errors.Is(err, errOAuthStateMismatch): diff --git a/httpapi/query.go b/httpapi/query.go new file mode 100644 index 0000000..14391d8 --- /dev/null +++ b/httpapi/query.go @@ -0,0 +1,50 @@ +package httpapi + +import ( + "fmt" + "net/http" + "strconv" + "strings" +) + +// Pagination and filtering bounds for the admin list endpoints. Named +// constants rather than literals at each call site so the cap is one +// decision: every one of these endpoints returns rows an operator reads +// by eye, and an unbounded limit is a way to make the API buffer a whole +// table into a JSON response. +const ( + defaultListLimit = 50 + maxListLimit = 500 +) + +// queryInt reads an optional integer query parameter, falling back to def +// when it is absent or empty. A value that is not a number, or is outside +// [min, max], is an error the caller turns into a 400 — deliberately not +// clamped silently, because a caller asking for limit=100000 and getting +// 500 back has no way to tell that from a table that happens to hold 500 +// rows. +func queryInt(r *http.Request, name string, def, min, max int) (int, error) { + raw := strings.TrimSpace(r.URL.Query().Get(name)) + if raw == "" { + return def, nil + } + n, err := strconv.Atoi(raw) + if err != nil { + return 0, fmt.Errorf("%s must be a number", name) + } + if n < min || n > max { + return 0, fmt.Errorf("%s must be between %d and %d", name, min, max) + } + return n, nil +} + +// queryLimit is queryInt with the shared list bounds, which is what every +// admin list endpoint wants. +func queryLimit(r *http.Request) (int, error) { + return queryInt(r, "limit", defaultListLimit, 1, maxListLimit) +} + +// queryString reads an optional, trimmed string query parameter. +func queryString(r *http.Request, name string) string { + return strings.TrimSpace(r.URL.Query().Get(name)) +} diff --git a/httpapi/router.go b/httpapi/router.go index 066b28b..31cb82a 100644 --- a/httpapi/router.go +++ b/httpapi/router.go @@ -5,17 +5,23 @@ import ( "net/http" "github.com/crydensync/cryden/v2" + "github.com/crydensync/cryden/v2/store" "github.com/crydensync/api/config" ) // Deps is everything the route table needs to build its handlers. It is a // struct rather than a parameter list because the admin surface keeps -// growing: every endpoint added under /v1/admin needs something the route -// table does not have today, and a struct absorbs that without every -// existing call site growing a positional argument. It also lets a test -// build a router with exactly the dependencies the endpoint under test -// needs and leave the rest nil. +// growing: the hash-migration report alone needs stores the engine holds +// unexported, so they can only come from whoever constructed them +// (main.go). A struct also lets a test build a router with exactly the +// dependencies the endpoint under test needs and leave the rest nil. +// +// Every store here is the SAME instance main.go handed to cryden. Building +// a second one would be worse than wasteful — the hash-migration count and +// the engine's own writes would be reading different objects, and a +// repo-owned store the engine writes through would be invisible to the +// endpoint that reports on it. type Deps struct { Engine *cryden.Engine @@ -24,6 +30,13 @@ type Deps struct { // reports the database as unconfigured rather than panicking. DB *sql.DB Config config.Config + + // Audit and Users back GET /v1/admin/security/hash-migration — cryden + // exposes no bulk way to inspect stored password hashes, so the + // migration is measured from the audit events the engine already + // records against the user total. + Audit store.AuditStore + Users store.UserStore } // NewRouter builds the full route table. Called once from main.go. @@ -42,6 +55,7 @@ func NewRouter(d Deps) http.Handler { magicLink := &MagicLinkHandlers{Engine: engine} recovery := &RecoveryHandlers{Engine: engine} apiKeys := &APIKeyHandlers{Engine: engine} + security := &SecurityHandlers{Audit: d.Audit, Users: d.Users, Config: d.Config} mux := http.NewServeMux() @@ -130,12 +144,16 @@ func NewRouter(d Deps) http.Handler { apiKeys.Revoke(w, r, r.PathValue("keyID")) })) - // Admin endpoints — the first in this repo, hence the note. Everything - // under /v1/admin goes through RequireAdmin (middleware.go), which - // needs the `role` claim an operator's token carries. OAuth provider - // health is this repo's own logic: cryden has no concept of a provider - // being reachable, only of whether it is configured. + // Admin endpoints. Everything under /v1/admin goes through RequireAdmin + // (middleware.go), which needs the `role` claim an operator's token + // carries. OAuth provider health and the hash-migration report are both + // this repo's own logic — cryden has no concept of a provider being + // reachable, and no bulk way to read stored hash algorithms. + // + // Every endpoint here is read-only, and has to stay that way: see + // CLAUDE.md's hard rule about the admin surface. mux.HandleFunc("GET /v1/admin/oauth/health", RequireAdmin(engine, oauthHealth.Health)) + mux.HandleFunc("GET /v1/admin/security/hash-migration", RequireAdmin(engine, security.HashMigration)) return mux } diff --git a/httpapi/security_handlers.go b/httpapi/security_handlers.go new file mode 100644 index 0000000..421ceca --- /dev/null +++ b/httpapi/security_handlers.go @@ -0,0 +1,161 @@ +package httpapi + +import ( + "net/http" + "time" + + "github.com/crydensync/cryden/v2/store" + + "github.com/crydensync/api/config" +) + +// SecurityHandlers answers the admin security reports. +type SecurityHandlers struct { + // Audit and Users are the same store instances main.go handed cryden + // (see Deps). Reading them directly is what makes this report possible + // at all: cryden exposes no bulk way to inspect stored password hashes, + // so the migration is measured from the audit events the engine already + // records on an upgrade, against the user total. + // + // Both may be nil in a router built without them (tests). That is a + // wiring fact, not a server fault, so it answers 404 rather than 500 — + // the same shape as every other unconfigured feature in this API. + Audit store.AuditStore + Users store.UserStore + Config config.Config +} + +// hashMigrationDefaultWindowDays is the reporting window the endpoint uses +// when the caller does not ask for another one. A week matches the cadence +// an operator actually checks a migration on, and is the same window +// cryden's own weekly digest uses. +const hashMigrationDefaultWindowDays = 7 + +// hasherDTO reports what this deployment writes NEW password hashes with. +// It is read from configuration, not from the users table: cryden exposes +// no bulk way to inspect the algorithm behind a stored hash, and adding one +// would be the engine's job rather than this repo's (see CLAUDE.md's +// ownership rule). So this block describes the destination of the +// migration, never its current position — the counts below are the +// position. +type hasherDTO struct { + // Algorithm is "bcrypt" or "argon2id" — whichever PASSWORD_HASHER + // selected. Both are always verifiable: cryden wraps whichever hasher it + // is given in a MultiHasher that picks the verifier from each stored + // hash's own format, so old hashes keep working and are rewritten one + // successful login at a time. + Algorithm string `json:"algorithm"` + + // The Argon2id cost parameters, omitted entirely for bcrypt — they would + // be meaningless there, and omitempty on a uint32 keeps them out rather + // than reporting four zeroes a reader would have to know to ignore. + MemoryKiB uint32 `json:"memory_kib,omitempty"` + Iterations uint32 `json:"iterations,omitempty"` + Parallelism uint8 `json:"parallelism,omitempty"` +} + +// hashMigrationDTO is the whole report. The naming of the fields is +// deliberate and load-bearing: +// +// - UpgradedEvents counts EVENTS, not users. A user whose hash is +// rewritten twice — a second cost increase a year later — contributes +// two, so this number can exceed TotalUsers. Calling it "upgraded_users" +// would be reporting a figure that is right most of the time and +// quietly wrong exactly when someone is watching it closely. +// - EstimatedRemaining is therefore ESTIMATED, and floored at zero for +// the same reason: TotalUsers - UpgradedEvents is only a user count if +// every user upgraded exactly once, which is the normal case and not a +// guarantee. +// +// Both names are the honest description of what is being computed, and the +// README and openapi/spec.yaml say the same thing in prose so the number +// does not get read as something it isn't. +// +// UpgradedEventsInWindow is the field that actually answers "is this +// draining": the all-time count only ever rises, while a windowed one falls +// to zero as the last stragglers log in. +type hashMigrationDTO struct { + Hasher hasherDTO `json:"hasher"` + TotalUsers int `json:"total_users"` + UpgradedEvents int `json:"upgraded_events"` + EstimatedRemaining int `json:"estimated_remaining"` + WindowDays int `json:"window_days"` + UpgradedEventsInWindow int `json:"upgraded_events_in_window"` +} + +// HashMigration — admin required (see router.go). Reports how far a +// password-hash migration has got, as the engine's own audit events against +// the user total. Read-only by construction: it calls two count methods and +// records nothing, the same rule every endpoint on the admin surface +// follows (see CLAUDE.md). +// +// An optional window_days query parameter sets the reporting window; it +// defaults to a week and is bounded, so a caller cannot ask for a window so +// wide the count stops meaning anything. +func (h *SecurityHandlers) HashMigration(w http.ResponseWriter, r *http.Request) { + if h.Audit == nil || h.Users == nil { + writeErr(w, errAdminStoresUnavailable) + return + } + + windowDays, err := queryInt(r, "window_days", hashMigrationDefaultWindowDays, 1, 365) + if err != nil { + writeBadRequest(w, err.Error()) + return + } + + ctx := r.Context() + + total, err := h.Users.Count(ctx) + if err != nil { + writeErr(w, err) + return + } + + // A zero time.Time as `since` is an open lower bound rather than a date + // anyone chose: both store implementations read it as "at or after the + // beginning of time", so this is the all-time count. CountByType returns + // one entry per type that actually occurred and omits the rest, which is + // why a missing key reads as zero here rather than as an error. + allTime, err := h.Audit.CountByType(ctx, time.Time{}) + if err != nil { + writeErr(w, err) + return + } + windowed, err := h.Audit.CountByType(ctx, time.Now().AddDate(0, 0, -windowDays)) + if err != nil { + writeErr(w, err) + return + } + + upgraded := allTime[store.EventPasswordHashUpgraded] + remaining := total - upgraded + if remaining < 0 { + remaining = 0 + } + + writeData(w, http.StatusOK, hashMigrationDTO{ + Hasher: h.hasher(), + TotalUsers: total, + UpgradedEvents: upgraded, + EstimatedRemaining: remaining, + WindowDays: windowDays, + UpgradedEventsInWindow: windowed[store.EventPasswordHashUpgraded], + }) +} + +func (h *SecurityHandlers) hasher() hasherDTO { + if h.Config.PasswordHasher != config.PasswordHasherArgon2id { + // The engine's own default hasher. Its cost is cryden's default + // BcryptCost, which this repo does not expose as a knob — see + // PROGRESS.md's note on why BCRYPT_COST was left out of Tier 3. + return hasherDTO{Algorithm: config.PasswordHasherBcrypt} + } + p := h.Config.Argon2idParams + return hasherDTO{ + Algorithm: config.PasswordHasherArgon2id, + MemoryKiB: p.Memory, + Iterations: p.Iterations, + Parallelism: p.Parallelism, + } +} diff --git a/httpapi/security_handlers_test.go b/httpapi/security_handlers_test.go new file mode 100644 index 0000000..e261650 --- /dev/null +++ b/httpapi/security_handlers_test.go @@ -0,0 +1,410 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/crydensync/cryden/v2" + "github.com/crydensync/cryden/v2/security" + "github.com/crydensync/cryden/v2/store" + "github.com/crydensync/cryden/v2/store/memory" + "github.com/crydensync/cryden/v2/token" + + "github.com/crydensync/api/config" +) + +// hashMigrationResponse mirrors the endpoint's own DTO field by field, so +// a renamed or dropped field fails here rather than silently changing the +// contract an operator's dashboard reads. +type hashMigrationResponse struct { + Data struct { + Hasher struct { + Algorithm string `json:"algorithm"` + MemoryKiB uint32 `json:"memory_kib"` + Iterations uint32 `json:"iterations"` + Parallelism uint8 `json:"parallelism"` + } `json:"hasher"` + TotalUsers int `json:"total_users"` + UpgradedEvents int `json:"upgraded_events"` + EstimatedRemaining int `json:"estimated_remaining"` + WindowDays int `json:"window_days"` + UpgradedEventsInWindow int `json:"upgraded_events_in_window"` + } `json:"data"` +} + +// securityFixture is an engine whose stores this test holds directly, on +// the Argon2id configuration. Holding the stores is the point: the +// interesting assertion is that the endpoint's counts track what the +// engine itself wrote, which is only checkable against the same objects. +type securityFixture struct { + engine *cryden.Engine + users *memory.UserStore + audit *memory.AuditStore + router http.Handler + + adminID string + adminToken string + subjectID string + subjectEmail string +} + +func newSecurityFixture(t *testing.T, cfg config.Config) securityFixture { + t.Helper() + ctx := context.Background() + + users := memory.NewUserStore() + audit := memory.NewAuditStore() + + var adminID string + engineCfg := cryden.Config{ + JWTSecret: "test-secret", + Users: users, + Sessions: memory.NewSessionStore(), + Audit: audit, + Verifications: memory.NewVerificationStore(), + EmailSender: stubMailSender{}, + MagicLinkSender: stubMailSender{}, + // The claims provider is the same mechanism main.go uses to put a + // `role` claim on an operator's token, and the only way to get a + // token RequireAdmin accepts. + AccessTokenClaims: token.ClaimsFunc(func(_ context.Context, userID string) (map[string]any, error) { + if userID == adminID { + return map[string]any{"role": "admin"}, nil + } + return nil, nil + }), + } + if cfg.PasswordHasher == config.PasswordHasherArgon2id { + hasher, err := security.NewArgon2idHasher(cfg.Argon2idParams) + if err != nil { + t.Fatalf("building the Argon2id hasher: %v", err) + } + engineCfg.Hasher = hasher + } + engine, err := cryden.New(engineCfg) + if err != nil { + t.Fatalf("cryden.New on the in-memory stores: %v", err) + } + + admin, err := cryden.SignUp(ctx, engine, "operator@example.com", testPassword, "203.0.113.1") + if err != nil { + t.Fatalf("signup (operator): %v", err) + } + // Set between signup and login: the claim is attached when a token is + // issued, exactly as it is in production. + adminID = admin.ID + adminTokens, err := cryden.Login(ctx, engine, "operator@example.com", testPassword, "203.0.113.1", chromeOnMacOS) + if err != nil { + t.Fatalf("login (operator): %v", err) + } + + const subjectEmail = "dana@example.com" + subject, err := cryden.SignUp(ctx, engine, subjectEmail, testPassword, "203.0.113.2") + if err != nil { + t.Fatalf("signup (subject): %v", err) + } + + return securityFixture{ + engine: engine, + users: users, + audit: audit, + router: NewRouter(Deps{Engine: engine, Audit: audit, Users: users, Config: cfg}), + adminID: admin.ID, + adminToken: adminTokens.AccessToken, + subjectID: subject.ID, + subjectEmail: subjectEmail, + } +} + +func (f securityFixture) report(t *testing.T, token, query string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/v1/admin/security/hash-migration"+query, nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + f.router.ServeHTTP(rec, req) + return rec +} + +func decodeReport(t *testing.T, rec *httptest.ResponseRecorder) hashMigrationResponse { + t.Helper() + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + var resp hashMigrationResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decoding %s: %v", rec.Body.String(), err) + } + return resp +} + +// The migration is exercised end to end rather than against a hand-seeded +// audit row: a real bcrypt hash is planted in the user store, a real login +// rewrites it with the engine's own Argon2id hasher, and the endpoint is +// asserted to see that. A seeded event would prove the counting works and +// prove nothing about whether a real upgrade produces one. +func TestHashMigrationTracksARealBcryptToArgon2idUpgrade(t *testing.T) { + ctx := context.Background() + cfg := config.Config{ + PasswordHasher: config.PasswordHasherArgon2id, + Argon2idParams: security.DefaultArgon2idParams, + } + f := newSecurityFixture(t, cfg) + + // Before: two accounts, both written with Argon2id, so nothing has + // ever needed upgrading. + before := decodeReport(t, f.report(t, f.adminToken, "")) + if before.Data.TotalUsers != 2 { + t.Fatalf("total_users = %d, want 2", before.Data.TotalUsers) + } + if before.Data.UpgradedEvents != 0 { + t.Fatalf("upgraded_events = %d before anything was upgraded, want 0", before.Data.UpgradedEvents) + } + if before.Data.WindowDays != hashMigrationDefaultWindowDays { + t.Errorf("window_days = %d, want the default %d", before.Data.WindowDays, hashMigrationDefaultWindowDays) + } + + // Plant a bcrypt hash, which is the state an account that predates the + // switch is actually in. + bcryptHasher, err := security.NewBcryptHasher(10) + if err != nil { + t.Fatalf("building the bcrypt hasher: %v", err) + } + bcryptHash, err := bcryptHasher.Hash(testPassword) + if err != nil { + t.Fatalf("hashing with bcrypt: %v", err) + } + if err := f.users.UpdatePasswordHash(ctx, f.subjectID, bcryptHash); err != nil { + t.Fatalf("planting the bcrypt hash: %v", err) + } + + // A bcrypt hash is still a stored row, so the position has NOT moved + // — the count is of upgrades performed, not of hashes that will need + // one. This is the assertion that keeps the field honest. + pending := decodeReport(t, f.report(t, f.adminToken, "")) + if pending.Data.UpgradedEvents != 0 { + t.Errorf("upgraded_events = %d with an un-upgraded hash in place, want 0", pending.Data.UpgradedEvents) + } + if pending.Data.EstimatedRemaining != 2 { + t.Errorf("estimated_remaining = %d, want 2 — nothing has been upgraded yet", pending.Data.EstimatedRemaining) + } + + // The login itself is what migrates the row: the engine verifies the + // bcrypt hash through its MultiHasher, sees it is not what it would + // write now, and rewrites it. + if _, err := cryden.Login(ctx, f.engine, f.subjectEmail, testPassword, "203.0.113.2", chromeOnMacOS); err != nil { + t.Fatalf("login (subject): %v — a bcrypt hash must still verify on an Argon2id engine", err) + } + + // The rewrite really happened, read back off the store the report is + // counting against. + stored, err := f.users.GetByID(ctx, f.subjectID) + if err != nil { + t.Fatalf("reading the subject back: %v", err) + } + if got := security.IdentifyHash(stored.PasswordHash); got != security.AlgorithmArgon2id { + t.Fatalf("stored hash is %s after a successful login, want argon2id", got) + } + + after := decodeReport(t, f.report(t, f.adminToken, "")) + if after.Data.UpgradedEvents != 1 { + t.Errorf("upgraded_events = %d, want 1", after.Data.UpgradedEvents) + } + if after.Data.UpgradedEventsInWindow != 1 { + t.Errorf("upgraded_events_in_window = %d, want 1 — the upgrade just happened", after.Data.UpgradedEventsInWindow) + } + if after.Data.EstimatedRemaining != 1 { + t.Errorf("estimated_remaining = %d, want 1 (two users, one upgraded)", after.Data.EstimatedRemaining) + } + if after.Data.TotalUsers != 2 { + t.Errorf("total_users = %d, want 2 — a login must not change the user total", after.Data.TotalUsers) + } +} + +// The hasher block describes what this deployment would WRITE, which is +// configuration and not anything read back per user. Both branches are +// pinned because the bcrypt one is the default every existing deployment +// is on. +func TestHashMigrationReportsTheConfiguredHasher(t *testing.T) { + t.Run("argon2id", func(t *testing.T) { + params := security.Argon2idParams{Memory: 32768, Iterations: 2, Parallelism: 2, SaltLength: 16, KeyLength: 32} + f := newSecurityFixture(t, config.Config{PasswordHasher: config.PasswordHasherArgon2id, Argon2idParams: params}) + report := decodeReport(t, f.report(t, f.adminToken, "")) + + if report.Data.Hasher.Algorithm != config.PasswordHasherArgon2id { + t.Errorf("algorithm = %q, want argon2id", report.Data.Hasher.Algorithm) + } + // The configured values, not the defaults — the whole reason the + // report is allowed to state them. + if report.Data.Hasher.MemoryKiB != params.Memory || report.Data.Hasher.Iterations != params.Iterations || report.Data.Hasher.Parallelism != params.Parallelism { + t.Errorf("hasher = %+v, want the configured %+v", report.Data.Hasher, params) + } + }) + + t.Run("bcrypt", func(t *testing.T) { + f := newSecurityFixture(t, config.Config{PasswordHasher: config.PasswordHasherBcrypt}) + rec := f.report(t, f.adminToken, "") + report := decodeReport(t, rec) + + if report.Data.Hasher.Algorithm != config.PasswordHasherBcrypt { + t.Errorf("algorithm = %q, want bcrypt", report.Data.Hasher.Algorithm) + } + // The cost fields are omitted rather than sent as zeroes a reader + // would have to know to ignore — so this asserts on the raw JSON, + // which is where that decision is actually observable. + if got := rec.Body.String(); jsonHasKey(t, got, "memory_kib") { + t.Errorf("bcrypt report carries memory_kib: %s", got) + } + }) +} + +// A count that can exceed the user total, and an estimate that can go +// negative, are both handled rather than papered over: the fields say +// what they are, and remaining is floored. This drives the event count +// past the user total directly, which is the state a second cost increase +// a year later actually produces. +func TestHashMigrationFloorsEstimatedRemaining(t *testing.T) { + f := newSecurityFixture(t, config.Config{ + PasswordHasher: config.PasswordHasherArgon2id, + Argon2idParams: security.DefaultArgon2idParams, + }) + + for i := 0; i < 3; i++ { + if err := f.audit.Record(context.Background(), store.AuditEvent{ + Type: store.EventPasswordHashUpgraded, + UserID: f.subjectID, + }); err != nil { + t.Fatalf("recording an upgrade event: %v", err) + } + } + + report := decodeReport(t, f.report(t, f.adminToken, "")) + if report.Data.UpgradedEvents != 3 { + t.Errorf("upgraded_events = %d, want 3", report.Data.UpgradedEvents) + } + // Three events, two users. The honest answer is "no users remain", + // not minus one. + if report.Data.EstimatedRemaining != 0 { + t.Errorf("estimated_remaining = %d, want 0 — a negative remainder must be floored", report.Data.EstimatedRemaining) + } +} + +// window_days is bounded and rejected rather than clamped: a caller that +// asked for 5000 days and got 365 back has no way to tell that from a +// window that happens to hold the same count. +func TestHashMigrationWindowBounds(t *testing.T) { + f := newSecurityFixture(t, config.Config{ + PasswordHasher: config.PasswordHasherArgon2id, + Argon2idParams: security.DefaultArgon2idParams, + }) + + report := decodeReport(t, f.report(t, f.adminToken, "?window_days=30")) + if report.Data.WindowDays != 30 { + t.Errorf("window_days = %d, want the requested 30", report.Data.WindowDays) + } + + for _, query := range []string{"?window_days=0", "?window_days=366", "?window_days=soon"} { + rec := f.report(t, f.adminToken, query) + if rec.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400 (body %s)", query, rec.Code, rec.Body.String()) + } + } +} + +// A router built without the stores answers 404 rather than 500: that is +// a wiring fact, not a server fault, and it is the same shape every other +// unconfigured feature in this API uses. +func TestHashMigrationWithoutStoresIsNotFound(t *testing.T) { + ctx := context.Background() + var adminID string + engine := newTestEngineWithClaims(t, token.ClaimsFunc(func(_ context.Context, userID string) (map[string]any, error) { + if userID == adminID { + return map[string]any{"role": "admin"}, nil + } + return nil, nil + })) + + admin, err := cryden.SignUp(ctx, engine, "operator@example.com", testPassword, "203.0.113.1") + if err != nil { + t.Fatalf("signup: %v", err) + } + adminID = admin.ID + tokens, err := cryden.Login(ctx, engine, "operator@example.com", testPassword, "203.0.113.1", chromeOnMacOS) + if err != nil { + t.Fatalf("login: %v", err) + } + + // Deliberately no Audit or Users, which is the deployment this repo + // cannot yet rule out: the engine builds fine without them being + // handed to the router. + router := NewRouter(Deps{Engine: engine, Config: config.Config{}}) + req := httptest.NewRequest(http.MethodGet, "/v1/admin/security/hash-migration", nil) + req.Header.Set("Authorization", "Bearer "+tokens.AccessToken) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404 (body %s)", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "not_configured") { + t.Errorf("body = %s, want the not_configured code", rec.Body.String()) + } +} + +// The second admin route in this repo, and the first that reads anything. +// It is behind the same gate as the OAuth health report, and a read +// endpoint being harmless is not a reason to widen it. +func TestHashMigrationRouteIsGatedByRequireAdmin(t *testing.T) { + f := newSecurityFixture(t, config.Config{ + PasswordHasher: config.PasswordHasherArgon2id, + Argon2idParams: security.DefaultArgon2idParams, + }) + + if rec := f.report(t, "", ""); rec.Code != http.StatusUnauthorized { + t.Errorf("no token: status = %d, want 401", rec.Code) + } + if rec := f.report(t, "not-a-real-token", ""); rec.Code != http.StatusUnauthorized { + t.Errorf("garbage token: status = %d, want 401", rec.Code) + } + + // An ordinary end user's token carries no role claim at all, so this + // is the 403 every flavour of "not an operator" gets. + userTokens, err := cryden.Login(context.Background(), f.engine, f.subjectEmail, testPassword, "203.0.113.2", chromeOnMacOS) + if err != nil { + t.Fatalf("login (subject): %v", err) + } + rec := f.report(t, userTokens.AccessToken, "") + if rec.Code != http.StatusForbidden { + t.Fatalf("ordinary user: status = %d, want 403 (body %s)", rec.Code, rec.Body.String()) + } + + if rec := f.report(t, f.adminToken, ""); rec.Code != http.StatusOK { + t.Errorf("operator: status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } +} + +// jsonHasKey reports whether a JSON object in body has the given key at +// the top level of data, used to assert on fields that are omitted +// entirely rather than sent as zero. +func jsonHasKey(t *testing.T, body, key string) bool { + t.Helper() + var decoded map[string]json.RawMessage + if err := json.Unmarshal([]byte(body), &decoded); err != nil { + t.Fatalf("decoding %s: %v", body, err) + } + var data map[string]json.RawMessage + if err := json.Unmarshal(decoded["data"], &data); err != nil { + t.Fatalf("decoding data from %s: %v", body, err) + } + var hasher map[string]json.RawMessage + if err := json.Unmarshal(data["hasher"], &hasher); err != nil { + t.Fatalf("decoding hasher from %s: %v", body, err) + } + _, present := hasher[key] + return present +} diff --git a/main.go b/main.go index feda995..308205a 100644 --- a/main.go +++ b/main.go @@ -37,6 +37,15 @@ func main() { operators := operator.NewStore(db) + // Hoisted into locals rather than constructed inline in the config + // literal below, because the router needs these same two instances: + // GET /v1/admin/security/hash-migration counts what the engine wrote, + // and counting a different store object — or a second connection pool + // with its own snapshot — is how that report would quietly disagree + // with the engine it is reporting on. + users := postgres.NewUserStore(db) + audit := postgres.NewAuditStore(db) + // Email templates are optional and entirely this repo's: cryden owns // no message copy. An unset EMAIL_TEMPLATE_DIR leaves both senders // printing their own built-in line, byte for byte as before. @@ -51,9 +60,9 @@ func main() { engineCfg := cryden.Config{ JWTSecret: cfg.JWTSecret, - Users: postgres.NewUserStore(db), + Users: users, Sessions: postgres.NewSessionStore(db), - Audit: postgres.NewAuditStore(db), + Audit: audit, Verifications: postgres.NewVerificationStore(db), EmailSender: &consoleEmailSender{Templates: emailTemplates}, // dev stand-in — see email_sender.go MagicLinkSender: &consoleMagicLinkSender{BaseURL: cfg.BaseURL, Templates: emailTemplates}, // dev stand-in — see email_sender.go @@ -172,6 +181,8 @@ func main() { Engine: engine, DB: db, Config: cfg, + Audit: audit, + Users: users, }) limiter := httpapi.NewEdgeRateLimiter(cfg.EdgeRateLimit, cfg.EdgeRateLimitWindow) handler := httpapi.WithCORS(cfg.CORSOrigins, httpapi.WithEdgeRateLimit(limiter, router)) From eb5e9f81a66c2dc302743a66137e8a5dff1a0199 Mon Sep 17 00:00:00 2001 From: Raymond Nicholas Date: Tue, 15 Sep 2026 12:34:33 +0100 Subject: [PATCH 05/10] docs: document Tier 3 Stage 1 config and endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README and openapi/spec.yaml cover the new env vars, the API key endpoints and the hash-migration report — including why upgraded_events counts events rather than users. PROGRESS.md records the Stage 1 verification run and the four commits it landed as. Co-Authored-By: Claude Code --- README.md | 43 +++++- docs/development/PROGRESS.md | 167 +++++++++++++++++++++++ openapi/spec.yaml | 251 ++++++++++++++++++++++++++++++++++- 3 files changed, 459 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4a01456..fc9b0b2 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,12 @@ POST /v1/login/passkey/begin (completes a paused login) POST /v1/login/passkey/finish (completes a paused login) POST /v1/login/recovery-code (completes a paused login) +POST /v1/api-keys (auth required, raw key returned once) +GET /v1/api-keys (auth required) +DELETE /v1/api-keys/{keyID} (auth required) + GET /v1/admin/oauth/health (admin required) +GET /v1/admin/security/hash-migration (admin required) ``` `GET /v1/sessions` answers with *named* sessions: each entry keeps its `id`, `ip`, `user_agent` and `created_at`, and gains `label`, `device` and `location`, all computed on read from the session's own IP and User-Agent — nothing new is stored and no migration exists for it. `label` is the string a "your devices" screen shows (`Chrome on macOS`, or `Unknown device` for a client that sent no User-Agent). `location` is present but empty unless a geolocator is configured, and this repo wires none on purpose: every implementation of that interface calls somebody else's internet service, which is a deployment's decision rather than this repo's. The response shape is documented in `openapi/spec.yaml`. @@ -205,10 +210,46 @@ Because the claim is baked in at issue time, a grant takes effect on that user's Probes run concurrently with a 5-second timeout each, carry no OAuth parameters and cannot start or complete a login. This is api-side logic: cryden knows whether a provider is configured, not whether it is reachable. +`GET /v1/admin/security/hash-migration` reports how far a password-hash migration has got: + +```json +{"data": { + "hasher": {"algorithm": "argon2id", "memory_kib": 65536, "iterations": 3, "parallelism": 4}, + "total_users": 1234, + "upgraded_events": 900, + "estimated_remaining": 334, + "window_days": 7, + "upgraded_events_in_window": 120 +}} +``` + +Set `PASSWORD_HASHER=argon2id` and every login whose stored hash is out of date gets rewritten with Argon2id — that gradual rewrite is the migration, and there is no separate command to run. This endpoint only watches it: + +- `hasher` is what this deployment is configured to **write**, read from config, not from the users table. cryden deliberately exposes no bulk way to inspect stored hash algorithms, and adding one would be the engine's job rather than this repo's. +- `upgraded_events` counts **events**, not users. A user whose hash is rewritten twice — a second cost increase a year later — contributes two, so this number can exceed `total_users`. +- `estimated_remaining` is therefore *estimated*, floored at zero, and is `total_users - upgraded_events`. +- `upgraded_events_in_window` is the field that actually answers "is this draining": the all-time count only ever rises, while a windowed one falls to zero as the last stragglers log in. `window_days` (1–365, default 7) sets that window. + +## API keys + +`POST /v1/api-keys` mints a machine-to-machine credential for the calling user and returns the raw key **once** — cryden stores only its SHA-256 hash and can never reproduce it, so a caller that loses it has to mint a new one. The response carries the raw key, the stored record (`id`, `name`, `prefix`, `scopes`, `expires_at`, `expired`, `created_at`, `last_used_at`) and a `notice` saying so; a client that renders the key without that notice is the failure this guards against. + +```json +POST /v1/api-keys {"name": "ci deploy", "scopes": ["read"], "expires_in_days": 90} +``` + +- `expires_in_days` of `0` or absent means the key never expires, which is cryden's own default and the honest one for a credential living in a deploy pipeline's environment: revocation, not expiry, is what actually stops a key. +- `GET /v1/api-keys` lists the calling user's live keys. Revoked keys are absent; expired-but-unrevoked ones are present with `"expired": true`, because "your CI key expired on Tuesday" is exactly what someone needs to see to understand why a pipeline broke. +- `DELETE /v1/api-keys/{keyID}` revokes, irreversibly — the reason a key gets revoked is that somebody else may have it, so mint a new one rather than offering an un-revoke. + +Every one of these is scoped to the calling user by cryden itself, which derives the user ID from the verified token rather than from the request. A key belonging to another account, a key that does not exist, and an already-revoked key all answer the same `404 api_key_not_found` — a caller can never learn whether somebody else's key exists. + +**No endpoint in this repo authenticates *with* an API key yet.** These three manage them; cryden's `auth.AuthenticateAPIKey` is the other half, and wiring it into a `RequireAPIKey` middleware is a separate change. + ## Design notes - `CORS_ORIGINS` is required, no wildcard default — an API handling auth tokens should never allow every origin. -- `consoleEmailSender` (in `email_sender.go`) is a dev stand-in — logs verification tokens to the console instead of sending real email. Replace with a real provider (Resend, SES, SendGrid) before real users depend on email verification. +- `consoleEmailSender` (in `email_sender.go`) is a dev stand-in — logs verification tokens to the console instead of sending real email. Replace with a real provider (Resend, SES, SendGrid) before real users depend on email verification. Set `EMAIL_TEMPLATE_DIR` and it renders your own `verification.txt` / `magic_link.txt` (`text/template`, fields `{{.To}}`, `{{.Token}}`, `{{.URL}}`) instead of its built-in line; cryden owns no message copy on purpose, so templates are entirely this repo's — see `templates/`. - Every engine error is mapped to a stable `(status, code)` pair in `httpapi/errors.go` — add new engine errors there once, every handler benefits. `*auth.ErrOAuthEmailConflict` is the one non-sentinel case in that file (it's a struct carrying `Email`/`Provider`, unwrapped via `errors.As` rather than `errors.Is`). - The OAuth linking flow's HMAC-signed cookie (`oauth_handlers.go`) is genuinely new plumbing, not copied from an existing pattern elsewhere in this repo — worth reading closely if you're touching that code, not just trusting it because it compiles. - A paused login is a `200`, not an error: nothing failed, the caller just has one more step. `httpapi/second_factor.go` is the one place that response shape is written. diff --git a/docs/development/PROGRESS.md b/docs/development/PROGRESS.md index 9638154..174ad61 100644 --- a/docs/development/PROGRESS.md +++ b/docs/development/PROGRESS.md @@ -323,3 +323,170 @@ Next: Tier 3, on its own branch per `CODEX.md`. Still owed from before it: the first DB-backed smoke-test run, now worth doing against a `REDIS_URL`-less and a `REDIS_URL`-set instance so the shared limiter gets its first real exercise. + +## 2026-09-15 — Tier 3, Stage 1 (config, API keys, hash migration) + +Tier 3 was split into two stages with an explicit mid-point check-in. +Stage 1 is written; Stage 2 (per-user metadata + JWT claims, webhooks + +delivery log, shipped-events log) has not been started. + +Most of this session had **no command execution at all**. Every +command-executing tool — `Bash`, `Monitor`, and subagents alike — +failed with `deepseek-v4-flash is temporarily unavailable, so auto mode +cannot determine the safety of ...`. Only file reads and writes worked. +So Stage 1 was written, and its cryden symbols verified by reading the +module cache, without a single build. This is a *different* failure from +the `bwrap`/`apply_patch` breakage the Tier 2 entry describes: the Go +1.25.0 toolchain and the full module cache were still here and still +working — what was unavailable was command execution, not the toolchain. + +Command execution came back at the end of the session, and everything +was then actually run on `feat/tier3-config-and-endpoints` (branch +created once git was reachable): + +``` +go build ./... clean +go vet ./... clean +gofmt -l . empty, after two fixes (below) +go test -count=1 ./... + ok github.com/crydensync/api/config 0.012s + ok github.com/crydensync/api/httpapi 6.129s + ok github.com/crydensync/api/templates 0.008s +``` + +Every Tier 3 test was also confirmed passing individually, including +`TestHashMigrationTracksARealBcryptToArgon2idUpgrade`, which drives the +real upgrade path (sign up on Argon2id, overwrite the stored hash with +bcrypt, log in, assert the engine's own rewrite is reported) rather +than a hand-seeded audit row. + +Stage 1 landed as four commits on that branch, each verified on its own +(`go build`/`go vet`/`gofmt -l`/`go test -count=1` after every one, not +only the last): the `Deps` refactor, then config + templates, then the +API key endpoints, then the hash-migration report. Splitting them meant +reconstructing intermediate states of `main.go`, `httpapi/router.go` and +`httpapi/errors.go`, which each carry hunks belonging to more than one +commit — `git add -p` is unavailable in this environment, so each +file's part-way state was written, built and committed in order. The +final state of all three was diffed against the version the full-suite +run above covered; the only difference is one reworded doc comment in +`router.go`, and that exact tree was rebuilt and retested. + +`gofmt` was the one place the reasoning-first approach was actually +wrong, and it is worth recording which half: hand-reasoned struct field +alignment was **correct** — none of `apiKeyDTO`, `hasherDTO`, +`hashMigrationDTO` or `templates.Data` were flagged — but two spots +nobody had considered were: a `map[string]any` literal in +`apikey_handlers.go` whose keys needed aligning, and `main.go`'s +trailing `// dev stand-in` comments, which align against the longest +line in their group. Both fixed with `gofmt -w`. + +**What was checked before the toolchain was reachable** — since +`CODEX.md`'s rule is to say what was and was not done rather than to +imply a build — every cryden symbol Stage 1 calls was read directly out +of the module cache at `…/cryden/v2@v2.5.0`, first-hand, not recalled. +Confirmed: + +- `cryden.GenerateAPIKey(ctx, e, userID, name, scopes, ttl)`, + `ListAPIKeys(ctx, e, userID)`, `RevokeAPIKey(ctx, e, userID, keyID)` + all exist as root-package facade functions (`cryden.go`), with + exactly the shapes the handlers call. `NEXT.md`'s Tier 3 spec names + them the same way, so the spec was accurate here. +- `cryden.APIKey` is a **root-package** struct (`ID`, `Name`, `Prefix`, + `Scopes`, `ExpiresAt *time.Time`, `CreatedAt`, `LastUsedAt`) with an + `Expired()` method — *not* `store.APIKey`, which is the storage-side + record and does carry `KeyHash`. The handler's DTO is built from the + public one, which is what makes "no key hash can be marshalled by + accident" structural rather than a rule to remember. +- `cryden.ErrAPIKeysNotConfigured` exists (`cryden.go`), and + `auth.ErrInvalidAPIKey` / `ErrAPIKeyNotFound` / `ErrInvalidAPIKeyScope` + / `ErrInvalidAPIKeyTTL` exist with the messages mapped in + `httpapi/errors.go`. +- `auth.apiKeyPrefixFragment` builds the stored `Prefix` as + `"ck_" + first 8 chars of the secret` — so it is `ck_9f3a1c02`, not + the bare label. A first draft of the API-key test asserted `== "ck"` + and would have failed; found by reading the engine's implementation + and fixed before any run. +- `logger.ParseLevel` is case- and whitespace-insensitive, accepts + `warning`/`err` as well as `warn`/`error`, and returns the zero + `Level` with `ErrUnknownLevel` on a miss — the doc comment on it is + explicit that neither defaulting direction is acceptable. +- `logger`'s own package doc pins the intended composition for Stage 2 + as `NewMultiLogger(NewConsoleJSONLogger(), NewLevelFilter(NewMaskingRedactor(sink), level))` + — redaction *inside* the fan-out, so local stdout keeps the IP and + only the outbound copy loses it. + +Stage 1, by file: + +- `httpapi/router.go` — `NewRouter(engine, db, cfg)` becomes + `NewRouter(Deps)`, since Tier 3's admin endpoints need store instances + `cryden.Engine` keeps unexported. Two call sites: `main.go` and + `oauth_health_test.go`. Handlers guard a nil store and answer + `404 not_configured`. +- `config/config.go` — `PASSWORD_HASHER`, the five `ARGON2ID_*` knobs, + `API_KEY_PREFIX`, `LOG_LEVEL`, `CLOUD_LOGGING`, + `CLOUD_LOG_REDACTION`, `CLOUD_LOG_HASH_KEY`, `EMAIL_TEMPLATE_DIR`, + plus `envString`/`envUint32`/`envUint8`. The unsigned readers exist so + a minus sign is a startup failure rather than a value that wraps to + 255 lanes. +- `templates/` (new) — `text/template` over `verification.txt` / + `magic_link.txt`, fields `{{.To}} {{.Token}} {{.URL}}`. Each message + falls back independently; a directory with neither file, or an + unparseable one, is a startup failure. +- `email_sender.go` — both console senders render a configured template + when there is one and print their original line byte-for-byte when + there is not. +- `httpapi/apikey_handlers.go` + test — the three routes, one-time raw + key with a notice, bounded `expires_in_days`, scoped revoke. +- `httpapi/security_handlers.go` + test — + `GET /v1/admin/security/hash-migration`, behind `RequireAdmin`. +- `httpapi/errors.go`, `query.go`, `response.go`, `main.go`, + `.env.example`, `README.md`, `openapi/spec.yaml` (1.2). + +Two test bugs were found and fixed by reading rather than by a red test, +worth recording because neither would have been caught by a type check: +the `"ck"` prefix assertion above, and +`TestAPIKeyRevokeIsScopedToTheCallingUser` originally built its two +accounts on **separate engines**, so its 404 came from a key that simply +was not in that store — proving nothing about the `WHERE id = $1 AND +user_id = $2` predicate it claimed to test. Both accounts now share one +engine. + +Decisions and assumptions, none blocking: + +- **The hash-migration report's field names are load-bearing.** + `upgraded_events` counts events, so it can exceed `total_users` after + a second cost increase; `estimated_remaining` is therefore *estimated* + and floored at zero. Naming it `remaining` would be a number an + operator trusts more than they should. README and spec both say so in + prose. +- **Stage 1 ships cloud-logging config with nothing reading it yet.** + The user's own staging put "cloud-logger config" in Stage 1 and the + shipped-events log in Stage 2, so `CloudLogging`/`LogLevel`/ + `CloudLogRedaction` are parsed and validated now and composed in + `main.go` when Stage 2 lands. Since both stages land on one branch + before any merge, no release ever sees the dead switch — but a + reviewer reading Stage 1 alone will notice it, so it is said here. +- **`BCRYPT_COST` was not added.** A real engine knob with no env var + here, but Tier 3 asks for Argon2id; flagged rather than taken as + scope. +- **The Argon2id params are always assembled**, even when the selected + hasher is bcrypt, so the report can state what the deployment is + configured to write without rebuilding that answer from a second + place. + +Noticed while working, not fixed: + +- **`openapi/spec.yaml` still predates Tier 1**, unchanged from the Tier + 2 note. Stage 1 added only its own paths on top of that gap. +- **This repo still has no graceful shutdown.** `main.go` ends at + `log.Fatal(http.ListenAndServe(...))`. Stage 2's webhook worker wants + a context it can be stopped with, so the worker takes one and gets + `context.Background()` — introducing real shutdown is its own change + touching every component, and smuggling it in behind a worker would + not be honest about its size. + +Next: Stage 2, once someone can run a build. The check-in the user +asked for is the point at which this entry was written; Stage 1's code +should be built, vetted, formatted and tested before Stage 2 starts on +top of it. diff --git a/openapi/spec.yaml b/openapi/spec.yaml index 812ae86..2b611af 100644 --- a/openapi/spec.yaml +++ b/openapi/spec.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: CrydenSync API - version: "1.1" + version: "1.2" description: > A self-hosted HTTP wrapper around the CrydenSync auth engine. Every response follows one of two envelope shapes: {"data": ...} @@ -14,6 +14,10 @@ info: already returned keep their names and types, so a client reading only those is unaffected; a client that validates the object against an exhaustive schema is not. + + 1.2 is additive: API key management (POST/GET /api-keys and + DELETE /api-keys/{keyID}) plus GET /admin/security/hash-migration. + No existing path, field or status code changed. servers: - url: http://localhost:8080/v1 description: Local dev @@ -102,6 +106,90 @@ components: code: { type: string } message: { type: string } + APIKey: + type: object + description: > + A machine-to-machine credential belonging to one user. Note what + is absent: no field here can reproduce the key itself. The raw + value is returned exactly once, by POST /api-keys, and only its + SHA-256 hash is stored. + properties: + id: { type: string, format: uuid, description: What DELETE /api-keys/{keyID} takes. } + name: { type: string, description: Host-supplied label; not required to be unique or non-empty. } + prefix: + type: string + description: > + The leading, non-secret fragment of the raw key + ("ck_a1b2c3d4"), stored in the clear so a UI can say which + key is which without holding the key. Not a lookup key. + scopes: + type: array + items: { type: string } + description: > + Host-defined permission strings, stored and returned + verbatim — the engine never interprets one. Always an array, + never null, so a client can iterate without a nil check. No + endpoint in this API enforces them yet. + expires_at: + type: string + format: date-time + nullable: true + description: Null for a key that never expires, which is the default. + expired: + type: boolean + description: > + The server's own answer to "is this still usable" — the same + comparison made when the key is used, so the two can never + disagree. Sent rather than left for a client to derive from + expires_at. + created_at: { type: string, format: date-time } + last_used_at: + type: string + format: date-time + nullable: true + description: > + Null until the key is first used. Written at most once every + five minutes, so it answers "is anything still using this?" + rather than "when exactly was the last request". + + HashMigration: + type: object + description: > + How far a password-hash migration has got, counted from the + engine's own audit events against the user total. Read-only. + properties: + hasher: + type: object + description: > + What this deployment is configured to WRITE — read from + config, never from the users table. The engine exposes no + bulk way to inspect stored hash algorithms. + properties: + algorithm: { type: string, enum: [bcrypt, argon2id] } + memory_kib: { type: integer, description: Omitted entirely for bcrypt. } + iterations: { type: integer, description: Omitted entirely for bcrypt. } + parallelism: { type: integer, description: Omitted entirely for bcrypt. } + total_users: { type: integer } + upgraded_events: + type: integer + description: > + Counts EVENTS, not users. A user whose hash is rewritten + twice — a second cost increase a year later — contributes + two, so this number can exceed total_users. + estimated_remaining: + type: integer + description: > + total_users - upgraded_events, floored at zero. Estimated: + it is only a distinct-user count if every user upgraded + exactly once, which is the normal case and not a guarantee. + window_days: { type: integer, description: 1-365, default 7. } + upgraded_events_in_window: + type: integer + description: > + The field that answers "is this draining" — the all-time + count only ever rises, while a windowed one falls to zero as + the last stragglers log in. + responses: BadRequest: description: Malformed request body @@ -397,6 +485,167 @@ paths: '200': { description: Healthy } '503': { description: Database unreachable } + /api-keys: + post: + summary: Mint an API key for the authenticated user + description: > + Returns the raw key exactly once. Only its SHA-256 hash is + stored, so a caller that loses it must mint a new one — which + is what the `notice` field in the response says, and why a + client must render it. Scoped to the calling user by the engine + itself, from the verified token and never from the request. + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + name: { type: string, description: Label for the key; 100 characters or fewer. } + scopes: + type: array + items: { type: string } + description: Host-defined permission strings, stored verbatim. The engine never interprets them. + expires_in_days: + type: integer + minimum: 0 + maximum: 3650 + default: 0 + description: > + 0 or absent means the key never expires. Bounded so + that the conversion to a duration cannot overflow + into a negative TTL and come back as a confusing + invalid_api_key_ttl. + responses: + '201': + description: The key was created + content: + application/json: + schema: + type: object + properties: + data: + type: object + properties: + key: { type: string, description: The raw key. This is the only time it is ever returned. } + notice: { type: string, description: Human-readable warning that the key cannot be retrieved later. } + api_key: { $ref: '#/components/schemas/APIKey' } + '400': + description: > + bad_request for a malformed body, name over 100 characters, + or expires_in_days outside 0-3650; invalid_api_key_scope for + a scope that is empty or contains whitespace. + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } + '401': { $ref: '#/components/responses/Unauthorized' } + '404': + description: api_keys_not_configured — the engine has no API key store on this deployment. + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } + + get: + summary: List the authenticated user's API keys + description: > + Live keys only. Revoked keys are absent; expired-but-unrevoked + ones ARE included with "expired": true, because "it expired on + Tuesday" is the answer to why a pipeline broke. The raw key + never appears here or anywhere else. + security: [{ bearerAuth: [] }] + responses: + '200': + description: The calling user's keys + content: + application/json: + schema: + type: object + properties: + data: + type: array + items: { $ref: '#/components/schemas/APIKey' } + '401': { $ref: '#/components/responses/Unauthorized' } + '404': + description: api_keys_not_configured — the engine has no API key store on this deployment. + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } + + /api-keys/{keyID}: + delete: + summary: Revoke an API key + description: > + Irreversible — the reason a key gets revoked is that somebody + else may have it, so mint a new one rather than offering an + un-revoke. Ownership is enforced in the store statement, so a + key belonging to another account, a key that does not exist and + an already-revoked key all answer the same 404. + security: [{ bearerAuth: [] }] + parameters: + - name: keyID + in: path + required: true + schema: { type: string, format: uuid } + responses: + '200': + description: Revoked + content: + application/json: + schema: + type: object + properties: + data: + type: object + properties: + status: { type: string, example: api key revoked } + '401': { $ref: '#/components/responses/Unauthorized' } + '404': + description: api_key_not_found, or api_keys_not_configured when the deployment has no API key store. + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } + + /admin/security/hash-migration: + get: + summary: Progress of a password-hash migration + description: > + Admin only — an operator's token. Reports the engine's own + password_hash_upgraded audit events against the user total, + which is the only way to watch a bcrypt-to-Argon2id migration + drain: switching PASSWORD_HASHER rewrites one stored hash per + successful login, and there is no separate command to run. + Read-only by construction — it calls two count methods and + records nothing. + security: [{ bearerAuth: [] }] + parameters: + - name: window_days + in: query + required: false + schema: { type: integer, minimum: 1, maximum: 365, default: 7 } + description: > + The reporting window. Out of range or non-numeric is a 400, + not a silent clamp — a caller asking for 5000 and getting + 365 back has no way to tell that from a window that happens + to hold the same count. + responses: + '200': + description: The report + content: + application/json: + schema: + type: object + properties: + data: { $ref: '#/components/schemas/HashMigration' } + '400': { $ref: '#/components/responses/BadRequest' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': + description: not_configured — the router was built without the stores this report reads. + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } + /admin/oauth/health: get: summary: Reachability of each OAuth provider this API knows about From d43a73d125766227e26230e89cb5fcc9e9f53d08 Mon Sep 17 00:00:00 2001 From: Raymond Nicholas Date: Tue, 15 Sep 2026 12:58:21 +0100 Subject: [PATCH 06/10] feat: add per-user metadata with JWT claim mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds user_metadata (migrations/009) and the usermeta package: cryden's store.User has no metadata concept, so per CLAUDE.md's ownership rule the table and its rules are this repo's own. The store owns key validation, not the handler, because a metadata key is a claim name: a key of "sub" would not be ignored at login, it would fail the login, and a key of "role" would mint an operator token for a user the operators table has never heard of — access operator.Store.Revoke could not take away. Both are refused where an operator can still see why. Admin endpoints behind RequireAdmin, one key per call so no read-modify-write race exists for a console to lose: GET /v1/admin/users/{userID}/metadata PUT /v1/admin/users/{userID}/metadata/{key} DELETE /v1/admin/users/{userID}/metadata/{key} All three answer the same body, so a save is also the refresh, and the reserved names are listed so a claim-mapping UI can grey them out. usermeta.ClaimsProvider owns the merge into Config.AccessTokenClaims — two queries per login and per refresh, which is the price of claims that are current rather than frozen at signup. Co-Authored-By: Claude Code --- httpapi/errors.go | 12 + httpapi/metadata_handlers.go | 185 +++++++++++ httpapi/metadata_handlers_test.go | 459 ++++++++++++++++++++++++++ httpapi/router.go | 22 ++ main.go | 36 +- migrations/009_user_metadata.down.sql | 3 + migrations/009_user_metadata.up.sql | 31 ++ usermeta/claims.go | 73 ++++ usermeta/claims_test.go | 159 +++++++++ usermeta/memory.go | 96 ++++++ usermeta/store.go | 228 +++++++++++++ usermeta/store_test.go | 213 ++++++++++++ 12 files changed, 1501 insertions(+), 16 deletions(-) create mode 100644 httpapi/metadata_handlers.go create mode 100644 httpapi/metadata_handlers_test.go create mode 100644 migrations/009_user_metadata.down.sql create mode 100644 migrations/009_user_metadata.up.sql create mode 100644 usermeta/claims.go create mode 100644 usermeta/claims_test.go create mode 100644 usermeta/memory.go create mode 100644 usermeta/store.go create mode 100644 usermeta/store_test.go diff --git a/httpapi/errors.go b/httpapi/errors.go index 8782e0a..63e6451 100644 --- a/httpapi/errors.go +++ b/httpapi/errors.go @@ -9,6 +9,8 @@ import ( "github.com/crydensync/cryden/v2/auth" "github.com/crydensync/cryden/v2/store" "github.com/crydensync/cryden/v2/token" + + "github.com/crydensync/api/usermeta" ) // apiError is the (status, code, message) triple every handler @@ -139,6 +141,16 @@ func mapError(err error) apiError { return apiError{http.StatusBadRequest, "invalid_api_key_scope", "a scope must be non-empty and contain no whitespace"} case errors.Is(err, auth.ErrInvalidAPIKeyTTL): return apiError{http.StatusBadRequest, "invalid_api_key_ttl", "expiry cannot be in the past"} + // Per-user metadata. The two 400s are the reserved-claim rule reaching + // the client: a key of "sub" would not be ignored at login, it would + // fail the login, so it is refused where an operator can still see + // why. "role" is refused with it — see usermeta.RoleClaim. + case errors.Is(err, usermeta.ErrReservedKey): + return apiError{http.StatusBadRequest, "reserved_metadata_key", "that key names a claim this deployment sets itself, so it cannot be mapped as metadata"} + case errors.Is(err, usermeta.ErrInvalidKey): + return apiError{http.StatusBadRequest, "invalid_metadata_key", "a metadata key must start with a letter or underscore and contain only letters, digits, underscores, dots and dashes, up to 64 characters"} + case errors.Is(err, usermeta.ErrNotFound): + return apiError{http.StatusNotFound, "metadata_key_not_found", "no such metadata key on this user"} // The five "not configured" sentinels below mean this deployment has // not enabled that feature, not that the caller did anything wrong. // 404 rather than 500 so a client can hide the option instead of diff --git a/httpapi/metadata_handlers.go b/httpapi/metadata_handlers.go new file mode 100644 index 0000000..dedc1cd --- /dev/null +++ b/httpapi/metadata_handlers.go @@ -0,0 +1,185 @@ +package httpapi + +import ( + "encoding/json" + "net/http" + + "github.com/crydensync/cryden/v2/store" + + "github.com/crydensync/api/usermeta" +) + +// MetadataHandlers answers the admin per-user metadata endpoints — the +// table behind JWT claim mapping. +// +// Every route is behind RequireAdmin (router.go). Unlike the two reports +// beside them on the admin surface, these three are not read-only: PUT +// and DELETE are writes, and CLAUDE.md's read-only rule is about the +// AI-assisted tooling, not about the console's own settings. What that +// rule does mean here is that a write is always an explicit operator +// action on a named key — nothing on this surface may take a suggestion +// and apply it by itself. +type MetadataHandlers struct { + // Users is cryden's own user store, the same instance main.go handed + // the engine. It is here only to answer "does this user exist", so an + // unknown or malformed id is a 404 rather than a foreign-key failure + // out of Postgres surfacing as a 500. + Users store.UserStore + + // Meta is this repo's own store — see usermeta. It owns the rule + // about which keys may exist, so nothing here re-checks a key. + Meta usermeta.Store +} + +// metadataDTO is the shape all three endpoints answer with, so a console +// can hold one parser: PUT and DELETE return the same body GET does +// rather than a status message, which means a save is also the refresh. +type metadataDTO struct { + UserID string `json:"user_id"` + Metadata map[string]any `json:"metadata"` + + // ReservedClaimNames is included so a claim-mapping UI can grey these + // out rather than let an operator discover the rule by being + // rejected. It is both halves of the rule — the seven registered JWT + // names, and this api's own "role" (see usermeta.RoleClaim, which is + // the one with teeth). + ReservedClaimNames []string `json:"reserved_claim_names"` +} + +// List — admin required. Returns every key set on the user. +func (h *MetadataHandlers) List(w http.ResponseWriter, r *http.Request, userID string) { + if !h.ready(w, r, userID) { + return + } + h.writeMetadata(w, r, userID) +} + +// Put — admin required. Sets one key, creating it or replacing it. +// +// The body is {"value": }. A key is set on its own rather than +// the whole map being replaced, so two operators editing different fields +// of the same user cannot overwrite each other's work — the classic +// read-modify-write a "PUT the whole object" endpoint invites. +// +// The key is validated by the store, not here (see usermeta.ValidateKey): +// the reserved-claim rule is a property of the data, so it holds for +// every writer rather than for the writers that happen to know about it. +func (h *MetadataHandlers) Put(w http.ResponseWriter, r *http.Request, userID, key string) { + if !h.ready(w, r, userID) { + return + } + + // RawMessage rather than any, because "value" absent and "value": null + // have to be told apart: null is a legitimate value to store, and a + // missing field is a client bug. Decoding into an `any` collapses + // both to nil. + var req struct { + Value json.RawMessage `json:"value"` + } + if err := decodeJSON(r, &req); err != nil { + writeBadRequest(w, "invalid request body") + return + } + if len(req.Value) == 0 { + writeBadRequest(w, `value is required — send {"value": null} to store an explicit null`) + return + } + + var value any + if err := json.Unmarshal(req.Value, &value); err != nil { + writeBadRequest(w, "value must be valid JSON") + return + } + + if err := h.Meta.Set(r.Context(), userID, key, value); err != nil { + writeErr(w, err) + return + } + h.writeMetadata(w, r, userID) +} + +// Delete — admin required. Removes one key. +// +// A key that is not set answers 404 rather than a bare 200, matching +// DELETE /v1/api-keys/{id} and DELETE /v1/sessions/{id}: a console that +// removed the wrong field should be told, not shown a success it did not +// achieve. +func (h *MetadataHandlers) Delete(w http.ResponseWriter, r *http.Request, userID, key string) { + if !h.ready(w, r, userID) { + return + } + if err := h.Meta.Delete(r.Context(), userID, key); err != nil { + writeErr(w, err) + return + } + h.writeMetadata(w, r, userID) +} + +// ready answers the two things all three handlers need first: are the +// stores wired, and is this a user at all. It reports whether the caller +// should carry on. +func (h *MetadataHandlers) ready(w http.ResponseWriter, r *http.Request, userID string) bool { + if h.Users == nil || h.Meta == nil { + writeErr(w, errAdminStoresUnavailable) + return false + } + + // A path segment that cannot be a user id is answered 404, the same + // as one that simply is not a user. Two reasons: "no such user" is + // the honest description of both, and a malformed id handed straight + // to Postgres is a driver error — "invalid input syntax for type + // uuid" — which mapError would turn into a 500 and an operator would + // read as a bug in the API rather than as a stale bookmark. + if !looksLikeUUID(userID) { + writeErr(w, store.ErrNotFound) + return false + } + + if _, err := h.Users.GetByID(r.Context(), userID); err != nil { + writeErr(w, err) + return false + } + return true +} + +// writeMetadata re-reads and returns the user's whole set. Reading it +// back rather than echoing what was just written is deliberate: it is the +// one thing that proves the value that landed is the value stored, and it +// costs one query on an endpoint an operator calls by hand. +func (h *MetadataHandlers) writeMetadata(w http.ResponseWriter, r *http.Request, userID string) { + metadata, err := h.Meta.AllFor(r.Context(), userID) + if err != nil { + writeErr(w, err) + return + } + writeData(w, http.StatusOK, metadataDTO{ + UserID: userID, + Metadata: metadata, + ReservedClaimNames: usermeta.ReservedKeys(), + }) +} + +// looksLikeUUID reports whether s has the canonical UUID shape — +// 8-4-4-4-12 hexadecimal digits. It checks the shape rather than parsing +// the value, because the only question being asked is whether the string +// can safely reach a UUID column; whether it names a row is the query's +// business, not this function's. +func looksLikeUUID(s string) bool { + if len(s) != 36 { + return false + } + for i, c := range s { + switch i { + case 8, 13, 18, 23: + if c != '-' { + return false + } + default: + isHex := (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') + if !isHex { + return false + } + } + } + return true +} diff --git a/httpapi/metadata_handlers_test.go b/httpapi/metadata_handlers_test.go new file mode 100644 index 0000000..390539f --- /dev/null +++ b/httpapi/metadata_handlers_test.go @@ -0,0 +1,459 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/crydensync/cryden/v2" + "github.com/crydensync/cryden/v2/store/memory" + + "github.com/crydensync/api/config" + "github.com/crydensync/api/usermeta" +) + +// roleFunc lets a test answer RoleFor without a database, the same job +// operator.Store does in production. +type roleFunc func(ctx context.Context, userID string) (string, bool, error) + +func (f roleFunc) RoleFor(ctx context.Context, userID string) (string, bool, error) { + return f(ctx, userID) +} + +// metadataFixture is an engine whose claims provider is the real one — +// usermeta.ClaimsProvider, the same call main.go makes — plus a router +// wired with the same user store the engine holds. +// +// That last part matters: Deps.Users is what the handlers use to answer +// "does this user exist", and a second store instance would 404 every +// user the engine had just created. +type metadataFixture struct { + engine *cryden.Engine + router http.Handler + meta *usermeta.MemoryStore + + // user and operator are two real accounts: the first is an ordinary + // end user whose metadata is being mapped, the second holds the role + // claim RequireAdmin wants. + userID string + userToken string + opID string + opToken string +} + +func newMetadataFixture(t *testing.T) metadataFixture { + t.Helper() + ctx := context.Background() + + users := memory.NewUserStore() + meta := usermeta.NewMemoryStore() + + // The operator is identified by id, which is only known after signup + // — so the closure reads it from here, exactly as operator.Store + // would answer from its table. + var operatorID string + roles := roleFunc(func(_ context.Context, userID string) (string, bool, error) { + if userID == operatorID { + return "admin", true, nil + } + return "", false, nil + }) + + engine, err := cryden.New(cryden.Config{ + JWTSecret: "test-secret", + Users: users, + Sessions: memory.NewSessionStore(), + Audit: memory.NewAuditStore(), + Verifications: memory.NewVerificationStore(), + EmailSender: stubMailSender{}, + MagicLinkSender: stubMailSender{}, + APIKeys: memory.NewAPIKeyStore(), + APIKeyPrefix: "ck", + AccessTokenClaims: usermeta.ClaimsProvider(meta, roles), + }) + if err != nil { + t.Fatalf("building engine: %v", err) + } + + operator, err := cryden.SignUp(ctx, engine, "operator@example.com", testPassword, "203.0.113.1") + if err != nil { + t.Fatalf("signup (operator): %v", err) + } + operatorID = operator.ID + opTokens, err := cryden.Login(ctx, engine, "operator@example.com", testPassword, "203.0.113.1", chromeOnMacOS) + if err != nil { + t.Fatalf("login (operator): %v", err) + } + + user, err := cryden.SignUp(ctx, engine, "user@example.com", testPassword, "203.0.113.2") + if err != nil { + t.Fatalf("signup (user): %v", err) + } + userTokens, err := cryden.Login(ctx, engine, "user@example.com", testPassword, "203.0.113.2", chromeOnMacOS) + if err != nil { + t.Fatalf("login (user): %v", err) + } + + return metadataFixture{ + engine: engine, + router: NewRouter(Deps{Engine: engine, Config: config.Config{}, Users: users, Meta: meta}), + meta: meta, + userID: user.ID, + userToken: userTokens.AccessToken, + opID: operator.ID, + opToken: opTokens.AccessToken, + } +} + +func (f metadataFixture) call(t *testing.T, method, path, body, token string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + f.router.ServeHTTP(rec, req) + return rec +} + +func metadataPath(userID, key string) string { + base := "/v1/admin/users/" + userID + "/metadata" + if key != "" { + return base + "/" + key + } + return base +} + +type metadataResponse struct { + Data struct { + UserID string `json:"user_id"` + Metadata map[string]any `json:"metadata"` + ReservedClaimNames []string `json:"reserved_claim_names"` + } `json:"data"` +} + +func decodeMetadata(t *testing.T, rec *httptest.ResponseRecorder) metadataResponse { + t.Helper() + var resp metadataResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decoding %s: %v", rec.Body.String(), err) + } + return resp +} + +// The claim the whole feature exists for. This asserts through a token +// the engine actually issued, not through the store: it is the only +// check that proves the wiring in main.go's Config.AccessTokenClaims is +// connected to the table the admin endpoints write. +func TestMetadataKeyReachesAFreshlyIssuedToken(t *testing.T) { + f := newMetadataFixture(t) + + rec := f.call(t, http.MethodPut, metadataPath(f.userID, "plan"), `{"value":"pro"}`, f.opToken) + if rec.Code != http.StatusOK { + t.Fatalf("PUT status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + + // A NEW login: claims are attached when a token is issued, so a + // token minted before the write would prove nothing. + tokens, err := cryden.Login(context.Background(), f.engine, "user@example.com", testPassword, "203.0.113.2", chromeOnMacOS) + if err != nil { + t.Fatalf("login: %v", err) + } + + userID, claims, err := cryden.VerifyTokenWithClaims(f.engine, tokens.AccessToken) + if err != nil { + t.Fatalf("verifying token: %v", err) + } + if userID != f.userID { + t.Fatalf("token subject = %q, want %q", userID, f.userID) + } + if claims["plan"] != "pro" { + t.Errorf("claims = %v, want plan=pro", claims) + } + // An ordinary user, so no role — see usermeta.RoleClaim for why that + // absence has to stay an absence. + if _, hasRole := claims["role"]; hasRole { + t.Errorf("claims = %v, want no role claim for an ordinary user", claims) + } +} + +// The escalation this feature would otherwise open. Every metadata key +// becomes a claim, and RequireAdmin reads "role" — so a key of "role" +// would mint an operator token for a user the operators table has never +// heard of, and revoking an operator would not take it away. +func TestMetadataCannotGrantOperatorStatus(t *testing.T) { + f := newMetadataFixture(t) + + rec := f.call(t, http.MethodPut, metadataPath(f.userID, "role"), `{"value":"admin"}`, f.opToken) + if rec.Code != http.StatusBadRequest { + t.Fatalf("PUT role status = %d, want 400 (body %s)", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "reserved_metadata_key") { + t.Errorf("body = %s, want reserved_metadata_key", rec.Body.String()) + } + + // And the user still cannot reach an admin route. + tokens, err := cryden.Login(context.Background(), f.engine, "user@example.com", testPassword, "203.0.113.2", chromeOnMacOS) + if err != nil { + t.Fatalf("login: %v", err) + } + admin := f.call(t, http.MethodGet, metadataPath(f.userID, ""), "", tokens.AccessToken) + if admin.Code != http.StatusForbidden { + t.Errorf("admin route with the user's token: status = %d, want 403", admin.Code) + } +} + +// A registered JWT claim name is refused where an operator can still see +// why. Stored, it would not be ignored — cryden's checkExtraClaims +// rejects it while building the token, so every login for that user +// would fail with the cause sitting in a different table. +func TestMetadataRefusesRegisteredClaimNames(t *testing.T) { + f := newMetadataFixture(t) + + for _, key := range []string{"sub", "iss", "aud", "exp", "nbf", "iat", "jti"} { + rec := f.call(t, http.MethodPut, metadataPath(f.userID, key), `{"value":"x"}`, f.opToken) + if rec.Code != http.StatusBadRequest { + t.Errorf("PUT %q status = %d, want 400", key, rec.Code) + continue + } + if !strings.Contains(rec.Body.String(), "reserved_metadata_key") { + t.Errorf("PUT %q body = %s, want reserved_metadata_key", key, rec.Body.String()) + } + } + + // The login still works, which is the point of refusing at write time. + if _, err := cryden.Login(context.Background(), f.engine, "user@example.com", testPassword, "203.0.113.2", chromeOnMacOS); err != nil { + t.Fatalf("login after refused writes: %v", err) + } +} + +// A key that is not a claim name is refused by the store, and the refusal +// reaches the client as a 400 with a code that says which rule it broke. +// +// Two of the keys below are percent-escaped because that is the only way a +// console can put them in a path at all: the router matches one segment and +// hands the handler the *decoded* value, so %2F arrives as "a/b" and is +// judged on its merits rather than being rejected by URL parsing. +func TestMetadataRejectsKeysThatAreNotClaimNames(t *testing.T) { + f := newMetadataFixture(t) + + for _, tc := range []struct{ name, key string }{ + {"starts with a digit", "1st"}, + {"contains a slash", "a%2Fb"}, + {"too long", strings.Repeat("a", 65)}, + {"only whitespace", "%20"}, + } { + t.Run(tc.name, func(t *testing.T) { + rec := f.call(t, http.MethodPut, metadataPath(f.userID, tc.key), `{"value":"x"}`, f.opToken) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (body %s)", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "invalid_metadata_key") { + t.Errorf("body = %s, want invalid_metadata_key", rec.Body.String()) + } + }) + } + + // None of them landed, so a refused key is refused rather than written + // and reported. + if keys := f.meta.Keys(f.userID); len(keys) != 0 { + t.Errorf("keys after refused requests = %v, want none", keys) + } +} + +// "value": null is a value. A missing field is a client bug. Both are +// easy to conflate by decoding into an `any`, so the distinction is +// asserted rather than assumed. +func TestMetadataDistinguishesNullFromAbsent(t *testing.T) { + f := newMetadataFixture(t) + + rec := f.call(t, http.MethodPut, metadataPath(f.userID, "note"), `{"value":null}`, f.opToken) + if rec.Code != http.StatusOK { + t.Fatalf("PUT null status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + if _, present := decodeMetadata(t, rec).Data.Metadata["note"]; !present { + t.Error("a stored null is missing from the response — null is a value, not an absence") + } + + for _, body := range []string{`{}`, `{"value":`} { + if rec := f.call(t, http.MethodPut, metadataPath(f.userID, "note"), body, f.opToken); rec.Code != http.StatusBadRequest { + t.Errorf("PUT %s status = %d, want 400", body, rec.Code) + } + } +} + +func TestMetadataRoundTripAndDelete(t *testing.T) { + f := newMetadataFixture(t) + + if rec := f.call(t, http.MethodPut, metadataPath(f.userID, "tenant"), `{"value":"acme"}`, f.opToken); rec.Code != http.StatusOK { + t.Fatalf("PUT status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + if rec := f.call(t, http.MethodPut, metadataPath(f.userID, "seats"), `{"value":12}`, f.opToken); rec.Code != http.StatusOK { + t.Fatalf("PUT status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + + rec := f.call(t, http.MethodGet, metadataPath(f.userID, ""), "", f.opToken) + if rec.Code != http.StatusOK { + t.Fatalf("GET status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + got := decodeMetadata(t, rec) + if got.Data.UserID != f.userID { + t.Errorf("user_id = %q, want %q", got.Data.UserID, f.userID) + } + if got.Data.Metadata["tenant"] != "acme" { + t.Errorf("tenant = %#v, want \"acme\"", got.Data.Metadata["tenant"]) + } + // JSON numbers come back as float64 — the same thing the JSONB column + // decodes to, so a console sees one shape in both stores. + if got.Data.Metadata["seats"] != float64(12) { + t.Errorf("seats = %#v, want float64(12)", got.Data.Metadata["seats"]) + } + + // The reserved list is what a console greys out, so it has to be + // present and complete rather than a token gesture. + if len(got.Data.ReservedClaimNames) != 8 { + t.Errorf("reserved_claim_names = %v, want the seven registered names plus role", got.Data.ReservedClaimNames) + } + + // Replace, not accumulate. + if rec := f.call(t, http.MethodPut, metadataPath(f.userID, "tenant"), `{"value":"globex"}`, f.opToken); rec.Code != http.StatusOK { + t.Fatalf("PUT (replace) status = %d, want 200", rec.Code) + } + if all := decodeMetadata(t, f.call(t, http.MethodGet, metadataPath(f.userID, ""), "", f.opToken)).Data.Metadata; all["tenant"] != "globex" { + t.Errorf("tenant = %#v, want \"globex\"", all["tenant"]) + } + + // DELETE answers with the updated set, so a save is also the refresh. + rec = f.call(t, http.MethodDelete, metadataPath(f.userID, "tenant"), "", f.opToken) + if rec.Code != http.StatusOK { + t.Fatalf("DELETE status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + if _, present := decodeMetadata(t, rec).Data.Metadata["tenant"]; present { + t.Error("the deleted key is still in the response") + } + + // Deleting it twice reports that there was nothing to delete, rather + // than showing a success that did not happen. + again := f.call(t, http.MethodDelete, metadataPath(f.userID, "tenant"), "", f.opToken) + if again.Code != http.StatusNotFound { + t.Errorf("second DELETE status = %d, want 404", again.Code) + } + if !strings.Contains(again.Body.String(), "metadata_key_not_found") { + t.Errorf("body = %s, want metadata_key_not_found", again.Body.String()) + } + + // The store agrees with the responses. + if keys := f.meta.Keys(f.userID); len(keys) != 1 || keys[0] != "seats" { + t.Errorf("stored keys = %v, want just \"seats\"", keys) + } +} + +// An unknown user, and an id that could never be a user, both answer 404. +// The second half is the one that matters: handed straight to Postgres, a +// malformed id is a driver error — "invalid input syntax for type uuid" — +// which mapError turns into a 500 an operator reads as a bug in the API. +func TestMetadataUnknownOrMalformedUserIs404(t *testing.T) { + f := newMetadataFixture(t) + + for _, tc := range []struct{ name, userID string }{ + {"well-formed but unknown", "01a0a4ce-5453-78d3-9126-000000000000"}, + {"not a uuid at all", "not-a-uuid"}, + {"a uuid with a stray character", "01a0a4ce-5453-78d3-9126-52268da8da5z"}, + {"too short", "01a0a4ce-5453-78d3-9126-52268da8da5"}, + } { + t.Run(tc.name, func(t *testing.T) { + for _, call := range []struct { + method, body string + }{ + {http.MethodGet, ""}, + {http.MethodPut, `{"value":1}`}, + {http.MethodDelete, ""}, + } { + path := metadataPath(tc.userID, "plan") + if call.method == http.MethodGet { + path = metadataPath(tc.userID, "") + } + rec := f.call(t, call.method, path, call.body, f.opToken) + if rec.Code != http.StatusNotFound { + t.Errorf("%s status = %d, want 404 (body %s)", call.method, rec.Code, rec.Body.String()) + } + if rec.Code == http.StatusInternalServerError { + t.Errorf("%s reached the database with a malformed id", call.method) + } + } + }) + } +} + +// Every route here is admin-only, and the gate is the same one the rest of +// the admin surface uses. +func TestMetadataRoutesRequireAdmin(t *testing.T) { + f := newMetadataFixture(t) + + paths := []struct{ method, path, body string }{ + {http.MethodGet, metadataPath(f.userID, ""), ""}, + {http.MethodPut, metadataPath(f.userID, "plan"), `{"value":"pro"}`}, + {http.MethodDelete, metadataPath(f.userID, "plan"), ""}, + } + + for _, tc := range paths { + rec := f.call(t, tc.method, tc.path, tc.body, "") + if rec.Code != http.StatusUnauthorized { + t.Errorf("%s %s with no token: status = %d, want 401", tc.method, tc.path, rec.Code) + } + + rec = f.call(t, tc.method, tc.path, tc.body, f.userToken) + if rec.Code != http.StatusForbidden { + t.Errorf("%s %s with an ordinary user's token: status = %d, want 403 (body %s)", + tc.method, tc.path, rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "not_operator") { + t.Errorf("%s %s body = %s, want not_operator", tc.method, tc.path, rec.Body.String()) + } + } + + // And nothing was written by any of those attempts. + if keys := f.meta.Keys(f.userID); len(keys) != 0 { + t.Errorf("keys after refused requests = %v, want none", keys) + } +} + +// A router built without the metadata stores is a wiring fact, not a +// server fault — 404, the same shape as every other unconfigured feature. +func TestMetadataWithoutStoresIs404(t *testing.T) { + f := newMetadataFixture(t) + router := NewRouter(Deps{Engine: f.engine, Config: config.Config{}}) + + req := httptest.NewRequest(http.MethodGet, metadataPath(f.userID, ""), nil) + req.Header.Set("Authorization", "Bearer "+f.opToken) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404 (body %s)", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "not_configured") { + t.Errorf("body = %s, want not_configured", rec.Body.String()) + } +} + +// Two accounts, one table: nothing an operator does to one user's +// metadata may be visible on another's. +func TestMetadataIsScopedPerUser(t *testing.T) { + f := newMetadataFixture(t) + + if rec := f.call(t, http.MethodPut, metadataPath(f.userID, "plan"), `{"value":"pro"}`, f.opToken); rec.Code != http.StatusOK { + t.Fatalf("PUT status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + + rec := f.call(t, http.MethodGet, metadataPath(f.opID, ""), "", f.opToken) + if rec.Code != http.StatusOK { + t.Fatalf("GET (operator) status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + if got := decodeMetadata(t, rec).Data.Metadata; len(got) != 0 { + t.Errorf("the operator sees %v on their own record, want nothing", got) + } +} diff --git a/httpapi/router.go b/httpapi/router.go index 31cb82a..e7daf4b 100644 --- a/httpapi/router.go +++ b/httpapi/router.go @@ -8,6 +8,7 @@ import ( "github.com/crydensync/cryden/v2/store" "github.com/crydensync/api/config" + "github.com/crydensync/api/usermeta" ) // Deps is everything the route table needs to build its handlers. It is a @@ -37,6 +38,11 @@ type Deps struct { // records against the user total. Audit store.AuditStore Users store.UserStore + + // Meta backs the per-user metadata endpoints. This repo's own table + // and package — cryden's store.User has no metadata concept and will + // not gain one (see usermeta's package doc). + Meta usermeta.Store } // NewRouter builds the full route table. Called once from main.go. @@ -56,6 +62,7 @@ func NewRouter(d Deps) http.Handler { recovery := &RecoveryHandlers{Engine: engine} apiKeys := &APIKeyHandlers{Engine: engine} security := &SecurityHandlers{Audit: d.Audit, Users: d.Users, Config: d.Config} + metadata := &MetadataHandlers{Users: d.Users, Meta: d.Meta} mux := http.NewServeMux() @@ -155,5 +162,20 @@ func NewRouter(d Deps) http.Handler { mux.HandleFunc("GET /v1/admin/oauth/health", RequireAdmin(engine, oauthHealth.Health)) mux.HandleFunc("GET /v1/admin/security/hash-migration", RequireAdmin(engine, security.HashMigration)) + // Per-user metadata — the table behind JWT claim mapping. Per key + // rather than a whole-map PUT, so two operators editing different + // fields of one user cannot overwrite each other's work. The user id + // is a path segment and is never read from the body, so there is no + // second place it could come from. + mux.HandleFunc("GET /v1/admin/users/{userID}/metadata", RequireAdmin(engine, func(w http.ResponseWriter, r *http.Request) { + metadata.List(w, r, r.PathValue("userID")) + })) + mux.HandleFunc("PUT /v1/admin/users/{userID}/metadata/{key}", RequireAdmin(engine, func(w http.ResponseWriter, r *http.Request) { + metadata.Put(w, r, r.PathValue("userID"), r.PathValue("key")) + })) + mux.HandleFunc("DELETE /v1/admin/users/{userID}/metadata/{key}", RequireAdmin(engine, func(w http.ResponseWriter, r *http.Request) { + metadata.Delete(w, r, r.PathValue("userID"), r.PathValue("key")) + })) + return mux } diff --git a/main.go b/main.go index 308205a..f285307 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "context" "database/sql" "log" "net/http" @@ -12,12 +11,12 @@ import ( "github.com/crydensync/cryden/v2" "github.com/crydensync/cryden/v2/security" "github.com/crydensync/cryden/v2/store/postgres" - "github.com/crydensync/cryden/v2/token" "github.com/crydensync/api/config" "github.com/crydensync/api/httpapi" "github.com/crydensync/api/operator" "github.com/crydensync/api/templates" + "github.com/crydensync/api/usermeta" ) func main() { @@ -46,6 +45,11 @@ func main() { users := postgres.NewUserStore(db) audit := postgres.NewAuditStore(db) + // Per-user metadata: this repo's own table, merged into the access + // token's claims below. See usermeta's package doc for why the + // reserved-key rule lives in the store rather than in the handler. + metadata := usermeta.NewStore(db) + // Email templates are optional and entirely this repo's: cryden owns // no message copy. An unset EMAIL_TEMPLATE_DIR leaves both senders // printing their own built-in line, byte for byte as before. @@ -76,20 +80,19 @@ func main() { APIKeys: postgres.NewAPIKeyStore(db), APIKeyPrefix: cfg.APIKeyPrefix, - // Attaches a "role" claim for console operators only — an - // ordinary end user's token gets no extra claims at all, not - // even role="user". See operator/store.go for why this is a - // separate table rather than anything on cryden's own User. - AccessTokenClaims: token.ClaimsFunc(func(ctx context.Context, userID string) (map[string]any, error) { - role, isOperator, err := operators.RoleFor(ctx, userID) - if err != nil { - return nil, err - } - if !isOperator { - return nil, nil - } - return map[string]any{"role": role}, nil - }), + // The claims every access token carries for its user: "role" for + // console operators, plus every metadata key the admin console has + // mapped. usermeta.ClaimsProvider owns the merge — including the + // rule that "role" is refused as a metadata key, which is exactly + // why it lives in a package a test can reach rather than in a + // closure here. + // + // This runs on EVERY login and every refresh — roughly once per + // ACCESS_TOKEN_TTL per active session — and costs two queries. + // That is the price of claims that are current rather than + // frozen at signup, and it is why the engine calls a claims + // provider on the hot path only when a host asks it to. + AccessTokenClaims: usermeta.ClaimsProvider(metadata, operators), } // Password hashing. Leaving Hasher unset is what selects bcrypt — the @@ -183,6 +186,7 @@ func main() { Config: cfg, Audit: audit, Users: users, + Meta: metadata, }) limiter := httpapi.NewEdgeRateLimiter(cfg.EdgeRateLimit, cfg.EdgeRateLimitWindow) handler := httpapi.WithCORS(cfg.CORSOrigins, httpapi.WithEdgeRateLimit(limiter, router)) diff --git a/migrations/009_user_metadata.down.sql b/migrations/009_user_metadata.down.sql new file mode 100644 index 0000000..de9fcbf --- /dev/null +++ b/migrations/009_user_metadata.down.sql @@ -0,0 +1,3 @@ +-- 009_user_metadata.down.sql + +DROP TABLE IF EXISTS user_metadata; diff --git a/migrations/009_user_metadata.up.sql b/migrations/009_user_metadata.up.sql new file mode 100644 index 0000000..244e260 --- /dev/null +++ b/migrations/009_user_metadata.up.sql @@ -0,0 +1,31 @@ +-- 009_user_metadata.up.sql +-- +-- Arbitrary per-user metadata, owned by this api layer. cryden's own +-- store.User deliberately has no metadata concept and will not gain +-- one (see cryden's docs/design-decisions.md) — so the row that a +-- console maps into JWT claims lives here, keyed off the engine's own +-- user id, exactly like operators (003) does. +-- +-- The shape is one row per key rather than one JSON blob per user. +-- A blob would make "set field X" a read-modify-write, so two console +-- operators saving at the same moment would silently lose one edit; +-- per-key rows make every write a single upsert and every delete a +-- single statement, with no window between them. +-- +-- value is JSONB, not TEXT, and that is load-bearing rather than +-- stylistic: everything in this table becomes a JWT claim, and a claim +-- has to marshal to JSON. Storing JSON means a value that could not be +-- a claim cannot be stored in the first place. + +CREATE TABLE user_metadata ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + -- The claim name this value will appear under. A non-reserved + -- keyword in Postgres, so it needs no quoting. + key TEXT NOT NULL, + value JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Composite rather than a surrogate id: (user_id, key) is the real + -- identity here, and it is also the index every read uses. + PRIMARY KEY (user_id, key) +); diff --git a/usermeta/claims.go b/usermeta/claims.go new file mode 100644 index 0000000..b402216 --- /dev/null +++ b/usermeta/claims.go @@ -0,0 +1,73 @@ +package usermeta + +import ( + "context" + + "github.com/crydensync/cryden/v2/token" +) + +// RoleLookup is what the claims provider needs to know about console +// operators. operator.Store satisfies it as it stands, and it is declared +// here rather than imported so this package keeps its one dependency — +// the claim names — and does not reach into a sibling package for a +// single method. +type RoleLookup interface { + RoleFor(ctx context.Context, userID string) (role string, isOperator bool, err error) +} + +// ClaimsProvider builds the token.ClaimsProvider that main.go wires into +// cryden.Config.AccessTokenClaims: every metadata key set on the user, +// plus "role" for an operator. +// +// It is a function in this package rather than a closure written inline +// in main.go for one reason — a closure in package main cannot be +// reached by a test, and the merge is the half of this feature that +// storage tests cannot prove. Asserting that a stored key reaches a real +// token is only meaningful if the code under test is the code that runs +// in production, not a copy of it written next to the assertion. +// +// Two properties worth stating, because both are load-bearing: +// +// - A user with no metadata who is not an operator gets NO claims at +// all — nil, not an empty map. The absence of a "role" claim is what +// RequireAdmin reads as "ordinary user" (see middleware.go). +// - "role" is written after the metadata, so it always wins the map. +// That is a second line of defence and nothing more: the real rule +// is that Set refuses "role" as a key (see RoleClaim), because a +// merge order is a guarantee that lasts until someone reorders two +// statements. +// +// operators may be nil for a host with no operator table at all, in which +// case every user's claims are their metadata alone. +func ClaimsProvider(meta Store, operators RoleLookup) token.ClaimsProvider { + return token.ClaimsFunc(func(ctx context.Context, userID string) (map[string]any, error) { + var ( + role string + isOperator bool + ) + if operators != nil { + var err error + role, isOperator, err = operators.RoleFor(ctx, userID) + if err != nil { + return nil, err + } + } + + mapped, err := meta.AllFor(ctx, userID) + if err != nil { + return nil, err + } + if !isOperator && len(mapped) == 0 { + return nil, nil + } + + claims := make(map[string]any, len(mapped)+1) + for key, value := range mapped { + claims[key] = value + } + if isOperator { + claims[RoleClaim] = role + } + return claims, nil + }) +} diff --git a/usermeta/claims_test.go b/usermeta/claims_test.go new file mode 100644 index 0000000..f9e343c --- /dev/null +++ b/usermeta/claims_test.go @@ -0,0 +1,159 @@ +package usermeta + +import ( + "context" + "errors" + "testing" +) + +// stubRoles is a RoleLookup with one operator in it, so the merge can be +// tested without a database or an operators table. +type stubRoles struct { + operatorID string + role string + err error +} + +func (s stubRoles) RoleFor(_ context.Context, userID string) (string, bool, error) { + if s.err != nil { + return "", false, s.err + } + if userID == s.operatorID { + return s.role, true, nil + } + return "", false, nil +} + +func TestClaimsProviderMergesMetadataAndRole(t *testing.T) { + store := NewMemoryStore() + ctx := context.Background() + + if err := store.Set(ctx, "user-1", "plan", "pro"); err != nil { + t.Fatalf("Set: %v", err) + } + if err := store.Set(ctx, "user-1", "tenant", "acme"); err != nil { + t.Fatalf("Set: %v", err) + } + + provider := ClaimsProvider(store, stubRoles{operatorID: "user-1", role: "admin"}) + claims, err := provider.AccessTokenClaims(ctx, "user-1") + if err != nil { + t.Fatalf("AccessTokenClaims: %v", err) + } + + if claims["plan"] != "pro" || claims["tenant"] != "acme" { + t.Errorf("claims = %v, want the stored metadata", claims) + } + if claims[RoleClaim] != "admin" { + t.Errorf("claims[%q] = %v, want \"admin\"", RoleClaim, claims[RoleClaim]) + } +} + +// A user who is neither an operator nor has any metadata gets nil, not an +// empty map. That absence is what RequireAdmin reads as "ordinary user", +// and a test that accepted an empty map here would not notice the +// difference until an admin route started letting people through. +func TestClaimsProviderReturnsNothingForAPlainUser(t *testing.T) { + claims, err := ClaimsProvider(NewMemoryStore(), stubRoles{operatorID: "someone-else"}). + AccessTokenClaims(context.Background(), "user-1") + if err != nil { + t.Fatalf("AccessTokenClaims: %v", err) + } + if claims != nil { + t.Errorf("claims = %v, want nil for a user with neither metadata nor a role", claims) + } +} + +// Metadata alone is enough to produce claims — an ordinary user with a +// mapped field is the normal case this feature exists for. +func TestClaimsProviderReturnsMetadataForANonOperator(t *testing.T) { + store := NewMemoryStore() + ctx := context.Background() + if err := store.Set(ctx, "user-1", "plan", "pro"); err != nil { + t.Fatalf("Set: %v", err) + } + + claims, err := ClaimsProvider(store, stubRoles{operatorID: "someone-else"}). + AccessTokenClaims(ctx, "user-1") + if err != nil { + t.Fatalf("AccessTokenClaims: %v", err) + } + if claims["plan"] != "pro" { + t.Errorf("claims = %v, want the stored metadata", claims) + } + if _, hasRole := claims[RoleClaim]; hasRole { + t.Error("a non-operator's token carries a role claim") + } +} + +// The second line of defence. Set refuses "role" outright (RoleClaim), so +// this can only be reached by a store that bypassed validation — but the +// merge order still has to put the real role last, because the day +// someone relaxes the write rule is the day this becomes the only thing +// standing between metadata and an operator token. +func TestClaimsProviderLetsTheRealRoleWinOverMetadata(t *testing.T) { + ctx := context.Background() + provider := ClaimsProvider(injectingStore{key: RoleClaim, value: "admin"}, + stubRoles{operatorID: "user-1", role: "support"}) + + claims, err := provider.AccessTokenClaims(ctx, "user-1") + if err != nil { + t.Fatalf("AccessTokenClaims: %v", err) + } + if claims[RoleClaim] != "support" { + t.Errorf("claims[%q] = %v, want the operator's real role", RoleClaim, claims[RoleClaim]) + } +} + +// A store error must not be swallowed: a claims provider that returned a +// partial map on error would mint a token with fewer claims than the +// user has, which is worse than failing the login. +func TestClaimsProviderPropagatesStoreErrors(t *testing.T) { + wantErr := errors.New("store is down") + _, err := ClaimsProvider(failingStore{err: wantErr}, stubRoles{}). + AccessTokenClaims(context.Background(), "user-1") + if !errors.Is(err, wantErr) { + t.Errorf("error = %v, want %v", err, wantErr) + } + + _, err = ClaimsProvider(NewMemoryStore(), stubRoles{err: wantErr}). + AccessTokenClaims(context.Background(), "user-1") + if !errors.Is(err, wantErr) { + t.Errorf("role lookup error = %v, want %v", err, wantErr) + } +} + +// A host with no operator table at all is a real configuration; nil must +// not panic. +func TestClaimsProviderAllowsNoOperatorLookup(t *testing.T) { + store := NewMemoryStore() + ctx := context.Background() + if err := store.Set(ctx, "user-1", "plan", "pro"); err != nil { + t.Fatalf("Set: %v", err) + } + + claims, err := ClaimsProvider(store, nil).AccessTokenClaims(ctx, "user-1") + if err != nil { + t.Fatalf("AccessTokenClaims: %v", err) + } + if claims["plan"] != "pro" { + t.Errorf("claims = %v, want the stored metadata", claims) + } +} + +// injectingStore is a Store that hands back a key Set would have refused. +// It exists only to reach the merge-order defence above, which is +// otherwise unreachable by design. +type injectingStore struct{ key, value string } + +func (s injectingStore) AllFor(context.Context, string) (map[string]any, error) { + return map[string]any{s.key: s.value}, nil +} +func (s injectingStore) Set(context.Context, string, string, any) error { return nil } +func (s injectingStore) Delete(context.Context, string, string) error { return nil } + +type failingStore struct{ err error } + +func (s failingStore) AllFor(context.Context, string) (map[string]any, error) { return nil, s.err } +func (s failingStore) Set(context.Context, string, string, any) error { return s.err } +func (s failingStore) Delete(context.Context, string, string) error { return s.err } diff --git a/usermeta/memory.go b/usermeta/memory.go new file mode 100644 index 0000000..92b9972 --- /dev/null +++ b/usermeta/memory.go @@ -0,0 +1,96 @@ +package usermeta + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "sync" +) + +// MemoryStore is the in-process Store, for tests and for any embedding +// host that wants the claims wiring without a database behind it. +// +// It is a faithful double rather than a convenient one, in the two places +// that is easy to get wrong: +// +// - Values are round-tripped through JSON on the way in, exactly as the +// Postgres column does. Storing the caller's `any` directly would let +// a test pass an int where the real store would hand back a float64, +// and a claims test would then be asserting something production +// cannot reproduce. +// - Validation is the same ValidateKey call, not a reimplementation of +// it, so the reserved-key rule cannot hold in one store and not the +// other. +type MemoryStore struct { + mu sync.RWMutex + // keyed by userID, then by metadata key. + rows map[string]map[string]any +} + +func NewMemoryStore() *MemoryStore { + return &MemoryStore{rows: make(map[string]map[string]any)} +} + +var _ Store = (*MemoryStore)(nil) + +func (s *MemoryStore) AllFor(_ context.Context, userID string) (map[string]any, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + out := make(map[string]any, len(s.rows[userID])) + for k, v := range s.rows[userID] { + out[k] = v + } + return out, nil +} + +func (s *MemoryStore) Set(_ context.Context, userID, key string, value any) error { + if err := ValidateKey(key); err != nil { + return err + } + raw, err := marshalValue(key, value) + if err != nil { + return err + } + var roundTripped any + if err := json.Unmarshal(raw, &roundTripped); err != nil { + return fmt.Errorf("decoding metadata %q: %w", key, err) + } + + s.mu.Lock() + defer s.mu.Unlock() + if s.rows[userID] == nil { + s.rows[userID] = make(map[string]any) + } + s.rows[userID][key] = roundTripped + return nil +} + +func (s *MemoryStore) Delete(_ context.Context, userID, key string) error { + // Not validated, matching PostgresStore.Delete — see its comment for + // why deletion is the one operation that must not re-check the rule. + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.rows[userID][key]; !ok { + return fmt.Errorf("%w: %q", ErrNotFound, key) + } + delete(s.rows[userID], key) + return nil +} + +// Keys is a test helper the PostgresStore has no equivalent for: the +// memory store can answer "what is set" in a stable order without a +// query, which keeps a failing assertion readable. Not part of Store, +// because nothing in production needs it. +func (s *MemoryStore) Keys(userID string) []string { + s.mu.RLock() + defer s.mu.RUnlock() + + out := make([]string, 0, len(s.rows[userID])) + for k := range s.rows[userID] { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/usermeta/store.go b/usermeta/store.go new file mode 100644 index 0000000..eef5b8b --- /dev/null +++ b/usermeta/store.go @@ -0,0 +1,228 @@ +// Package usermeta stores the per-user metadata that becomes JWT claims, +// and owns the rule about which keys may exist. +// +// It is this repo's own, not cryden's. cryden's store.User has no +// metadata concept on purpose — authorization and app-specific attributes +// are host decisions, the engine owns authentication mechanics — so the +// table (migrations/009_user_metadata.up.sql) and this package exist on +// top of it, keyed off the engine's own user id, the same way +// operator/store.go is. +// +// What the storage is FOR shapes the design: main.go merges every key in +// here into the claims of the access token it issues, so a key is not +// just a label, it is a claim name. That is why values are JSON (a claim +// must marshal), why reserved claim names are refused at write time (a +// key of "sub" would not be ignored at login, it would fail the login), +// and why "role" is refused alongside them (see RoleClaim). +package usermeta + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + + "github.com/crydensync/cryden/v2/token" +) + +// The three ways a call here can be refused. +var ( + // ErrReservedKey means the key names a claim that is not this repo's + // to set. + ErrReservedKey = errors.New("usermeta: this key is reserved") + // ErrInvalidKey means the key is not a usable claim name at all. + ErrInvalidKey = errors.New("usermeta: invalid metadata key") + // ErrNotFound means there is no such key on that user. Deleting an + // absent key is reported rather than ignored, so a console that + // removed the wrong field says so instead of showing success. + ErrNotFound = errors.New("usermeta: no such metadata key") +) + +// RoleClaim is this repo's own authorization claim — the one +// httpapi.RequireAdmin reads to decide whether a caller may use the admin +// console (see middleware.go and main.go's AccessTokenClaims provider). +// +// It is refused as a metadata key, and that is not a technicality. Every +// key in this package is merged into the access token's claims, so +// without this rule setting metadata key "role" to "admin" would mint an +// operator token for a user the operators table has never heard of — +// a second, hidden way to hand out console access that revoking an +// operator (operator.Store.Revoke) would not take away. Anyone who can +// reach the metadata endpoints is already an operator, so this is not an +// escalation across a privilege boundary; it is the removal of a way to +// grant a privilege that nothing else in the system can see. +const RoleClaim = "role" + +// maxKeyLength bounds a key. It is a claim name, so it is bounded by +// what a token can reasonably carry rather than by anything in the +// database. +const maxKeyLength = 64 + +// keyPattern is the shape a metadata key must have. It is deliberately +// stricter than JSON object keys and than Postgres identifiers: +// +// - It starts with a letter or underscore, so a key can never be +// mistaken for a number or collide with a JSON literal. +// - It allows dots, because the csax+ prototype's own metadata +// references are written "user.metadata.field" and a console +// exposing that spelling should be able to store it. +// +// The length is spelled inside the pattern rather than checked +// separately so there is one expression to read rather than two places +// the rule lives. +var keyPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_.-]{0,63}$`) + +// Store is the persistence this package offers. It is an interface with +// two implementations for the same reason cryden keeps store/postgres and +// store/memory apart: the handlers and the claims provider have to be +// testable without a database, and a double that is written against the +// same contract is the only way to test them that way honestly. +type Store interface { + // AllFor returns every key set on userID. Always non-nil, so a + // caller can range over it without a nil check and a JSON response + // renders {} rather than null. + AllFor(ctx context.Context, userID string) (map[string]any, error) + + // Set creates or replaces one key. The key is validated here, not by + // the caller — see ValidateKey for why that direction matters. + Set(ctx context.Context, userID, key string, value any) error + + // Delete removes one key, or returns ErrNotFound. Unlike Set it does + // not validate the key: see PostgresStore.Delete. + Delete(ctx context.Context, userID, key string) error +} + +// ReservedKeys returns every key Set refuses, sorted: the seven claim +// names from RFC 7519 that cryden will not let a host provider set, plus +// this repo's own RoleClaim. +// +// Exported so GET /v1/admin/users/{id}/metadata can hand the list to a +// console's claim-mapping UI. An operator who can see which names are +// taken does not have to discover the rule by being rejected, and a UI +// that greys them out cannot be the reason someone picks "aud". +func ReservedKeys() []string { + keys := append(token.ReservedClaimNames(), RoleClaim) + sort.Strings(keys) + return keys +} + +// ValidateKey is the rule, in one place. Both stores call it before they +// write, which is what makes "the rule lives in the store, not the +// handler" true rather than aspirational: any future caller — a second +// endpoint, a bootstrap command, a migration — goes through a Store and +// gets the same answer, with no way to bypass it by not knowing about it. +func ValidateKey(key string) error { + if !keyPattern.MatchString(key) { + return fmt.Errorf("%w: %q must start with a letter or underscore, contain only letters, digits, underscores, dots and dashes, and be at most %d characters", + ErrInvalidKey, key, maxKeyLength) + } + if token.IsReservedClaim(key) { + return fmt.Errorf("%w: %q is a registered JWT claim name", ErrReservedKey, key) + } + if key == RoleClaim { + return fmt.Errorf("%w: %q is this api's own operator claim", ErrReservedKey, key) + } + return nil +} + +// PostgresStore is the real store. Constructed once in main.go with the +// same *sql.DB every other store in this repo gets. +type PostgresStore struct { + db *sql.DB +} + +func NewStore(db *sql.DB) *PostgresStore { + return &PostgresStore{db: db} +} + +var _ Store = (*PostgresStore)(nil) + +func (s *PostgresStore) AllFor(ctx context.Context, userID string) (map[string]any, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT key, value FROM user_metadata WHERE user_id = $1 ORDER BY key`, userID) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make(map[string]any) + for rows.Next() { + var ( + key string + raw []byte + ) + if err := rows.Scan(&key, &raw); err != nil { + return nil, err + } + var value any + if err := json.Unmarshal(raw, &value); err != nil { + // A JSONB column can only hold valid JSON, so this is not a + // data problem a caller caused — it means the column holds + // something that did not come through this package. + return nil, fmt.Errorf("decoding metadata %q: %w", key, err) + } + out[key] = value + } + return out, rows.Err() +} + +func (s *PostgresStore) Set(ctx context.Context, userID, key string, value any) error { + if err := ValidateKey(key); err != nil { + return err + } + raw, err := marshalValue(key, value) + if err != nil { + return err + } + // raw is passed as a string rather than a []byte, which is not + // cosmetic: lib/pq sends a []byte as bytea hex, which a JSONB column + // rejects, while a string arrives as text and Postgres parses it as + // the jsonb the parameter's target type says it is. + // + // The upsert is one statement rather than a SELECT-then-INSERT, so + // two operators saving the same key at the same moment cannot lose + // one of the writes. + _, err = s.db.ExecContext(ctx, ` + INSERT INTO user_metadata (user_id, key, value) VALUES ($1, $2, $3) + ON CONFLICT (user_id, key) DO UPDATE SET value = EXCLUDED.value, updated_at = now() + `, userID, key, string(raw)) + return err +} + +func (s *PostgresStore) Delete(ctx context.Context, userID, key string) error { + // Deliberately NOT validated, unlike Set. Deletion is the one + // operation that has to keep working on data the current rules would + // refuse to create: if a name is ever added to the reserved list + // after rows already exist under it — by a future migration, a + // direct write, or a relaxed rule — a validating Delete would make + // those rows permanently unclearable through the API. A key that is + // not there answers ErrNotFound either way, which is the honest + // answer for a reserved name that was never stored. + res, err := s.db.ExecContext(ctx, + `DELETE FROM user_metadata WHERE user_id = $1 AND key = $2`, userID, key) + if err != nil { + return err + } + affected, err := res.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + return fmt.Errorf("%w: %q", ErrNotFound, key) + } + return nil +} + +// marshalValue enforces the one contract the storage has to keep: a value +// that cannot be a JWT claim cannot be stored. Both stores marshal, so a +// test double refuses exactly what the real one does. +func marshalValue(key string, value any) ([]byte, error) { + raw, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("the value for %q must be JSON-encodable: %w", key, err) + } + return raw, nil +} diff --git a/usermeta/store_test.go b/usermeta/store_test.go new file mode 100644 index 0000000..64cfca2 --- /dev/null +++ b/usermeta/store_test.go @@ -0,0 +1,213 @@ +package usermeta + +import ( + "context" + "errors" + "strings" + "testing" +) + +// The rule this whole package exists to enforce. Every one of these would +// otherwise be discovered at login: cryden's checkExtraClaims rejects a +// reserved name when it builds the token, so a stored key of "sub" would +// not be ignored — it would make every subsequent login for that user +// fail, with the cause sitting in a different table from the symptom. +func TestReservedClaimNamesAreRefusedAtWriteTime(t *testing.T) { + store := NewMemoryStore() + ctx := context.Background() + + for _, key := range ReservedKeys() { + err := store.Set(ctx, "user-1", key, "whatever") + if !errors.Is(err, ErrReservedKey) { + t.Errorf("Set(%q) error = %v, want ErrReservedKey", key, err) + } + } + + // Nothing was stored, so nothing can reach a token. + if keys := store.Keys("user-1"); len(keys) != 0 { + t.Errorf("keys after refused writes = %v, want none", keys) + } +} + +// RoleClaim is the one reserved key that is not a JWT registered name, +// and the one with teeth: every metadata key becomes a claim, and +// RequireAdmin reads "role". A metadata key of "role" would mint an +// operator token for someone the operators table has never heard of — +// and operator.Store.Revoke would not take it away, because it never +// granted it. +func TestRoleCannotBeSetAsMetadata(t *testing.T) { + store := NewMemoryStore() + + err := store.Set(context.Background(), "user-1", RoleClaim, "admin") + if !errors.Is(err, ErrReservedKey) { + t.Fatalf("Set(%q, %q) error = %v, want ErrReservedKey", RoleClaim, "admin", err) + } +} + +// The reserved list is what the admin endpoint hands a console to grey +// out, so it has to contain both halves: the engine's seven and this +// repo's one. +func TestReservedKeysCoversTheEngineSetAndRole(t *testing.T) { + reserved := ReservedKeys() + + for _, want := range []string{"iss", "sub", "aud", "exp", "nbf", "iat", "jti", RoleClaim} { + var found bool + for _, key := range reserved { + if key == want { + found = true + break + } + } + if !found { + t.Errorf("ReservedKeys() = %v, missing %q", reserved, want) + } + } + + // Sorted, so the endpoint's response is stable and a diff of two + // responses is meaningful. + for i := 1; i < len(reserved); i++ { + if reserved[i-1] > reserved[i] { + t.Fatalf("ReservedKeys() = %v, want sorted", reserved) + } + } + + // A fresh slice each call: a caller appending to it must not be able + // to edit what the check reads. + reserved[0] = "mutated" + if ReservedKeys()[0] == "mutated" { + t.Error("ReservedKeys() shares its backing array between calls") + } +} + +func TestKeyShapeIsEnforced(t *testing.T) { + store := NewMemoryStore() + ctx := context.Background() + + valid := []string{"plan", "tenant_id", "user.metadata.field", "_private", "a", "A1-b_c.d"} + for _, key := range valid { + if err := store.Set(ctx, "user-1", key, 1); err != nil { + t.Errorf("Set(%q) error = %v, want it accepted", key, err) + } + } + + invalid := []string{ + "", // empty + "1st", // starts with a digit + "has space", // whitespace + "has/slash", // not in the allowed set + "emoji🙂", // non-ASCII + strings.Repeat("a", 65), // one past the 64-character bound + } + for _, key := range invalid { + if err := store.Set(ctx, "user-1", key, 1); !errors.Is(err, ErrInvalidKey) { + t.Errorf("Set(%q) error = %v, want ErrInvalidKey", key, err) + } + } + + // And the bound itself is inclusive: 64 characters is a legal key. + if err := store.Set(ctx, "user-1", strings.Repeat("a", 64), 1); err != nil { + t.Errorf("Set with a 64-character key = %v, want it accepted", err) + } +} + +func TestValuesMustBeJSONEncodable(t *testing.T) { + store := NewMemoryStore() + + // A channel is the simplest value encoding/json refuses. This is not + // a hypothetical: the value ends up inside a JWT, so a value that + // cannot marshal has to be refused here rather than at the next + // login, where the failure would be one the user cannot act on. + if err := store.Set(context.Background(), "user-1", "plan", make(chan int)); err == nil { + t.Fatal("Set with an unmarshalable value returned nil, want an error") + } + if err := store.Set(context.Background(), "user-1", "plan", nil); err != nil { + t.Errorf("Set with a null value = %v, want it accepted — null is valid JSON", err) + } +} + +// The double has to round-trip like the JSONB column does, or a claims +// test would assert a shape production cannot reproduce. +func TestMemoryStoreRoundTripsThroughJSON(t *testing.T) { + store := NewMemoryStore() + ctx := context.Background() + + if err := store.Set(ctx, "user-1", "count", 7); err != nil { + t.Fatalf("Set: %v", err) + } + + all, err := store.AllFor(ctx, "user-1") + if err != nil { + t.Fatalf("AllFor: %v", err) + } + if _, isInt := all["count"].(int); isInt { + t.Error("value came back as int — the real store returns what JSON decodes to (float64)") + } + if all["count"] != float64(7) { + t.Errorf("count = %#v, want float64(7)", all["count"]) + } +} + +func TestSetReplacesAndDeleteReportsAbsence(t *testing.T) { + store := NewMemoryStore() + ctx := context.Background() + + if err := store.Set(ctx, "user-1", "plan", "free"); err != nil { + t.Fatalf("Set: %v", err) + } + if err := store.Set(ctx, "user-1", "plan", "pro"); err != nil { + t.Fatalf("Set (replace): %v", err) + } + + all, err := store.AllFor(ctx, "user-1") + if err != nil { + t.Fatalf("AllFor: %v", err) + } + if all["plan"] != "pro" { + t.Errorf("plan = %#v, want \"pro\" — the second write must replace, not accumulate", all["plan"]) + } + + if err := store.Delete(ctx, "user-1", "plan"); err != nil { + t.Fatalf("Delete: %v", err) + } + // Deleting twice is reported, not ignored: a console that removed the + // wrong field should say so rather than show success. + if err := store.Delete(ctx, "user-1", "plan"); !errors.Is(err, ErrNotFound) { + t.Errorf("second Delete error = %v, want ErrNotFound", err) + } +} + +// AllFor is always non-nil, so a JSON response renders {} rather than +// null and the claims provider can range over it unconditionally. +func TestAllForIsNeverNil(t *testing.T) { + all, err := NewMemoryStore().AllFor(context.Background(), "nobody") + if err != nil { + t.Fatalf("AllFor: %v", err) + } + if all == nil { + t.Fatal("AllFor returned a nil map, want an empty one") + } + if len(all) != 0 { + t.Errorf("AllFor for an unknown user = %v, want empty", all) + } +} + +// One user's keys are not another's. The table is keyed by (user_id, key) +// and every statement carries the user_id predicate, so this is a check +// on the double rather than on the real store — but a double that leaked +// across users would make every handler test above it meaningless. +func TestStoresAreScopedPerUser(t *testing.T) { + store := NewMemoryStore() + ctx := context.Background() + + if err := store.Set(ctx, "user-1", "plan", "pro"); err != nil { + t.Fatalf("Set: %v", err) + } + + other, err := store.AllFor(ctx, "user-2") + if err != nil { + t.Fatalf("AllFor: %v", err) + } + if len(other) != 0 { + t.Errorf("user-2 sees %v, want nothing", other) + } +} From b3292c921de8acda6ec5d0255857a315ccf66ec9 Mon Sep 17 00:00:00 2001 From: Raymond Nicholas Date: Tue, 15 Sep 2026 13:14:21 +0100 Subject: [PATCH 07/10] feat: add webhook delivery log and background worker Engine events are enqueued into this repo's own table and delivered by a background worker, never inline on cryden's login request path. Co-Authored-By: Claude Code --- .env.example | 28 + config/config.go | 93 ++++ config/config_test.go | 116 +++++ httpapi/router.go | 15 + httpapi/webhook_handlers.go | 147 ++++++ httpapi/webhook_handlers_test.go | 477 +++++++++++++++++ main.go | 61 +++ migrations/010_webhook_deliveries.down.sql | 10 + migrations/010_webhook_deliveries.up.sql | 99 ++++ webhook/memory.go | 231 +++++++++ webhook/sender.go | 146 ++++++ webhook/sender_test.go | 281 ++++++++++ webhook/store.go | 390 ++++++++++++++ webhook/worker.go | 408 +++++++++++++++ webhook/worker_test.go | 568 +++++++++++++++++++++ 15 files changed, 3070 insertions(+) create mode 100644 httpapi/webhook_handlers.go create mode 100644 httpapi/webhook_handlers_test.go create mode 100644 migrations/010_webhook_deliveries.down.sql create mode 100644 migrations/010_webhook_deliveries.up.sql create mode 100644 webhook/memory.go create mode 100644 webhook/sender.go create mode 100644 webhook/sender_test.go create mode 100644 webhook/store.go create mode 100644 webhook/worker.go create mode 100644 webhook/worker_test.go diff --git a/.env.example b/.env.example index 8ed9c5d..1c93e7b 100644 --- a/.env.example +++ b/.env.example @@ -120,3 +120,31 @@ CLOUD_LOG_HASH_KEY= # parse, is a startup failure — a template directory that silently did # nothing is worse than one that refused to start. EMAIL_TEMPLATE_DIR= + +# Webhook deliveries. WEBHOOK_URL is the on/off switch: leave it unset and +# the engine dispatches nothing, no delivery worker runs, and +# GET /v1/admin/webhooks/deliveries answers 404 not_configured. The other +# three are only read when it is set, and setting any of them without it +# is a startup failure rather than a setting that silently does nothing. +# +# Events are queued in the webhook_deliveries table and delivered by a +# background worker, never inline: cryden calls the sender on the login +# request path, so an HTTP call there would be your receiver's downtime +# becoming your users' login latency. A delivery is retried with +# exponential backoff (30s doubling to 30m) until WEBHOOK_MAX_ATTEMPTS is +# spent, then recorded as failed and left readable — retrying forever is a +# load generator pointed at someone else's server. +# +# Each request carries X-Cryden-Signature: "sha256=" plus the lowercase +# hex HMAC-SHA256 of the raw body under WEBHOOK_SECRET. Unset means +# deliveries go out unsigned and the header is absent entirely; a receiver +# on a private network is a legitimate reason to do that, and a signature +# over an empty key is not. +# +# WEBHOOK_EVENTS is a comma-separated list of audit event types. Unset +# uses cryden's own default set, which deliberately excludes +# login_success, login_failed and token_rotated. +WEBHOOK_URL= +WEBHOOK_SECRET= +WEBHOOK_EVENTS= +WEBHOOK_MAX_ATTEMPTS= diff --git a/config/config.go b/config/config.go index 10ab761..c63f35b 100644 --- a/config/config.go +++ b/config/config.go @@ -10,6 +10,7 @@ import ( "github.com/crydensync/cryden/v2/logger" "github.com/crydensync/cryden/v2/security" + "github.com/crydensync/cryden/v2/store" ) type Config struct { @@ -192,6 +193,46 @@ type Config struct { // template is a startup failure, the same class of typo as an // unparseable REDIS_URL. EmailTemplateDir string + + // WebhookURL is where engine events are delivered. Empty means this + // deployment dispatches none: Config.Webhooks stays nil, nothing is + // written to the delivery log, and the admin endpoint that reads it + // answers 404 rather than an empty list. + // + // Set, and main.go wires the enqueue-and-worker pair the engine's own + // doc comment asks for — cryden calls a sender synchronously on the + // request path, so nothing here may make an HTTP call on that path. + WebhookURL string + + // WebhookSecret is the HMAC-SHA256 key every delivery is signed with, + // in the X-Cryden-Signature header. It is the receiver's only basis for + // believing a request came from here, so it should be a value of its + // own rather than a reuse of JWT_SECRET or ENCRYPTION_KEY — the same + // key separation cryden asks for on CLOUD_LOG_HASH_KEY, and for the + // same reason: this one is shared with a third party by design. + // + // Empty is allowed and means the deliveries go out unsigned, which is a + // legitimate configuration for an endpoint on a trusted network. The + // worker says so once at startup rather than leaving it to be + // discovered. + WebhookSecret string + + // WebhookEvents selects which engine events are delivered. Empty leaves + // it to cryden, whose DefaultWebhookEvents is the actionable, + // low-volume subset — deliberately excluding login_success, + // login_failed and token_rotated, which are the three a host is most + // likely to ask for and most likely to regret. + // + // Setting this without WebhookURL is a startup failure, matching + // cryden's own rule for the same pair of fields: a subscription to + // nothing is a typo, not a configuration. + WebhookEvents []store.AuditEventType + + // WebhookMaxAttempts is how many times a delivery may be attempted + // before it is recorded as failed. Bounded on purpose — a delivery log + // that retries forever is a load generator pointed at a third party — + // and the row stays readable afterwards either way. + WebhookMaxAttempts int } // PasswordHasher values. Bcrypt is the engine's own default and what an @@ -432,6 +473,58 @@ func Load() (Config, error) { // text. main.go is what reports a directory that is set but broken. cfg.EmailTemplateDir = os.Getenv("EMAIL_TEMPLATE_DIR") + // Webhooks. WEBHOOK_URL is the on/off switch; everything else only + // means anything with it set, which is why the block below refuses the + // combination rather than letting three settings silently do nothing. + cfg.WebhookURL = os.Getenv("WEBHOOK_URL") + cfg.WebhookSecret = os.Getenv("WEBHOOK_SECRET") + if cfg.WebhookMaxAttempts, err = envInt("WEBHOOK_MAX_ATTEMPTS", 5); err != nil { + return cfg, err + } + if cfg.WebhookMaxAttempts < 1 { + // Zero would mean "never attempt a delivery", which is not a + // configuration anyone means to write, and a negative one would + // make the budget check meaningless. + return cfg, fmt.Errorf("WEBHOOK_MAX_ATTEMPTS must be at least 1, got %d", cfg.WebhookMaxAttempts) + } + if events := os.Getenv("WEBHOOK_EVENTS"); events != "" { + for _, e := range strings.Split(events, ",") { + if e = strings.TrimSpace(e); e != "" { + cfg.WebhookEvents = append(cfg.WebhookEvents, store.AuditEventType(e)) + } + } + } + + if cfg.WebhookURL == "" { + // Named individually rather than checked as a group, because the + // fix is different for each and an operator reading "webhook + // configuration is incomplete" would have to go and look. + // + // This mirrors cryden's own rule for Config.WebhookEvents without + // Config.Webhooks: a subscription to nothing is a typo. It also + // covers the two knobs cryden cannot name, having no idea what + // environment variables exist. + var orphaned []string + if cfg.WebhookSecret != "" { + orphaned = append(orphaned, "WEBHOOK_SECRET") + } + if len(cfg.WebhookEvents) > 0 { + orphaned = append(orphaned, "WEBHOOK_EVENTS") + } + if os.Getenv("WEBHOOK_MAX_ATTEMPTS") != "" { + // Non-empty rather than LookupEnv, because empty counts as + // unset throughout this package (envInt above reads it that + // way too). A variable an operator emptied is not a live + // setting, and reporting it as one would make an env file + // full of blank placeholders refuse to start. + orphaned = append(orphaned, "WEBHOOK_MAX_ATTEMPTS") + } + if len(orphaned) > 0 { + return cfg, fmt.Errorf("%s set without WEBHOOK_URL — there is nowhere to deliver to", + strings.Join(orphaned, ", ")) + } + } + return cfg, nil } diff --git a/config/config_test.go b/config/config_test.go index 607b59b..b1333b0 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -40,6 +40,10 @@ var tieredEnvVars = []string{ "CLOUD_LOG_REDACTION", "CLOUD_LOG_HASH_KEY", "EMAIL_TEMPLATE_DIR", + "WEBHOOK_URL", + "WEBHOOK_SECRET", + "WEBHOOK_EVENTS", + "WEBHOOK_MAX_ATTEMPTS", } func loadForTest(t *testing.T, env map[string]string) (Config, error) { @@ -202,6 +206,18 @@ func TestTier3DefaultsComeFromTheEngine(t *testing.T) { if cfg.EmailTemplateDir != "" { t.Errorf("EmailTemplateDir = %q, want empty (built-in sender text)", cfg.EmailTemplateDir) } + // No WEBHOOK_URL means no webhook anything: the off switch is the URL + // itself, because a secret and an event list with nowhere to deliver to + // describe nothing. + if cfg.WebhookURL != "" { + t.Errorf("WebhookURL = %q, want empty (webhooks dispatched by nobody)", cfg.WebhookURL) + } + if len(cfg.WebhookEvents) != 0 { + t.Errorf("WebhookEvents = %v, want empty so cryden's own default set applies", cfg.WebhookEvents) + } + if cfg.WebhookMaxAttempts != 5 { + t.Errorf("WebhookMaxAttempts = %d, want 5", cfg.WebhookMaxAttempts) + } } // One env var must move exactly one field. This is the same trap the @@ -284,3 +300,103 @@ func TestTier3CloudLogHashKeyIsRequiredOnlyForHashRedaction(t *testing.T) { t.Errorf("CloudLogHashKey = %q, want the value that was set", cfg.CloudLogHashKey) } } + +// The event list is a subscription, so whitespace around an entry is a +// typo a person makes by hand and does not mean an event type with a +// space in it — which would match nothing and deliver nothing, silently. +func TestTier3WebhookEventsAreParsedAndTrimmed(t *testing.T) { + cfg, err := loadForTest(t, map[string]string{ + "WEBHOOK_URL": "https://hooks.example.com/v1", + "WEBHOOK_SECRET": "shared-with-the-receiver", + "WEBHOOK_EVENTS": "account_locked, password_reset ,, email_verified ", + }) + if err != nil { + t.Fatalf("Load() failed: %v", err) + } + + if cfg.WebhookURL != "https://hooks.example.com/v1" { + t.Errorf("WebhookURL = %q", cfg.WebhookURL) + } + got := make([]string, 0, len(cfg.WebhookEvents)) + for _, e := range cfg.WebhookEvents { + got = append(got, string(e)) + } + want := "account_locked,password_reset,email_verified" + if strings.Join(got, ",") != want { + t.Errorf("WebhookEvents = %v, want %s (trimmed, empties dropped)", got, want) + } + + // An empty list is left empty rather than filled in here: cryden is + // what turns "no events" into DefaultWebhookEvents, and duplicating + // that list in this repo is how the two would come to disagree. + cfg, err = loadForTest(t, map[string]string{"WEBHOOK_URL": "https://hooks.example.com/v1"}) + if err != nil { + t.Fatalf("Load() failed: %v", err) + } + if len(cfg.WebhookEvents) != 0 { + t.Errorf("WebhookEvents = %v with none set, want empty", cfg.WebhookEvents) + } +} + +// A secret, an event list or an attempt budget with no URL is a typo, not +// a deployment: each of them describes how to deliver to somewhere that +// does not exist. This is the same rule cryden applies to +// WebhookEvents-without-Webhooks, and it is enforced here because the +// failure mode is a setting an operator believes is in force. +func TestTier3WebhookSettingsWithoutAURLAreStartupErrors(t *testing.T) { + for name, env := range map[string]map[string]string{ + "a secret with nowhere to send it": {"WEBHOOK_SECRET": "shared-with-the-receiver"}, + "a subscription to nothing": {"WEBHOOK_EVENTS": "account_locked"}, + "a retry budget for no deliveries": {"WEBHOOK_MAX_ATTEMPTS": "9"}, + "all three, still no destination": {"WEBHOOK_SECRET": "s", "WEBHOOK_EVENTS": "account_locked", "WEBHOOK_MAX_ATTEMPTS": "9"}, + } { + t.Run(name, func(t *testing.T) { + _, err := loadForTest(t, env) + if err == nil { + t.Fatalf("%v was accepted with no WEBHOOK_URL, want an error", env) + } + if !strings.Contains(err.Error(), "without WEBHOOK_URL") { + t.Errorf("error = %q, want it to name WEBHOOK_URL as the missing half", err) + } + }) + } + + // The other direction, which is the one that must keep working: a URL + // on its own is a complete configuration. + if _, err := loadForTest(t, map[string]string{"WEBHOOK_URL": "https://hooks.example.com/v1"}); err != nil { + t.Errorf("a WEBHOOK_URL on its own was rejected: %v", err) + } +} + +// Zero attempts would mean a delivery that is queued and never tried, and +// the row would be a permanent pending — so it is refused rather than +// read as "the default". +func TestTier3WebhookMaxAttemptsIsBounded(t *testing.T) { + cfg, err := loadForTest(t, map[string]string{ + "WEBHOOK_URL": "https://hooks.example.com/v1", + "WEBHOOK_MAX_ATTEMPTS": "2", + }) + if err != nil { + t.Fatalf("Load() failed: %v", err) + } + if cfg.WebhookMaxAttempts != 2 { + t.Errorf("WebhookMaxAttempts = %d, want 2", cfg.WebhookMaxAttempts) + } + + for value, want := range map[string]string{ + "0": "must be at least 1", + "-1": "must be at least 1", + "twice": "must be a number", + } { + _, err := loadForTest(t, map[string]string{ + "WEBHOOK_URL": "https://hooks.example.com/v1", + "WEBHOOK_MAX_ATTEMPTS": value, + }) + if err == nil { + t.Fatalf("WEBHOOK_MAX_ATTEMPTS=%s was accepted, want an error", value) + } + if !strings.Contains(err.Error(), want) { + t.Errorf("WEBHOOK_MAX_ATTEMPTS=%s: error = %q, want it to contain %q", value, err, want) + } + } +} diff --git a/httpapi/router.go b/httpapi/router.go index e7daf4b..0fba09b 100644 --- a/httpapi/router.go +++ b/httpapi/router.go @@ -9,6 +9,7 @@ import ( "github.com/crydensync/api/config" "github.com/crydensync/api/usermeta" + "github.com/crydensync/api/webhook" ) // Deps is everything the route table needs to build its handlers. It is a @@ -43,6 +44,13 @@ type Deps struct { // and package — cryden's store.User has no metadata concept and will // not gain one (see usermeta's package doc). Meta usermeta.Store + + // Hooks backs the admin webhook delivery log. Nil unless WEBHOOK_URL is + // set, because the log is only written by a running delivery worker — + // there is nothing to report on in a deployment that dispatches no + // webhooks, and the handler answers 404 rather than an empty list an + // operator would read as "nothing has failed". + Hooks webhook.Store } // NewRouter builds the full route table. Called once from main.go. @@ -63,6 +71,7 @@ func NewRouter(d Deps) http.Handler { apiKeys := &APIKeyHandlers{Engine: engine} security := &SecurityHandlers{Audit: d.Audit, Users: d.Users, Config: d.Config} metadata := &MetadataHandlers{Users: d.Users, Meta: d.Meta} + hooks := &WebhookHandlers{Store: d.Hooks} mux := http.NewServeMux() @@ -177,5 +186,11 @@ func NewRouter(d Deps) http.Handler { metadata.Delete(w, r, r.PathValue("userID"), r.PathValue("key")) })) + // The webhook delivery log — what this deployment has announced to the + // operator's endpoint, and what it failed to. Read-only: there is no + // endpoint here that re-queues or deletes a delivery, deliberately (see + // WebhookHandlers). + mux.HandleFunc("GET /v1/admin/webhooks/deliveries", RequireAdmin(engine, hooks.Deliveries)) + return mux } diff --git a/httpapi/webhook_handlers.go b/httpapi/webhook_handlers.go new file mode 100644 index 0000000..f21bba4 --- /dev/null +++ b/httpapi/webhook_handlers.go @@ -0,0 +1,147 @@ +package httpapi + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/crydensync/api/webhook" +) + +// WebhookHandlers answers the admin webhook delivery log. +// +// Read-only, like the two reports beside it on the admin surface: it lists +// what the delivery worker has done and can take no action. Deliberately no +// "retry this delivery" endpoint here — that would be a write on a surface +// whose neighbouring feature is the AI-assisted tooling CLAUDE.md keeps +// read-only, and re-queuing a delivery is a decision an operator should make +// in the database with the evidence in front of them rather than through a +// button whose consequences are a third party's. +type WebhookHandlers struct { + // Store is this repo's own delivery log (see the webhook package). Nil + // when WEBHOOK_URL is unset, which is a wiring fact rather than a + // server fault — cryden's own "not configured" convention, answered as + // a 404. + Store webhook.Store +} + +// deliveryDTO is one row of the log. It carries the payload as raw JSON +// rather than re-encoding it, so what a console shows is the body that was +// actually POSTed — including key order, which is what a receiver's +// signature is computed over. +type deliveryDTO struct { + ID int64 `json:"id"` + EventID string `json:"event_id"` + EventType string `json:"event_type"` + UserID string `json:"user_id,omitempty"` + IP string `json:"ip,omitempty"` + + Payload json.RawMessage `json:"payload"` + + Status webhook.Status `json:"status"` + + // Attempts counts attempts STARTED, which is why it is worth reporting + // rather than hiding: a delivery at 3 of 5 tells an operator the + // endpoint has been failing, and a row that failed outright says how + // many tries it got. + Attempts int `json:"attempts"` + + // ResponseCode is absent when no response arrived at all — a connection + // failure or a timeout, which is a different problem from a receiver + // answering 500, and one an operator fixes in a different place. + ResponseCode int `json:"response_code,omitempty"` + + // Error is the receiver's own words where it gave any, so a failing + // delivery can be diagnosed from this response alone. + Error string `json:"error,omitempty"` + + DurationMS int `json:"duration_ms"` + + CreatedAt time.Time `json:"created_at"` + NextAttemptAt time.Time `json:"next_attempt_at"` + DeliveredAt *time.Time `json:"delivered_at,omitempty"` +} + +// deliveriesDTO is the whole response. Statuses is included so a console can +// offer the filter without hardcoding the four values, and Status echoes the +// filter in force — an operator looking at a short list needs to know +// whether it is short because of the filter or because of the traffic. +type deliveriesDTO struct { + Deliveries []deliveryDTO `json:"deliveries"` + Count int `json:"count"` + Status webhook.Status `json:"status,omitempty"` + Statuses []webhook.Status `json:"statuses"` +} + +// Deliveries — admin required (see router.go). Lists the delivery log +// newest first, optionally filtered by status. +func (h *WebhookHandlers) Deliveries(w http.ResponseWriter, r *http.Request) { + if h.Store == nil { + writeErr(w, errAdminStoresUnavailable) + return + } + + limit, err := queryLimit(r) + if err != nil { + writeBadRequest(w, err.Error()) + return + } + + var status webhook.Status + if raw := queryString(r, "status"); raw != "" { + var err error + status, err = webhook.ParseStatus(raw) + if err != nil { + // Reported as a 400 naming the four valid values rather than as + // an unmapped error: an empty list would look like "no + // deliveries", which is the one thing a filter must never be + // able to be confused with. + writeBadRequest(w, fmt.Sprintf("status must be one of: %s", strings.Join(statusNames(), ", "))) + return + } + } + + rows, err := h.Store.List(r.Context(), status, limit) + if err != nil { + writeErr(w, err) + return + } + + out := make([]deliveryDTO, 0, len(rows)) + for _, d := range rows { + out = append(out, deliveryDTO{ + ID: d.ID, + EventID: d.EventID, + EventType: d.EventType, + UserID: d.UserID, + IP: d.IP, + Payload: d.Payload, + Status: d.Status, + Attempts: d.Attempts, + ResponseCode: d.ResponseCode, + Error: d.Error, + DurationMS: d.DurationMS, + CreatedAt: d.CreatedAt, + NextAttemptAt: d.NextAttemptAt, + DeliveredAt: d.DeliveredAt, + }) + } + + writeData(w, http.StatusOK, deliveriesDTO{ + Deliveries: out, + Count: len(out), + Status: status, + Statuses: webhook.Statuses(), + }) +} + +func statusNames() []string { + statuses := webhook.Statuses() + out := make([]string, 0, len(statuses)) + for _, s := range statuses { + out = append(out, string(s)) + } + return out +} diff --git a/httpapi/webhook_handlers_test.go b/httpapi/webhook_handlers_test.go new file mode 100644 index 0000000..92e59ea --- /dev/null +++ b/httpapi/webhook_handlers_test.go @@ -0,0 +1,477 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/crydensync/cryden/v2" + "github.com/crydensync/cryden/v2/notify" + "github.com/crydensync/cryden/v2/store/memory" + "github.com/crydensync/cryden/v2/token" + + "github.com/crydensync/api/config" + "github.com/crydensync/api/webhook" +) + +// webhookClock is a clock a test moves by hand, so a listing's ordering is +// something the test decides rather than something it hopes the wall clock +// separated by enough microseconds to be reproducible. +type webhookClock struct { + mu sync.Mutex + t time.Time +} + +func newWebhookClock() *webhookClock { + return &webhookClock{t: time.Date(2026, time.September, 15, 12, 0, 0, 0, time.UTC)} +} + +func (c *webhookClock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *webhookClock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = c.t.Add(d) +} + +// deliveriesResponse mirrors the endpoint's DTO field by field, so a +// renamed or dropped field fails here rather than silently changing the +// contract an operator's console reads. +type deliveriesResponse struct { + Data struct { + Deliveries []struct { + ID int64 `json:"id"` + EventID string `json:"event_id"` + EventType string `json:"event_type"` + UserID string `json:"user_id"` + IP string `json:"ip"` + Payload json.RawMessage `json:"payload"` + Status string `json:"status"` + Attempts int `json:"attempts"` + ResponseCode int `json:"response_code"` + Error string `json:"error"` + DurationMS int `json:"duration_ms"` + CreatedAt time.Time `json:"created_at"` + DeliveredAt *time.Time `json:"delivered_at"` + } `json:"deliveries"` + Count int `json:"count"` + Status string `json:"status"` + Statuses []string `json:"statuses"` + } `json:"data"` +} + +// deliveryOutcome is what a test reads off one raw row, with presence +// recorded separately from value: an omitted response_code and a +// response_code of 0 decode to the same int, and the endpoint's whole +// reason for omitting it is that those two are different things. +type deliveryOutcome struct { + status string + responseCode int + errText string + hasDelivered bool + hasCode bool +} + +type webhookFixture struct { + store *webhook.MemoryStore + clock *webhookClock + router http.Handler + + adminToken string + userToken string +} + +// newWebhookFixture builds an engine whose operator carries the admin +// claim, a router holding the delivery log, and both tokens. The store is +// filled through webhook.Sender rather than by hand, so what these tests +// list is the body the sender actually produced — the same bytes the +// worker would POST, which is the whole point of the log. +func newWebhookFixture(t *testing.T) webhookFixture { + t.Helper() + ctx := context.Background() + + store := webhook.NewMemoryStore() + clock := newWebhookClock() + store.Clock = clock.now + + var adminID string + engine, err := cryden.New(cryden.Config{ + JWTSecret: "test-secret", + Users: memory.NewUserStore(), + Sessions: memory.NewSessionStore(), + Audit: memory.NewAuditStore(), + Verifications: memory.NewVerificationStore(), + EmailSender: stubMailSender{}, + MagicLinkSender: stubMailSender{}, + AccessTokenClaims: token.ClaimsFunc(func(_ context.Context, userID string) (map[string]any, error) { + if userID == adminID { + return map[string]any{"role": "admin"}, nil + } + return nil, nil + }), + }) + if err != nil { + t.Fatalf("cryden.New on the in-memory stores: %v", err) + } + + admin, err := cryden.SignUp(ctx, engine, "operator@example.com", testPassword, "203.0.113.1") + if err != nil { + t.Fatalf("signup (operator): %v", err) + } + adminID = admin.ID + adminTokens, err := cryden.Login(ctx, engine, "operator@example.com", testPassword, "203.0.113.1", chromeOnMacOS) + if err != nil { + t.Fatalf("login (operator): %v", err) + } + + const userEmail = "dana@example.com" + if _, err := cryden.SignUp(ctx, engine, userEmail, testPassword, "203.0.113.2"); err != nil { + t.Fatalf("signup (user): %v", err) + } + userTokens, err := cryden.Login(ctx, engine, userEmail, testPassword, "203.0.113.2", chromeOnMacOS) + if err != nil { + t.Fatalf("login (user): %v", err) + } + + return webhookFixture{ + store: store, + clock: clock, + router: NewRouter(Deps{Engine: engine, Config: config.Config{}, Hooks: store}), + adminToken: adminTokens.AccessToken, + userToken: userTokens.AccessToken, + } +} + +// seed records an event through the real sender and returns its row id. +func (f webhookFixture) seed(t *testing.T, eventType, reason string) int64 { + t.Helper() + sender := &webhook.Sender{Store: f.store} + err := sender.SendWebhook(context.Background(), notify.WebhookEvent{ + ID: "evt_" + eventType, + Type: eventType, + UserID: "01a0a4ce-5453-78d3-9126-52268da8da5c", + IP: "203.0.113.9", + Metadata: map[string]string{"reason": reason}, + OccurredAt: f.clock.now(), + }) + if err != nil { + t.Fatalf("enqueueing %s: %v", eventType, err) + } + rows, err := f.store.List(context.Background(), "", 10) + if err != nil { + t.Fatalf("List: %v", err) + } + return rows[0].ID +} + +// resolve drives a seeded row to a terminal state the way the worker +// would, so the listing's status filter has something real to filter on. +func (f webhookFixture) resolve(t *testing.T, id int64, delivered bool) { + t.Helper() + ctx := context.Background() + if _, err := f.store.ClaimDue(ctx, f.clock.now(), 10, 5, time.Minute); err != nil { + t.Fatalf("ClaimDue: %v", err) + } + if delivered { + if err := f.store.MarkDelivered(ctx, id, webhook.Result{Code: http.StatusOK, Duration: 42 * time.Millisecond}); err != nil { + t.Fatalf("MarkDelivered: %v", err) + } + return + } + if err := f.store.MarkFailed(ctx, id, webhook.Result{Code: http.StatusInternalServerError, Duration: 7 * time.Millisecond, Err: "receiver answered 500"}, nil); err != nil { + t.Fatalf("MarkFailed: %v", err) + } +} + +func (f webhookFixture) list(t *testing.T, token, query string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/v1/admin/webhooks/deliveries"+query, nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + f.router.ServeHTTP(rec, req) + return rec +} + +func decodeDeliveries(t *testing.T, rec *httptest.ResponseRecorder) deliveriesResponse { + t.Helper() + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + var resp deliveriesResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decoding %s: %v", rec.Body.String(), err) + } + return resp +} + +func TestWebhookDeliveriesListsNewestFirst(t *testing.T) { + f := newWebhookFixture(t) + + first := f.seed(t, "account_locked", "too many attempts") + f.clock.advance(time.Minute) + second := f.seed(t, "password_reset", "requested by the user") + + resp := decodeDeliveries(t, f.list(t, f.adminToken, "")) + if resp.Data.Count != 2 { + t.Fatalf("count = %d, want 2", resp.Data.Count) + } + if len(resp.Data.Deliveries) != 2 { + t.Fatalf("listed %d rows, want 2", len(resp.Data.Deliveries)) + } + if resp.Data.Deliveries[0].ID != second || resp.Data.Deliveries[1].ID != first { + t.Errorf("order = %d, %d, want the newest (%d) first", resp.Data.Deliveries[0].ID, resp.Data.Deliveries[1].ID, second) + } + + // The filter values are reported rather than hardcoded, so a console + // can offer the filter without a second source of truth for it. + want := []string{"pending", "in_flight", "delivered", "failed"} + if strings.Join(resp.Data.Statuses, ",") != strings.Join(want, ",") { + t.Errorf("statuses = %v, want %v", resp.Data.Statuses, want) + } + // No filter in force, so nothing is echoed back. + if resp.Data.Status != "" { + t.Errorf("status = %q on an unfiltered listing, want empty", resp.Data.Status) + } +} + +// The log's reason for existing: what was sent is readable after the fact, +// for a retry as much as a first attempt. The payload is emitted as raw +// JSON rather than a re-encoding — or worse, a base64 string — which is +// the property a console displaying "what we signed" depends on. +func TestWebhookDeliveriesServesTheBodyThatWasSent(t *testing.T) { + f := newWebhookFixture(t) + f.seed(t, "account_locked", "too many attempts") + + rec := f.list(t, f.adminToken, "") + resp := decodeDeliveries(t, rec) + d := resp.Data.Deliveries[0] + + var body struct { + ID string `json:"id"` + Type string `json:"type"` + Metadata map[string]string `json:"metadata"` + } + if err := json.Unmarshal(d.Payload, &body); err != nil { + t.Fatalf("payload is not a JSON object: %v (%s)", err, d.Payload) + } + if body.ID != "evt_account_locked" || body.Type != "account_locked" { + t.Errorf("payload id/type = %q/%q", body.ID, body.Type) + } + if body.Metadata["reason"] != "too many attempts" { + t.Errorf("payload metadata = %v, want the event's own", body.Metadata) + } + + // The raw body, not the decoded DTO: json.RawMessage is inlined + // verbatim and []byte would have become a base64 string, so this is + // the assertion that distinguishes the two. + if !strings.Contains(rec.Body.String(), `"payload":{`) { + t.Errorf("payload was not emitted as a JSON object: %s", rec.Body.String()) + } +} + +// A row that reached a terminal state is readable with the evidence that +// decided it, and the two failure shapes stay distinguishable: a receiver +// that answered 500 and an endpoint that never answered at all are fixed +// in different places. +func TestWebhookDeliveriesCarryTheirOutcome(t *testing.T) { + f := newWebhookFixture(t) + + delivered := f.seed(t, "account_locked", "one") + f.clock.advance(time.Minute) + failed := f.seed(t, "account_locked", "two") + f.clock.advance(time.Minute) + unreachable := f.seed(t, "account_locked", "three") + + f.resolve(t, delivered, true) + f.resolve(t, failed, false) + // A connection that never answered: code 0, with the transport's own + // error and nothing that looks like an HTTP status. + if _, err := f.store.ClaimDue(context.Background(), f.clock.now(), 10, 5, time.Minute); err != nil { + t.Fatalf("ClaimDue: %v", err) + } + if err := f.store.MarkFailed(context.Background(), unreachable, webhook.Result{ + Code: 0, Duration: 2 * time.Second, Err: "dial tcp 203.0.113.9:443: connect: connection refused", + }, nil); err != nil { + t.Fatalf("MarkFailed: %v", err) + } + + // Read off the raw response rather than the DTO, because the two + // assertions that matter most here are about fields being ABSENT — + // response_code on a connection that never answered, delivered_at on a + // row that failed — and an absent field is invisible once decoded. + rec := f.list(t, f.adminToken, "") + var raw struct { + Data struct { + Deliveries []map[string]json.RawMessage `json:"deliveries"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &raw); err != nil { + t.Fatalf("decoding %s: %v", rec.Body.String(), err) + } + byID := map[int64]deliveryOutcome{} + for _, row := range raw.Data.Deliveries { + var id int64 + if err := json.Unmarshal(row["id"], &id); err != nil { + t.Fatalf("decoding id: %v", err) + } + got := deliveryOutcome{hasDelivered: row["delivered_at"] != nil, hasCode: row["response_code"] != nil} + if err := json.Unmarshal(row["status"], &got.status); err != nil { + t.Fatalf("decoding status: %v", err) + } + if got.hasCode { + if err := json.Unmarshal(row["response_code"], &got.responseCode); err != nil { + t.Fatalf("decoding response_code: %v", err) + } + } + if raw, ok := row["error"]; ok { + if err := json.Unmarshal(raw, &got.errText); err != nil { + t.Fatalf("decoding error: %v", err) + } + } + byID[id] = got + } + + if got := byID[delivered]; got.status != "delivered" || got.responseCode != http.StatusOK || !got.hasDelivered { + t.Errorf("delivered row = %+v, want delivered with a 200 and delivered_at set", got) + } + if got := byID[failed]; got.status != "failed" || got.responseCode != http.StatusInternalServerError { + t.Errorf("failed row = %+v, want failed with a 500", got) + } + // The important half: no response code at all, not a zero an operator + // would read as a status. + if got := byID[unreachable]; got.status != "failed" || got.hasCode { + t.Errorf("unreachable row = %+v, want failed with response_code absent", got) + } + if got := byID[unreachable]; !strings.Contains(got.errText, "connection refused") { + t.Errorf("unreachable row error = %q, want the transport's own words", got.errText) + } +} + +func TestWebhookDeliveriesFiltersByStatus(t *testing.T) { + f := newWebhookFixture(t) + + // Resolved while it is the only row, deliberately: ClaimDue sweeps + // every due row in one pass, so seeding both first and then resolving + // one would leave the other claimed and in_flight — real worker + // behaviour, but not the state this test is about. + ok := f.seed(t, "account_locked", "one") + f.resolve(t, ok, true) + + f.clock.advance(time.Minute) + f.seed(t, "password_reset", "two") + + pending := decodeDeliveries(t, f.list(t, f.adminToken, "?status=pending")) + if pending.Data.Count != 1 { + t.Fatalf("pending count = %d, want 1", pending.Data.Count) + } + if pending.Data.Status != "pending" { + t.Errorf("status = %q, want the filter echoed back", pending.Data.Status) + } + + delivered := decodeDeliveries(t, f.list(t, f.adminToken, "?status=delivered")) + if delivered.Data.Count != 1 { + t.Fatalf("delivered count = %d, want 1", delivered.Data.Count) + } + if delivered.Data.Deliveries[0].ID != ok { + t.Errorf("delivered row = %d, want %d", delivered.Data.Deliveries[0].ID, ok) + } +} + +// An unrecognized status is a 400 that names the four real values. It must +// not be an empty list, which is indistinguishable from "no deliveries" — +// the one thing a filter must never be able to look like. +func TestWebhookDeliveriesRejectsAnUnknownStatus(t *testing.T) { + f := newWebhookFixture(t) + f.seed(t, "account_locked", "one") + + for _, query := range []string{"?status=done", "?status=PENDING", "?status=all"} { + rec := f.list(t, f.adminToken, query) + if rec.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400 (body %s)", query, rec.Code, rec.Body.String()) + continue + } + body := rec.Body.String() + for _, want := range []string{"pending", "in_flight", "delivered", "failed"} { + if !strings.Contains(body, want) { + t.Errorf("%s: body = %s, want the valid values including %q", query, body, want) + } + } + } +} + +func TestWebhookDeliveriesBoundsTheLimit(t *testing.T) { + f := newWebhookFixture(t) + for i := 0; i < 3; i++ { + f.seed(t, "account_locked", "x") + f.clock.advance(time.Minute) + } + + limited := decodeDeliveries(t, f.list(t, f.adminToken, "?limit=2")) + if limited.Data.Count != 2 { + t.Errorf("count = %d with limit=2, want 2", limited.Data.Count) + } + + // Bounded, not clamped: a caller that asked for 100000 and got 500 back + // has no way to tell that from a table holding 500 rows. + for _, query := range []string{"?limit=0", "?limit=501", "?limit=lots"} { + rec := f.list(t, f.adminToken, query) + if rec.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400 (body %s)", query, rec.Code, rec.Body.String()) + } + } +} + +// A router built without a delivery log answers 404 rather than 500 — a +// wiring fact, not a server fault, and the same shape every unconfigured +// feature in this API uses. +// +// The handler is called directly rather than through a router because this +// repo has no engine without the stores either: the deployment this covers +// is one where WEBHOOK_URL is unset, which leaves Deps.Hooks genuinely nil +// while everything else is wired. That guard is a single branch, and this +// is the only way to reach it. +func TestWebhookDeliveriesWithoutAStoreIsNotFound(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/v1/admin/webhooks/deliveries", nil) + rec := httptest.NewRecorder() + + h := &WebhookHandlers{} + h.Deliveries(rec, req) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404 (body %s)", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "not_configured") { + t.Errorf("body = %s, want the not_configured code", rec.Body.String()) + } +} + +// The delivery log is evidence about accounts and about a third party's +// responses, so it sits behind the same gate as every other admin report. +func TestWebhookDeliveriesRouteIsGatedByRequireAdmin(t *testing.T) { + f := newWebhookFixture(t) + f.seed(t, "account_locked", "one") + + if rec := f.list(t, "", ""); rec.Code != http.StatusUnauthorized { + t.Errorf("no token: status = %d, want 401", rec.Code) + } + if rec := f.list(t, "not-a-real-token", ""); rec.Code != http.StatusUnauthorized { + t.Errorf("garbage token: status = %d, want 401", rec.Code) + } + if rec := f.list(t, f.userToken, ""); rec.Code != http.StatusForbidden { + t.Errorf("ordinary user: status = %d, want 403 (body %s)", rec.Code, rec.Body.String()) + } + if rec := f.list(t, f.adminToken, ""); rec.Code != http.StatusOK { + t.Errorf("operator: status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } +} diff --git a/main.go b/main.go index f285307..1b8c5de 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "database/sql" "log" "net/http" @@ -17,6 +18,7 @@ import ( "github.com/crydensync/api/operator" "github.com/crydensync/api/templates" "github.com/crydensync/api/usermeta" + "github.com/crydensync/api/webhook" ) func main() { @@ -50,6 +52,23 @@ func main() { // reserved-key rule lives in the store rather than in the handler. metadata := usermeta.NewStore(db) + // Webhook delivery log: this repo's own table, and the queue the + // sender writes to. Declared as the interface rather than as + // *webhook.PostgresStore so that leaving WEBHOOK_URL unset leaves it + // genuinely nil — a typed nil inside a non-nil interface is a value + // that passes every nil-interface check and then panics on use, and + // the router's handlers guard on exactly that check. + var webhookStore webhook.Store + var webhookWake chan struct{} + if cfg.WebhookURL != "" { + webhookStore = webhook.NewStore(db) + // Capacity 1, used purely as a nudge: the sender's job is to make + // the row and return, so a full channel must drop the hint rather + // than block. The worker's poll interval is the safety net for a + // hint dropped here. + webhookWake = make(chan struct{}, 1) + } + // Email templates are optional and entirely this repo's: cryden owns // no message copy. An unset EMAIL_TEMPLATE_DIR leaves both senders // printing their own built-in line, byte for byte as before. @@ -175,11 +194,52 @@ func main() { log.Printf("engine rate limiting is Redis-backed") } + // Webhooks, set here rather than in the literal above for the same + // reason the hasher is: assigning a nil *webhook.Sender into a + // notify.WebhookSender field would make it non-nil as far as cryden + // can tell, and cryden reads a non-nil sender with no events as a + // request for DefaultWebhookEvents. So the field is only ever touched + // when there is an endpoint to deliver to. + // + // Setting these two fields IS the dispatch wiring: cryden wraps + // Config.Audit in its own webhookRecorder, so there is no registry to + // populate. Leaving WebhookEvents empty is the deliberate default — + // cryden then uses DefaultWebhookEvents, which names the sixteen + // events worth waking someone for and excludes + // login_success/login_failed/token_rotated, the three that fire + // constantly and say nothing. + if webhookStore != nil { + engineCfg.Webhooks = &webhook.Sender{Store: webhookStore, Wake: webhookWake} + engineCfg.WebhookEvents = cfg.WebhookEvents + } + engine, err := cryden.New(engineCfg) if err != nil { log.Fatalf("failed to construct cryden engine: %v", err) } + // The delivery worker. Started only when there is somewhere to + // deliver to, so an unconfigured deployment runs no goroutine at all. + // + // Run takes context.Background() because this repo has no graceful + // shutdown anywhere yet — main.go ends at log.Fatal(ListenAndServe), + // which exits the process and every goroutine with it. Introducing a + // real shutdown touches every component and is its own change; noted + // in PROGRESS.md as still owed rather than smuggled in here. + if webhookStore != nil { + worker := webhook.NewWorker(webhookStore, cfg.WebhookURL, cfg.WebhookSecret) + worker.MaxAttempts = cfg.WebhookMaxAttempts + worker.Wake = webhookWake + worker.Log = log.Default() + go worker.Run(context.Background()) + + events := len(cfg.WebhookEvents) + if events == 0 { + events = len(cryden.DefaultWebhookEvents()) + } + log.Printf("webhook deliveries enabled: %d event types, up to %d attempts each", events, cfg.WebhookMaxAttempts) + } + router := httpapi.NewRouter(httpapi.Deps{ Engine: engine, DB: db, @@ -187,6 +247,7 @@ func main() { Audit: audit, Users: users, Meta: metadata, + Hooks: webhookStore, }) limiter := httpapi.NewEdgeRateLimiter(cfg.EdgeRateLimit, cfg.EdgeRateLimitWindow) handler := httpapi.WithCORS(cfg.CORSOrigins, httpapi.WithEdgeRateLimit(limiter, router)) diff --git a/migrations/010_webhook_deliveries.down.sql b/migrations/010_webhook_deliveries.down.sql new file mode 100644 index 0000000..8b8943b --- /dev/null +++ b/migrations/010_webhook_deliveries.down.sql @@ -0,0 +1,10 @@ +-- 010_webhook_deliveries.down.sql +-- +-- Drops the indexes with the table rather than relying on the table drop +-- to take them: this repo's migrations are applied by hand as often as +-- by a runner, and an explicit drop is what makes the file safe to run +-- twice. + +DROP INDEX IF EXISTS idx_webhook_deliveries_created; +DROP INDEX IF EXISTS idx_webhook_deliveries_due; +DROP TABLE IF EXISTS webhook_deliveries; diff --git a/migrations/010_webhook_deliveries.up.sql b/migrations/010_webhook_deliveries.up.sql new file mode 100644 index 0000000..c0c8e3a --- /dev/null +++ b/migrations/010_webhook_deliveries.up.sql @@ -0,0 +1,99 @@ +-- 010_webhook_deliveries.up.sql +-- +-- The delivery log for engine webhooks: one row per event the host +-- chose to deliver, whether or not it has been delivered yet. This is +-- the queue AND the record — cryden calls Config.Webhooks synchronously +-- on the request path (see notify.WebhookSender's own doc comment), so +-- the only implementation worth deploying is an enqueue, and a queue +-- that is not also queryable cannot answer the question the log exists +-- for: "was that lockout actually announced?" +-- +-- user_id carries NO foreign key, deliberately. This is the same +-- reasoning audit_events and login_attempts are built on: a delivery is +-- evidence about an account, and evidence about a deleted account keeps +-- its value without the account. ON DELETE CASCADE here would mean +-- deleting a user silently rewrites the history of what that deletion +-- was reported to have triggered. +-- +-- There is no "pending" row for a delivery that could not be queued: +-- the row IS the queue. + +CREATE TABLE webhook_deliveries ( + -- The row's own identity, and the queue's ordering. A surrogate + -- rather than the engine's event id as the primary key, because + -- that id is not guaranteed to be present: cryden generates it with + -- crypto/rand and, on a generator failure, deliberately delivers the + -- event with an EMPTY id rather than dropping it ("the delivery is + -- worth more than its idempotency key" — see webhooks.go). Keying on + -- it would turn that decision into a unique-constraint violation and + -- lose exactly the event the engine went out of its way to keep. + id BIGSERIAL PRIMARY KEY, + + -- The engine's idempotency key for this occurrence, sent to the + -- receiver as X-Cryden-Event-Id. Indexed, not unique: it is the + -- receiver's dedupe key, and the engine's contract explicitly allows + -- it to be empty, where two empty values are two different events. + event_id TEXT NOT NULL DEFAULT '', + + -- The store.AuditEventType that was recorded, as a string. Not an + -- enum: this repo must not need a migration to learn about an event + -- type the engine adds. + event_type TEXT NOT NULL, + user_id UUID, + ip TEXT, + + -- The event body, as it goes on the wire. The delivery worker reads + -- this column back out and sends those bytes unchanged — it does not + -- rebuild the body from the other columns, because a second + -- implementation of the payload is free to drift from what the + -- console shows an operator, and "show me what we sent that + -- endpoint" is the question this log exists to answer. + -- + -- JSONB rather than TEXT does not cost that exactness, which is + -- worth spelling out since it is not obvious: Postgres emits a + -- JSONB value with its keys in a deterministic order, so reading + -- the column back twice yields identical bytes. The bytes signed + -- and the bytes displayed are the same bytes, and a receiver + -- checking a live delivery against the log computes the same + -- digest. That also means the digest is over the NORMALISED body, + -- not over whatever key order the sender happened to emit. + payload JSONB NOT NULL, + + -- pending -> in_flight -> delivered + -- -> pending (retry, next_attempt_at pushed out) + -- -> failed (out of attempts, or abandoned) + status TEXT NOT NULL DEFAULT 'pending', + + -- Attempts STARTED, incremented when a row is claimed rather than + -- when it fails. A process that died mid-delivery still made the + -- attempt — the receiver may well have got it, since at-least-once + -- is the only promise a queue can make. It follows that a crash can + -- leave this one above the configured maximum; that is preferred to + -- the alternative, a row stuck at "in flight" that nothing will ever + -- finish or report on. + attempts INT NOT NULL DEFAULT 0, + + response_code INT, + error TEXT, + duration_ms INT, + + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- When this row next becomes claimable. Set to now() at insert, so a + -- fresh event is due immediately. + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Stamped when a worker claims the row and cleared whenever it + -- resolves it. A row left in_flight past a staleness bound is one + -- whose worker stopped mid-delivery, and is reclaimed rather than + -- stranded. + claimed_at TIMESTAMPTZ, + delivered_at TIMESTAMPTZ +); + +-- The claim query's index, and the one an admin listing filtered by +-- status uses. Leading with status makes the partial scan cheap: the +-- rows a worker wants are the pending ones, and this never walks the +-- delivered history that will be the bulk of the table. +CREATE INDEX idx_webhook_deliveries_due ON webhook_deliveries(status, next_attempt_at); + +-- The admin listing's default view — newest first, unfiltered. +CREATE INDEX idx_webhook_deliveries_created ON webhook_deliveries(created_at DESC); diff --git a/webhook/memory.go b/webhook/memory.go new file mode 100644 index 0000000..40e345b --- /dev/null +++ b/webhook/memory.go @@ -0,0 +1,231 @@ +package webhook + +import ( + "context" + "fmt" + "sort" + "sync" + "time" +) + +// MemoryStore is the in-process Store, for tests and for any embedding host +// that wants webhook delivery without a database behind it. +// +// It is a faithful double rather than a convenient one, because the worker +// tests it carries are the only tests the worker's claim/backoff/terminal +// behaviour gets in this repo — the Postgres path needs a live server, and +// FOR UPDATE SKIP LOCKED has no meaning without one. So it reproduces the +// three promises that are easy to get subtly wrong: +// +// - ClaimDue's two branches, including that the in_flight branch ignores +// the attempt budget. A double that only reclaimed by budget would let +// a stranded row look fine here and stay stranded in production. +// - ClaimDue increments Attempts, at claim time. A double that +// incremented on failure would make every backoff test pass while the +// real counter meant something else. +// - Returned rows are copies. A caller holding a Delivery cannot reach +// the stored one, the way a scanned row cannot reach the database. A +// double that handed out its own pointers would let a worker "resolve" +// a delivery by editing a struct. +// +// What it does NOT reproduce is concurrency: it is one mutex, so it can +// never actually exercise two workers racing for the same row. That is a +// real limitation of testing this way and is said out loud in PROGRESS.md +// rather than implied away. +type MemoryStore struct { + mu sync.Mutex + rows map[int64]Delivery + next int64 + + // Clock is the store's own notion of "now", used for created_at and for + // the timestamps it stamps itself. ClaimDue takes now as a parameter and + // does not use this; it is here so a test can make created_at ordering + // deterministic. + Clock func() time.Time +} + +func NewMemoryStore() *MemoryStore { + return &MemoryStore{ + rows: make(map[int64]Delivery), + Clock: func() time.Time { return time.Now().UTC() }, + } +} + +var _ Store = (*MemoryStore)(nil) + +func (s *MemoryStore) Enqueue(_ context.Context, d Delivery) error { + s.mu.Lock() + defer s.mu.Unlock() + + s.next++ + d.ID = s.next + d.Status = StatusPending + d.Attempts = 0 + d.ResponseCode = 0 + d.Error = "" + d.DurationMS = 0 + d.ClaimedAt = nil + d.DeliveredAt = nil + if d.CreatedAt.IsZero() { + d.CreatedAt = s.Clock() + } + if d.NextAttemptAt.IsZero() { + d.NextAttemptAt = d.CreatedAt + } + // A copy of the payload, so a caller reusing its buffer cannot rewrite + // the stored body. + d.Payload = append([]byte(nil), d.Payload...) + s.rows[d.ID] = d + return nil +} + +func (s *MemoryStore) ClaimDue(_ context.Context, now time.Time, limit, maxAttempts int, staleAfter time.Duration) ([]Delivery, error) { + s.mu.Lock() + defer s.mu.Unlock() + + staleBefore := now.Add(-staleAfter) + + var due []Delivery + for _, d := range s.rows { + switch d.Status { + case StatusPending: + if d.Attempts < maxAttempts && !d.NextAttemptAt.After(now) { + due = append(due, d) + } + case StatusInFlight: + // No attempt-budget check here, deliberately — see the Store + // doc. This is the branch that keeps a row whose worker died + // from being stranded forever. + if d.ClaimedAt != nil && d.ClaimedAt.Before(staleBefore) { + due = append(due, d) + } + } + } + + // Ordered the way the SQL is: oldest due first, ties broken by id so + // the order is total and a test can rely on it. + sort.Slice(due, func(i, j int) bool { + if !due[i].NextAttemptAt.Equal(due[j].NextAttemptAt) { + return due[i].NextAttemptAt.Before(due[j].NextAttemptAt) + } + return due[i].ID < due[j].ID + }) + if len(due) > limit { + due = due[:limit] + } + + claimed := make([]Delivery, 0, len(due)) + for _, d := range due { + stored := s.rows[d.ID] + stored.Status = StatusInFlight + stored.Attempts++ + claimedAt := now + stored.ClaimedAt = &claimedAt + s.rows[d.ID] = stored + claimed = append(claimed, copyDelivery(stored)) + } + return claimed, nil +} + +func (s *MemoryStore) MarkDelivered(_ context.Context, id int64, r Result) error { + s.mu.Lock() + defer s.mu.Unlock() + + d, ok := s.rows[id] + if !ok { + return fmt.Errorf("%w: id %d", ErrNotFound, id) + } + d.Status = StatusDelivered + d.ClaimedAt = nil + deliveredAt := s.Clock() + d.DeliveredAt = &deliveredAt + d.ResponseCode = r.Code + d.DurationMS = int(r.Duration.Milliseconds()) + d.Error = "" + s.rows[id] = d + return nil +} + +func (s *MemoryStore) MarkFailed(_ context.Context, id int64, r Result, retryAt *time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + + d, ok := s.rows[id] + if !ok { + return fmt.Errorf("%w: id %d", ErrNotFound, id) + } + d.ClaimedAt = nil + d.ResponseCode = r.Code + d.DurationMS = int(r.Duration.Milliseconds()) + d.Error = r.Err + if retryAt != nil { + d.Status = StatusPending + d.NextAttemptAt = *retryAt + } else { + d.Status = StatusFailed + } + s.rows[id] = d + return nil +} + +func (s *MemoryStore) List(_ context.Context, status Status, limit int) ([]Delivery, error) { + s.mu.Lock() + defer s.mu.Unlock() + + out := make([]Delivery, 0, len(s.rows)) + for _, d := range s.rows { + if status != "" && d.Status != status { + continue + } + out = append(out, copyDelivery(d)) + } + // Newest first, ties broken by id descending — the SQL's ORDER BY + // created_at DESC, id DESC. + sort.Slice(out, func(i, j int) bool { + if !out[i].CreatedAt.Equal(out[j].CreatedAt) { + return out[i].CreatedAt.After(out[j].CreatedAt) + } + return out[i].ID > out[j].ID + }) + if len(out) > limit { + out = out[:limit] + } + return out, nil +} + +// Get returns one delivery by id, for a test that wants to assert on a row +// it did not get back from a claim. Not part of Store: nothing in +// production needs it, and PostgresStore has no equivalent. +func (s *MemoryStore) Get(id int64) (Delivery, bool) { + s.mu.Lock() + defer s.mu.Unlock() + d, ok := s.rows[id] + return copyDelivery(d), ok +} + +// Count returns how many rows exist, optionally filtered by status. A test +// helper, like Get. +func (s *MemoryStore) Count(status Status) int { + s.mu.Lock() + defer s.mu.Unlock() + n := 0 + for _, d := range s.rows { + if status == "" || d.Status == status { + n++ + } + } + return n +} + +func copyDelivery(d Delivery) Delivery { + d.Payload = append([]byte(nil), d.Payload...) + if d.ClaimedAt != nil { + t := *d.ClaimedAt + d.ClaimedAt = &t + } + if d.DeliveredAt != nil { + t := *d.DeliveredAt + d.DeliveredAt = &t + } + return d +} diff --git a/webhook/sender.go b/webhook/sender.go new file mode 100644 index 0000000..0c28c1e --- /dev/null +++ b/webhook/sender.go @@ -0,0 +1,146 @@ +package webhook + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/crydensync/cryden/v2/notify" +) + +// Sender is the notify.WebhookSender this repo hands the engine. +// +// SendWebhook records a delivery and returns. It makes NO HTTP call — that +// is the entire reason it exists in this shape. cryden calls it +// synchronously, in the same goroutine as the login that triggered it (see +// notify.WebhookSender's own doc comment), so an http.Client.Do in here is +// a third party's downtime becoming this deployment's login latency. +// +// The queue is the database row, and that choice is deliberate rather than +// incidental: a channel would be faster and would lose everything on +// restart — and a crash between the audit write and the delivery is +// precisely the case this log exists to make visible. The channel here is +// only a nudge telling the worker there is likely to be work. +type Sender struct { + // Store is where the delivery row goes. Enqueue is the only method this + // type calls. + Store Store + + // Wake is nudged, non-blockingly, after a successful enqueue. It is a + // hint and never a guarantee: a full buffer, a nil channel or a worker + // that is not running all simply mean the worker finds the row on its + // next poll instead. Nothing about correctness depends on it. + Wake chan<- struct{} + + // EnqueueTimeout bounds the insert. Zero means DefaultEnqueueTimeout. + // + // A bound exists because the insert runs on the request path: without + // one, a wedged database would hold a login open indefinitely. Five + // seconds is long enough that a merely slow insert still makes it and + // short enough that a broken one fails while the user is still waiting. + EnqueueTimeout time.Duration +} + +// DefaultEnqueueTimeout bounds the insert SendWebhook performs. +const DefaultEnqueueTimeout = 5 * time.Second + +var _ notify.WebhookSender = (*Sender)(nil) + +// payload is the body a receiver gets. It is a struct rather than a +// map[string]any so the wire format is a decision made here and visible in +// one place, and so adding a field cannot silently change the order of the +// existing ones. +type payload struct { + // ID is the engine's idempotency key for this occurrence, and the same + // value as the X-Cryden-Event-Id header. Always present, possibly + // empty: cryden generates it with crypto/rand and deliberately + // delivers an event without one rather than not at all, so an empty + // string here is information ("the generator failed") rather than a + // bug. + ID string `json:"id"` + + // Type is the recorded audit event type, e.g. "account_locked". A + // receiver should treat an unrecognised value as a reason to ignore the + // event, not to fail: the engine adds types, and this repo must not + // need a deploy to learn about one. + Type string `json:"type"` + + // UserID is whose account the event concerns, empty for the events that + // genuinely have no user behind them — a failed login naming an email + // nobody registered, for one. + UserID string `json:"user_id,omitempty"` + IP string `json:"ip,omitempty"` + + // OccurredAt is when the engine recorded the event, in UTC. + OccurredAt time.Time `json:"occurred_at"` + + // Metadata is the audit event's own metadata, unchanged — its keys are + // documented per event type on cryden's constants. + // + // Note what is NOT here: the attempt number. The body is built once, at + // enqueue, and every retry sends these same bytes, which is what lets + // the delivery log answer "what did we send?" for a retry as well as a + // first attempt. Per-attempt information travels in the + // X-Cryden-Delivery-Attempt header instead. + Metadata map[string]string `json:"metadata,omitempty"` +} + +// SendWebhook implements notify.WebhookSender. It returns an error only when +// the delivery could not be recorded; cryden logs that and lets the login +// succeed regardless, which is the contract (a webhook is a notification, +// not a gate). +// +// A panic is not reported as an error — it propagates, exactly as the +// interface documents. Nothing here can panic on a plausible input, and +// swallowing one would hide a real bug behind a log line. +func (s *Sender) SendWebhook(ctx context.Context, event notify.WebhookEvent) error { + body, err := json.Marshal(payload{ + ID: event.ID, + Type: event.Type, + UserID: event.UserID, + IP: event.IP, + OccurredAt: event.OccurredAt.UTC(), + Metadata: event.Metadata, + }) + if err != nil { + return fmt.Errorf("encoding webhook payload: %w", err) + } + + // context.WithoutCancel, deliberately, and this is the one place the + // obvious implementation is wrong. ctx is the triggering request's, and + // the engine's own doc comment warns that it may already be cancelled + // by the time a slow sender gets to use it. An insert that honoured + // that would drop the event precisely when the request went away — + // and this row is the only record that it ever happened. The event is + // worth more than the connection it arrived on, the same reasoning + // cryden applies to a failed id generation. + // + // The timeout is what stops that from becoming "a wedged database holds + // a login open forever": cancellation is dropped, the deadline is not. + timeout := s.EnqueueTimeout + if timeout <= 0 { + timeout = DefaultEnqueueTimeout + } + enqueueCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout) + defer cancel() + + if err := s.Store.Enqueue(enqueueCtx, Delivery{ + EventID: event.ID, + EventType: event.Type, + UserID: event.UserID, + IP: event.IP, + Payload: body, + }); err != nil { + return err + } + + // Non-blocking, and nil-safe: a send on a nil channel would block + // forever, but a nil channel in a select is simply never ready, so this + // falls through to the worker's next poll. + select { + case s.Wake <- struct{}{}: + default: + } + return nil +} diff --git a/webhook/sender_test.go b/webhook/sender_test.go new file mode 100644 index 0000000..41d6907 --- /dev/null +++ b/webhook/sender_test.go @@ -0,0 +1,281 @@ +package webhook + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/crydensync/cryden/v2/notify" +) + +// testClock is a clock a test moves by hand, so a retry schedule can be +// exercised in microseconds instead of in real backoff delays. +type testClock struct { + mu sync.Mutex + t time.Time +} + +func newTestClock() *testClock { + return &testClock{t: time.Date(2026, time.September, 15, 12, 0, 0, 0, time.UTC)} +} + +func (c *testClock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *testClock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = c.t.Add(d) +} + +// receiver is a stand-in for whatever an operator points WEBHOOK_URL at. It +// records what it was sent so a test can assert on the body and headers a +// real endpoint would see, and answers with a status the test chooses. +type receiver struct { + mu sync.Mutex + requests []received + status int + body string +} + +type received struct { + header http.Header + body []byte +} + +func (r *receiver) ServeHTTP(w http.ResponseWriter, req *http.Request) { + body := make([]byte, 0) + buf := make([]byte, 4096) + for { + n, err := req.Body.Read(buf) + body = append(body, buf[:n]...) + if err != nil { + break + } + } + r.mu.Lock() + r.requests = append(r.requests, received{header: req.Header.Clone(), body: body}) + status, respBody := r.status, r.body + r.mu.Unlock() + + if status == 0 { + status = http.StatusOK + } + w.WriteHeader(status) + _, _ = w.Write([]byte(respBody)) +} + +func (r *receiver) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.requests) +} + +func (r *receiver) last(t *testing.T) received { + t.Helper() + r.mu.Lock() + defer r.mu.Unlock() + if len(r.requests) == 0 { + t.Fatal("the receiver was never called") + } + return r.requests[len(r.requests)-1] +} + +// newReceiver starts a real HTTP server. The worker's HTTP path is the +// production one down to the socket, which is the strongest thing available +// here without a live third party. +func newReceiver(t *testing.T, status int, body string) (*receiver, string) { + t.Helper() + r := &receiver{status: status, body: body} + srv := httptest.NewServer(r) + t.Cleanup(srv.Close) + return r, srv.URL +} + +func testEvent() notify.WebhookEvent { + return notify.WebhookEvent{ + ID: "evt_01", + Type: "account_locked", + UserID: "01a0a4ce-5453-78d3-9126-52268da8da5c", + IP: "203.0.113.9", + Metadata: map[string]string{"reason": "too many failed attempts"}, + OccurredAt: time.Date(2026, time.September, 15, 11, 59, 0, 0, time.UTC), + } +} + +// The whole design in one assertion: the sender records the event and does +// NOT call the endpoint. cryden calls this on the login request path, so an +// HTTP call here would be a third party's downtime becoming login latency. +func TestSendWebhookEnqueuesWithoutCallingTheEndpoint(t *testing.T) { + rec, _ := newReceiver(t, http.StatusOK, "") + store := NewMemoryStore() + + sender := &Sender{Store: store} + if err := sender.SendWebhook(context.Background(), testEvent()); err != nil { + t.Fatalf("SendWebhook: %v", err) + } + + if rec.count() != 0 { + t.Errorf("the endpoint was called %d times, want 0 — the sender must only enqueue", rec.count()) + } + if n := store.Count(StatusPending); n != 1 { + t.Fatalf("pending rows = %d, want 1", n) + } + + rows, err := store.List(context.Background(), StatusPending, 10) + if err != nil { + t.Fatalf("List: %v", err) + } + d := rows[0] + if d.EventID != "evt_01" || d.EventType != "account_locked" { + t.Errorf("recorded %q/%q, want evt_01/account_locked", d.EventID, d.EventType) + } + if d.UserID == "" || d.IP == "" { + t.Errorf("recorded user/ip %q/%q, want both present", d.UserID, d.IP) + } + if d.Attempts != 0 { + t.Errorf("attempts = %d before any delivery, want 0", d.Attempts) + } +} + +// The body is built once, at enqueue, and is what the receiver gets. That is +// what lets the delivery log answer "what did we send?" for a retry as well +// as a first attempt. +func TestSendWebhookRecordsTheBodyItWillSend(t *testing.T) { + store := NewMemoryStore() + sender := &Sender{Store: store} + if err := sender.SendWebhook(context.Background(), testEvent()); err != nil { + t.Fatalf("SendWebhook: %v", err) + } + + rows, _ := store.List(context.Background(), "", 10) + var got payload + if err := json.Unmarshal(rows[0].Payload, &got); err != nil { + t.Fatalf("the stored payload is not the JSON body: %v (%s)", err, rows[0].Payload) + } + + if got.ID != "evt_01" || got.Type != "account_locked" { + t.Errorf("payload id/type = %q/%q", got.ID, got.Type) + } + if got.Metadata["reason"] != "too many failed attempts" { + t.Errorf("payload metadata = %v, want the event's own", got.Metadata) + } + if !got.OccurredAt.Equal(testEvent().OccurredAt) { + t.Errorf("occurred_at = %v, want %v", got.OccurredAt, testEvent().OccurredAt) + } +} + +// ctx on the request path may already be cancelled by the time a slow sender +// gets to it — cryden's own doc comment says so. An insert that honoured +// that cancellation would drop the event exactly when the request went away, +// and this row is the only record it ever happened. +func TestSendWebhookIgnoresACancelledRequestContext(t *testing.T) { + store := NewMemoryStore() + sender := &Sender{Store: store} + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if err := sender.SendWebhook(ctx, testEvent()); err != nil { + t.Fatalf("SendWebhook with a cancelled ctx: %v", err) + } + if n := store.Count(StatusPending); n != 1 { + t.Errorf("pending rows = %d, want 1 — the event outlives the connection it arrived on", n) + } +} + +// The nudge is a hint, never a guarantee: it must not be able to block the +// request path, whatever state the channel is in. +func TestSendWebhookNeverBlocksOnTheWakeupChannel(t *testing.T) { + full := make(chan struct{}, 1) + full <- struct{}{} + + for name, wake := range map[string]chan<- struct{}{ + "already full": full, + "nil": nil, + "unbuffered": make(chan struct{}), + } { + t.Run(name, func(t *testing.T) { + sender := &Sender{Store: NewMemoryStore(), Wake: wake} + + done := make(chan error, 1) + go func() { done <- sender.SendWebhook(context.Background(), testEvent()) }() + + select { + case err := <-done: + if err != nil { + t.Fatalf("SendWebhook: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("SendWebhook blocked on the wakeup channel — that is a login hanging") + } + }) + } +} + +// cryden logs a send error and lets the login succeed regardless. So the +// error has to actually reach it rather than being swallowed here, or a +// delivery pipeline that is broken reports nothing at all. +func TestSendWebhookReportsAnEnqueueFailure(t *testing.T) { + wantErr := errors.New("database is down") + sender := &Sender{Store: failingStore{err: wantErr}} + + if err := sender.SendWebhook(context.Background(), testEvent()); !errors.Is(err, wantErr) { + t.Errorf("error = %v, want %v", err, wantErr) + } +} + +// An event with no user behind it — a failed login naming an email nobody +// registered — is a real case, and its empty fields must not become a +// problem for the store. +func TestSendWebhookAcceptsAnEventWithNoUser(t *testing.T) { + store := NewMemoryStore() + sender := &Sender{Store: store} + + event := testEvent() + event.UserID = "" + event.IP = "" + event.Metadata = nil + + if err := sender.SendWebhook(context.Background(), event); err != nil { + t.Fatalf("SendWebhook: %v", err) + } + rows, _ := store.List(context.Background(), "", 10) + if rows[0].UserID != "" { + t.Errorf("user id = %q, want empty", rows[0].UserID) + } + + // And the body omits the fields rather than sending empty strings, so a + // receiver can tell "no user" from "the empty user". + var body map[string]any + if err := json.Unmarshal(rows[0].Payload, &body); err != nil { + t.Fatalf("payload: %v", err) + } + if _, present := body["user_id"]; present { + t.Errorf("payload = %s, want user_id omitted when there is no user", rows[0].Payload) + } + if body["id"] != "evt_01" { + t.Errorf("payload id = %v, want the event id present even when other fields are empty", body["id"]) + } +} + +// failingStore is a Store whose writes always fail, for the error paths. +type failingStore struct{ err error } + +func (s failingStore) Enqueue(context.Context, Delivery) error { return s.err } +func (s failingStore) ClaimDue(context.Context, time.Time, int, int, time.Duration) ([]Delivery, error) { + return nil, s.err +} +func (s failingStore) MarkDelivered(context.Context, int64, Result) error { return s.err } +func (s failingStore) MarkFailed(context.Context, int64, Result, *time.Time) error { + return s.err +} +func (s failingStore) List(context.Context, Status, int) ([]Delivery, error) { return nil, s.err } diff --git a/webhook/store.go b/webhook/store.go new file mode 100644 index 0000000..ec1299e --- /dev/null +++ b/webhook/store.go @@ -0,0 +1,390 @@ +// Package webhook delivers engine events to a host endpoint and keeps a +// queryable record of every delivery. +// +// It exists because cryden deliberately ships no webhook implementation +// (see notify.WebhookSender's own doc comment): the engine surfaces the +// event and knows nothing about an endpoint, a signing scheme or a retry +// policy. Those are all this repo's, along with the table +// (migrations/010_webhook_deliveries.up.sql), which per CLAUDE.md's +// ownership rule is this repo's own infrastructure and not cryden's. +// +// The shape of this package follows from ONE property of how cryden calls +// it: SendWebhook runs SYNCHRONOUSLY on the request path, in the same +// goroutine as the login that triggered it. So: +// +// - Sender does an INSERT and nothing else. It makes no HTTP call, so a +// login pays one insert and returns. The database row is the queue; the +// channel it nudges is only a hint that there is work. +// - Worker makes the HTTP calls, out of band, with retries and backoff. +// - Store is what both of them talk to, and is an interface with a +// Postgres implementation and an in-memory double for the same reason +// cryden keeps store/postgres and store/memory apart: the worker's +// claim/backoff/terminal behaviour has to be testable without a +// database, and a double written against the same contract is the only +// honest way to do that. +package webhook + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "time" +) + +// Status is where a delivery has got to. The four values are exhaustive: +// every row is in exactly one of them at all times, which is what makes +// "was that lockout announced?" answerable from this table alone. +type Status string + +const ( + // StatusPending is waiting for a worker to pick it up. A row that has + // failed and is waiting out its backoff is also pending — the difference + // is next_attempt_at, not the status. + StatusPending Status = "pending" + + // StatusInFlight has been claimed by a worker that has not resolved it + // yet. A row left here past the staleness bound is one whose worker + // stopped mid-delivery, and is reclaimed rather than stranded. + StatusInFlight Status = "in_flight" + + // StatusDelivered got a 2xx. + StatusDelivered Status = "delivered" + + // StatusFailed is terminal: either the attempts ran out, or the row was + // abandoned by a process that died mid-delivery one time too many. + // Terminal and readable, deliberately — a delivery log that deletes its + // failures is a delivery log nobody can act on. + StatusFailed Status = "failed" +) + +// Statuses returns every status, in the order a console would offer them +// as filters. Fresh each call so a caller cannot edit the set. +func Statuses() []Status { + return []Status{StatusPending, StatusInFlight, StatusDelivered, StatusFailed} +} + +// ParseStatus validates a status a caller supplied — a query parameter, in +// practice — against the four that exist. Anything else is an error rather +// than being passed through to a query that would return an empty list and +// look like "no deliveries" instead of "no such status". +func ParseStatus(s string) (Status, error) { + for _, known := range Statuses() { + if s == string(known) { + return known, nil + } + } + return "", fmt.Errorf("%w: %q", ErrInvalidStatus, s) +} + +// The two ways a call here can be refused. +var ( + // ErrInvalidStatus means a status filter named something that is not a + // status. + ErrInvalidStatus = errors.New("webhook: unknown delivery status") + // ErrNotFound means there is no such delivery row. + ErrNotFound = errors.New("webhook: no such delivery") +) + +// Delivery is one row: an event this deployment chose to deliver, and +// everything known about the attempt(s) to deliver it. +type Delivery struct { + // ID is the row's identity and the queue's ordering. See the migration + // for why this is a surrogate key and not the engine's event id. + ID int64 + + // EventID is the engine's idempotency key for this occurrence, sent to + // the receiver as X-Cryden-Event-Id. It may be empty: cryden generates + // it with crypto/rand and, on a generator failure, deliberately + // delivers the event without one rather than not at all. + EventID string + + // EventType is the recorded store.AuditEventType as a string, so this + // package needs no import from the engine's internals. + EventType string + + // UserID and IP are empty where the event had neither. + UserID string + IP string + + // Payload is the event body exactly as it goes on the wire. The worker + // sends these bytes unchanged; see the migration for why reading a + // JSONB column back is byte-exact. + Payload json.RawMessage + + Status Status + + // Attempts counts attempts STARTED, incremented when a row is claimed + // rather than when it fails — a process that died mid-delivery still + // made the attempt, and at-least-once is the only promise a queue can + // make. It can therefore exceed the configured maximum after a crash; + // see the migration for why that is preferred to a stranded row. + Attempts int + + // ResponseCode is the receiver's HTTP status, or 0 where no response + // was received at all (a connection failure, a timeout, a DNS error). + // 0 is never a real code, so it is unambiguous as "nothing came back". + ResponseCode int + + // Error is the failure that was recorded, or empty. + Error string + + // DurationMS is how long the last attempt took. + DurationMS int + + CreatedAt time.Time + NextAttemptAt time.Time + ClaimedAt *time.Time + DeliveredAt *time.Time +} + +// Result is what a worker learned by making one delivery attempt. +type Result struct { + // Code is the receiver's HTTP status, 0 if no response arrived. + Code int + // Duration is how long the attempt took. + Duration time.Duration + // Err is the failure to record, empty on success. + Err string +} + +// Store is the persistence behind both halves of this package. Both +// implementations are expected to keep the same promises, which are not all +// obvious from the signatures: +// +// - Enqueue never overwrites or dedupes. Two events are two rows, even if +// the engine handed over the same event id twice. +// - ClaimDue hands a row to exactly one caller. It is the only place a +// row goes from pending to in_flight, and it must be safe with several +// workers running at once — the Postgres one is a single statement +// using FOR UPDATE SKIP LOCKED for exactly that reason. +// - ClaimDue has two branches: a pending row that is due and inside its +// attempt budget, and an in_flight row whose worker went away. The +// second ignores the budget on purpose, so a crash cannot strand a row +// that nothing will ever finish or report on. +// - ClaimDue increments Attempts. That is a write, so a caller must not +// expect to call it twice for the same claim. +type Store interface { + // Enqueue records a delivery as pending and immediately due. + Enqueue(ctx context.Context, d Delivery) error + + // ClaimDue claims up to limit rows that are ready to be attempted: + // pending and due and under maxAttempts, or in_flight and claimed + // before now-staleAfter. now is passed in rather than read from the + // database so the caller's clock is the only clock. + ClaimDue(ctx context.Context, now time.Time, limit, maxAttempts int, staleAfter time.Duration) ([]Delivery, error) + + // MarkDelivered records a 2xx and makes the row terminal. + MarkDelivered(ctx context.Context, id int64, r Result) error + + // MarkFailed records a failed attempt. A nil retryAt gives up and makes + // the row terminal; a non-nil one re-queues it for that time. The + // caller decides which, because whether the budget is spent is a + // property of the attempt it just made. + MarkFailed(ctx context.Context, id int64, r Result, retryAt *time.Time) error + + // List returns deliveries newest first. An empty status means all of + // them. + List(ctx context.Context, status Status, limit int) ([]Delivery, error) +} + +// PostgresStore is the real store, on the same *sql.DB every other store in +// this repo gets. +type PostgresStore struct { + db *sql.DB +} + +func NewStore(db *sql.DB) *PostgresStore { + return &PostgresStore{db: db} +} + +var _ Store = (*PostgresStore)(nil) + +// deliveryColumns is the select list and scan order, written once so the +// three reads below cannot drift apart from each other. +const deliveryColumns = ` + id, event_id, event_type, COALESCE(user_id::text, ''), COALESCE(ip, ''), + payload, status, attempts, COALESCE(response_code, 0), COALESCE(error, ''), + COALESCE(duration_ms, 0), created_at, next_attempt_at, claimed_at, delivered_at +` + +// scanDelivery reads one row in deliveryColumns order. The nullable columns +// are coalesced in SQL rather than handled with sql.Null* here: this repo +// has no use for the distinction between "NULL" and "empty" for any of +// them, and a Scan target that can be NULL crashes the read rather than +// defaulting. +func scanDelivery(row interface{ Scan(...any) error }) (Delivery, error) { + var d Delivery + err := row.Scan( + &d.ID, &d.EventID, &d.EventType, &d.UserID, &d.IP, + &d.Payload, &d.Status, &d.Attempts, &d.ResponseCode, &d.Error, + &d.DurationMS, &d.CreatedAt, &d.NextAttemptAt, &d.ClaimedAt, &d.DeliveredAt, + ) + return d, err +} + +func (s *PostgresStore) Enqueue(ctx context.Context, d Delivery) error { + // The payload is passed as a string, not a []byte, which is not + // cosmetic: lib/pq sends a []byte as bytea hex, which a JSONB column + // rejects, while a string arrives as text and Postgres parses it as the + // jsonb the parameter's target type says it is. Same trap documented in + // usermeta/store.go. + // + // user_id and ip are passed as nil rather than "" when empty, so the + // columns hold NULL and COALESCE above has something to coalesce. An + // empty user id is a real case: a failed login naming an email nobody + // registered has no user behind it. + _, err := s.db.ExecContext(ctx, ` + INSERT INTO webhook_deliveries (event_id, event_type, user_id, ip, payload) + VALUES ($1, $2, $3, $4, $5) + `, d.EventID, d.EventType, nullIfEmpty(d.UserID), nullIfEmpty(d.IP), string(d.Payload)) + return err +} + +// ClaimDue is one statement, which is the point: a read-then-write in Go +// would let two workers claim the same row between the SELECT and the +// UPDATE, and the receiver would get the event twice with only one of them +// recorded. FOR UPDATE SKIP LOCKED is what makes a second worker safe +// rather than merely unlikely to collide. +// +// The two branches are deliberately asymmetric — see the Store doc. +func (s *PostgresStore) ClaimDue(ctx context.Context, now time.Time, limit, maxAttempts int, staleAfter time.Duration) ([]Delivery, error) { + rows, err := s.db.QueryContext(ctx, ` + UPDATE webhook_deliveries + SET status = 'in_flight', claimed_at = $1, attempts = attempts + 1 + WHERE id IN ( + SELECT id FROM webhook_deliveries + WHERE (status = 'pending' AND attempts < $2 AND next_attempt_at <= $1) + OR (status = 'in_flight' AND claimed_at < $3) + ORDER BY next_attempt_at + LIMIT $4 + FOR UPDATE SKIP LOCKED + ) + RETURNING `+deliveryColumns, + now, maxAttempts, now.Add(-staleAfter), limit) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []Delivery + for rows.Next() { + d, err := scanDelivery(rows) + if err != nil { + return nil, err + } + out = append(out, d) + } + return out, rows.Err() +} + +func (s *PostgresStore) MarkDelivered(ctx context.Context, id int64, r Result) error { + // The recorded error is cleared: a row that failed twice and then + // succeeded is delivered, and leaving the last failure next to a + // delivered status reads as though something is still wrong. The + // attempts count and the response code tell the story instead. + res, err := s.db.ExecContext(ctx, ` + UPDATE webhook_deliveries + SET status = 'delivered', delivered_at = now(), claimed_at = NULL, + response_code = $2, duration_ms = $3, error = NULL + WHERE id = $1 + `, id, nullIfZero(r.Code), int(r.Duration.Milliseconds())) + if err != nil { + return err + } + return mustHaveHit(res, id) +} + +func (s *PostgresStore) MarkFailed(ctx context.Context, id int64, r Result, retryAt *time.Time) error { + // Two statements rather than one with a CASE: the branch is a decision + // the worker makes about the attempt it just made, and keeping it in Go + // keeps it readable and testable. Both are still single statements, so + // neither has a read-modify-write window. + query := ` + UPDATE webhook_deliveries + SET status = 'failed', claimed_at = NULL, + response_code = $2, duration_ms = $3, error = $4 + WHERE id = $1 + ` + args := []any{id, nullIfZero(r.Code), int(r.Duration.Milliseconds()), nullIfEmpty(r.Err)} + if retryAt != nil { + query = ` + UPDATE webhook_deliveries + SET status = 'pending', claimed_at = NULL, next_attempt_at = $5, + response_code = $2, duration_ms = $3, error = $4 + WHERE id = $1 + ` + args = append(args, *retryAt) + } + + res, err := s.db.ExecContext(ctx, query, args...) + if err != nil { + return err + } + return mustHaveHit(res, id) +} + +func (s *PostgresStore) List(ctx context.Context, status Status, limit int) ([]Delivery, error) { + query := `SELECT ` + deliveryColumns + ` FROM webhook_deliveries` + args := []any{limit} + if status != "" { + // Two statements rather than one with ($1 = '' OR status = $1), so + // the filtered form can use idx_webhook_deliveries_due instead of + // scanning the delivered history it is trying to exclude. + query += ` WHERE status = $1 ORDER BY created_at DESC, id DESC LIMIT $2` + args = []any{string(status), limit} + } else { + query += ` ORDER BY created_at DESC, id DESC LIMIT $1` + } + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]Delivery, 0, limit) + for rows.Next() { + d, err := scanDelivery(rows) + if err != nil { + return nil, err + } + out = append(out, d) + } + return out, rows.Err() +} + +// mustHaveHit turns "the UPDATE matched no row" into ErrNotFound. A worker +// marking a row that is not there is not a normal case — it means the row +// was deleted underneath it — and reporting it keeps that visible rather +// than letting the worker believe it resolved something. +func mustHaveHit(res sql.Result, id int64) error { + affected, err := res.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + return fmt.Errorf("%w: id %d", ErrNotFound, id) + } + return nil +} + +// nullIfEmpty maps an empty string to a SQL NULL, and anything else +// through unchanged. Returned as any so it can be passed straight as a +// parameter. +func nullIfEmpty(s string) any { + if s == "" { + return nil + } + return s +} + +// nullIfZero maps a zero to a SQL NULL. Used for response_code, where 0 +// means "no response arrived" and a stored 0 would read as a real code. +func nullIfZero(n int) any { + if n == 0 { + return nil + } + return n +} diff --git a/webhook/worker.go b/webhook/worker.go new file mode 100644 index 0000000..887659d --- /dev/null +++ b/webhook/worker.go @@ -0,0 +1,408 @@ +package webhook + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "net/http" + "strconv" + "strings" + "time" +) + +// Defaults for every knob. A zero field means "use this", so a Worker built +// by hand cannot end up with a budget of zero attempts or a poll interval +// that spins the CPU. +const ( + DefaultPollInterval = 15 * time.Second + DefaultBatchSize = 20 + DefaultMaxAttempts = 5 + + // DefaultStaleAfter is how long a claimed row may sit in_flight before + // another pass treats its worker as gone. It has to be comfortably + // longer than the HTTP timeout below, or a merely slow delivery would + // be claimed out from under the worker still making it — and the + // receiver would get the event twice. + DefaultStaleAfter = 5 * time.Minute + + // defaultHTTPTimeout bounds one delivery attempt. The worker is + // single-goroutine and serial, so this is also the worst case added to + // every other delivery's latency in the same batch. + defaultHTTPTimeout = 10 * time.Second + + // backoffBase and backoffMax bound the exponential retry schedule: + // 30s, 1m, 2m, 4m, 8m ... out to 30m. At the default five attempts that + // is a delivery resolved or given up on inside about eight minutes, + // which is short enough that an operator watching the console sees the + // outcome of a change they just made. + backoffBase = 30 * time.Second + backoffMax = 30 * time.Minute + + // maxResponseSnippet bounds how much of a failing receiver's response + // body is kept. The point is to keep "your endpoint said: database is + // down" in the log, not to archive a third party's error pages. + maxResponseSnippet = 512 + + // userAgent identifies this worker to a receiver's access log. + userAgent = "cryden-webhook/1.0" + + // SignatureHeader carries Sign's output. Exported knowledge rather than + // an internal detail: it is the header a receiver implements against. + SignatureHeader = "X-Cryden-Signature" +) + +// Worker makes the deliveries. One goroutine is enough by design — SKIP +// LOCKED in the Postgres claim means raising the count later is safe and +// needs no change here, and the default event set is deliberately +// low-volume (see cryden.DefaultWebhookEvents). +type Worker struct { + Store Store + + // URL is where deliveries are POSTed. Required: a Worker with no URL + // does nothing rather than guessing. + URL string + + // Secret is the HMAC key for the signature header. Empty means the + // deliveries go out UNSIGNED and no signature header is sent — not a + // header computed over an empty key, which a receiver might mistake for + // a real one. A trusted endpoint on a private network is a legitimate + // configuration; the worker says so once at startup. + Secret string + + // MaxAttempts is the total number of attempts a delivery gets, + // including the first. Zero means DefaultMaxAttempts. + MaxAttempts int + + // Client makes the calls. Zero means a client with a 10s timeout that + // refuses to follow a redirect to a different host. + Client *http.Client + + // Log receives operational lines — a failed attempt, a panic, a batch + // error. Nil means silent, which is what a test wants. + Log *log.Logger + + // Now is the clock, for the retry schedule and for claim cutoffs. Zero + // means time.Now. + Now func() time.Time + + // PollInterval is how often the worker looks for work even when nothing + // has nudged it — the safety net for a nudge lost to a full buffer or a + // row due in the future whose backoff has just elapsed. Zero means + // DefaultPollInterval. + PollInterval time.Duration + + // BatchSize is the most rows claimed per pass. Zero means + // DefaultBatchSize. + BatchSize int + + // StaleAfter is how long an in_flight row may sit before it is + // reclaimed. Zero means DefaultStaleAfter. + StaleAfter time.Duration + + // Wake is the channel Sender nudges. Nil is fine: the worker then runs + // on its poll interval alone. + Wake <-chan struct{} +} + +// NewWorker builds a worker with every default filled in, so no exported +// field on the result is ever a zero that means something other than zero. +// Only the two arguments without a sensible default are required; the rest +// are set by the caller afterwards. +func NewWorker(store Store, url, secret string) *Worker { + w := &Worker{Store: store, URL: url, Secret: secret} + w.applyDefaults() + return w +} + +// Sign computes the value of SignatureHeader for a body: HMAC-SHA256 over +// the raw bytes, hex-encoded, prefixed with the algorithm so a receiver can +// tell which scheme it is looking at without having to guess. +// +// Exported, along with Verify, so a receiver written in Go can use this +// rather than reimplementing it — and so a test can assert a real signature +// rather than a string it produced itself. +// +// A receiver in another language needs this much: HMAC-SHA256, key = the +// shared secret as UTF-8 bytes, message = the request body byte for byte, +// lowercase hex, "sha256=" in front. +func Sign(secret string, body []byte) string { + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write(body) + return "sha256=" + hex.EncodeToString(mac.Sum(nil)) +} + +// Verify reports whether header is a valid signature for body under secret. +// Comparison is constant-time, so a receiver using this does not leak the +// expected digest through timing. +func Verify(secret string, body []byte, header string) bool { + return hmac.Equal([]byte(Sign(secret, body)), []byte(header)) +} + +// Run delivers until ctx is cancelled. It is meant to be started once, in +// its own goroutine; main.go passes context.Background() because this repo +// has no graceful shutdown anywhere yet (see PROGRESS.md — introducing one +// touches every component and is its own change, not a passenger on this +// one). +func (w *Worker) Run(ctx context.Context) { + w.applyDefaults() + if w.Secret == "" { + w.logf("webhook worker: WEBHOOK_SECRET is not set — deliveries are unsigned") + } + + ticker := time.NewTicker(w.PollInterval) + defer ticker.Stop() + + for { + // Drain before sleeping: a batch that came back full almost + // certainly means there is more waiting, and waiting a whole poll + // interval to find that out would put a backlog behind the + // interval for no reason. + for { + n, err := w.RunOnce(ctx) + if err != nil { + // A database error is not retried in a tight loop — the + // poll interval is the backoff, which keeps a broken + // connection from becoming a busy loop. + if ctx.Err() == nil { + w.logf("webhook worker: %v", err) + } + break + } + if n < w.BatchSize || ctx.Err() != nil { + break + } + } + + select { + case <-ctx.Done(): + return + case <-w.Wake: + case <-ticker.C: + } + } +} + +// RunOnce claims and delivers one batch, returning how many rows it claimed. +// It is the whole of the worker's behaviour without the loop, so tests drive +// it directly and deterministically instead of racing a goroutine. +// +// A cancelled ctx can leave claimed rows in_flight and un-attempted; the +// stale reclaim picks them up. That is why Attempts counts attempts +// STARTED rather than attempts that reached an endpoint. +func (w *Worker) RunOnce(ctx context.Context) (int, error) { + w.applyDefaults() + + claimed, err := w.Store.ClaimDue(ctx, w.Now(), w.BatchSize, w.MaxAttempts, w.StaleAfter) + if err != nil { + return 0, err + } + + for _, d := range claimed { + if ctx.Err() != nil { + break + } + w.deliver(ctx, d) + } + return len(claimed), nil +} + +// deliver makes one attempt and resolves the row. +func (w *Worker) deliver(ctx context.Context, d Delivery) { + // A panic here would otherwise take the process with it. That is worth + // diverging from the engine's own "a panicking sender should fail + // loudly" advice for: cryden's reasoning is about the REQUEST path, + // where failing loudly is cheap and is seen on the first request. This + // runs in a background goroutine, so the same bug would turn a webhook + // problem into "nobody can log in" — a far worse outcome than a + // retried delivery and a loud log line. + defer func() { + if r := recover(); r != nil { + w.logf("webhook worker: panic delivering %d (%s): %v", d.ID, d.EventType, r) + w.resolve(ctx, d, Result{Err: fmt.Sprintf("panic while delivering: %v", r)}) + } + }() + + w.resolve(ctx, d, w.post(ctx, d)) +} + +// resolve records the outcome of an attempt: delivered, retried, or given +// up on. +// +// Note the ordering in the caller: the attempt is always made first, and +// this only decides what to do with the result. So a row reclaimed from a +// crashed worker — whose Attempts is already at the budget — gets one more +// real attempt rather than being written off unvisited. That is deliberate, +// and it is bounded: ClaimDue only ignores the budget on its stale branch, +// and this method then resolves the row either way, so a crash costs at most +// one attempt beyond MaxAttempts and can never become a loop. Delivering an +// event the receiver may never have got is worth more than a row that says +// "failed" without anyone having tried. +func (w *Worker) resolve(ctx context.Context, d Delivery, result Result) { + if result.Code >= 200 && result.Code < 300 { + if err := w.Store.MarkDelivered(ctx, d.ID, result); err != nil { + w.logf("webhook worker: recording delivery %d as delivered: %v", d.ID, err) + } + return + } + + if result.Err == "" { + result.Err = fmt.Sprintf("receiver answered %d", result.Code) + } + + // d.Attempts is the count ClaimDue returned, which is already + // incremented for the attempt just made — so this is "attempt 5 of 5" + // on the fifth. Comparing before scheduling the retry is what makes + // MaxAttempts mean exactly that, rather than one attempt more. + if d.Attempts >= w.MaxAttempts { + w.logf("webhook worker: giving up on delivery %d (%s) after %d attempts: %s", + d.ID, d.EventType, d.Attempts, result.Err) + if err := w.Store.MarkFailed(ctx, d.ID, result, nil); err != nil { + w.logf("webhook worker: recording delivery %d as failed: %v", d.ID, err) + } + return + } + + retryAt := w.Now().Add(backoff(d.Attempts)) + if err := w.Store.MarkFailed(ctx, d.ID, result, &retryAt); err != nil { + w.logf("webhook worker: rescheduling delivery %d: %v", d.ID, err) + } +} + +// post makes the HTTP call and reports what happened. It never returns an +// error: a delivery failure is data to be recorded, not a Go error to be +// propagated, and every path here ends in a Result. +func (w *Worker) post(ctx context.Context, d Delivery) Result { + start := w.Now() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, w.URL, bytes.NewReader(d.Payload)) + if err != nil { + // A malformed WEBHOOK_URL. Retried rather than terminal, because + // the row would otherwise be lost to a typo an operator is about + // to fix — and the error is in the log either way. + return Result{Duration: w.Now().Sub(start), Err: err.Error()} + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", userAgent) + req.Header.Set("X-Cryden-Event-Type", d.EventType) + + // The headers below are informational and deliberately NOT covered by + // the signature, which is over the body alone. A receiver must not make + // a decision on them; it can use the attempt number for its own logs. + if d.EventID != "" { + req.Header.Set("X-Cryden-Event-Id", d.EventID) + } + req.Header.Set("X-Cryden-Delivery-Attempt", strconv.Itoa(d.Attempts)) + + if w.Secret != "" { + req.Header.Set(SignatureHeader, Sign(w.Secret, d.Payload)) + } + + resp, err := w.Client.Do(req) + if err != nil { + return Result{Duration: w.Now().Sub(start), Err: err.Error()} + } + defer resp.Body.Close() + + // Read a bounded snippet, which both keeps the connection reusable for + // the next delivery and gives a failing receiver's own words a place in + // the log — "your endpoint said: database is down" is the difference + // between a delivery log an operator can act on and a row that only + // says 500. + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, maxResponseSnippet)) + result := Result{Code: resp.StatusCode, Duration: w.Now().Sub(start)} + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + result.Err = fmt.Sprintf("receiver answered %d: %s", resp.StatusCode, snippetText(snippet)) + } + return result +} + +// snippetText renders a response body for the log: whitespace collapsed so +// it stays one line, and bounded again because a body full of no-break +// spaces would otherwise be a small wall of text in a console. +func snippetText(raw []byte) string { + text := strings.Join(strings.Fields(string(raw)), " ") + if len(text) > 200 { + text = text[:200] + "…" + } + if text == "" { + return "(no body)" + } + return text +} + +// backoff is the retry delay after attempt n (1-based). Doubling with a +// cap, no jitter: one worker claims rows in batches, so a set of deliveries +// failing together is already spread across passes rather than hammering a +// receiver in the same instant. Running several workers would want jitter +// here — noted rather than pre-built. +func backoff(attempt int) time.Duration { + if attempt < 1 { + attempt = 1 + } + // Capped before shifting so a large value shifts into a negative + // duration rather than being caught by the comparison below. + if attempt > 20 { + return backoffMax + } + d := backoffBase << (attempt - 1) + if d <= 0 || d > backoffMax { + return backoffMax + } + return d +} + +func (w *Worker) applyDefaults() { + if w.MaxAttempts <= 0 { + w.MaxAttempts = DefaultMaxAttempts + } + if w.Now == nil { + w.Now = func() time.Time { return time.Now().UTC() } + } + if w.PollInterval <= 0 { + w.PollInterval = DefaultPollInterval + } + if w.BatchSize <= 0 { + w.BatchSize = DefaultBatchSize + } + if w.StaleAfter <= 0 { + w.StaleAfter = DefaultStaleAfter + } + if w.Client == nil { + w.Client = defaultClient() + } +} + +// defaultClient bounds one attempt and refuses a redirect to another host. +// +// The timeout is the obvious half. The redirect rule is the less obvious +// one: the configured URL is where the operator wants their events to go, +// and following a redirect to a different host would deliver the body, and +// the signature over it, wherever that host said. A same-host redirect — +// http to https, a missing trailing slash — still works, which is the +// common legitimate case. +func defaultClient() *http.Client { + return &http.Client{ + Timeout: defaultHTTPTimeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + if req.URL.Host != via[0].URL.Host { + return fmt.Errorf("refusing to follow a redirect from %s to %s", via[0].URL.Host, req.URL.Host) + } + return nil + }, + } +} + +func (w *Worker) logf(format string, args ...any) { + if w.Log != nil { + w.Log.Printf(format, args...) + } +} diff --git a/webhook/worker_test.go b/webhook/worker_test.go new file mode 100644 index 0000000..91269a2 --- /dev/null +++ b/webhook/worker_test.go @@ -0,0 +1,568 @@ +package webhook + +import ( + "context" + "errors" + "net/http" + "net/url" + "strings" + "testing" + "time" +) + +// newTestWorker wires a worker to an in-memory store and a clock a test +// controls, pointed at a real HTTP endpoint. +func newTestWorker(t *testing.T, store Store, url, secret string) (*Worker, *testClock) { + t.Helper() + clock := newTestClock() + store.(*MemoryStore).Clock = clock.now + + w := NewWorker(store, url, secret) + w.Now = clock.now + w.BatchSize = 10 + return w, clock +} + +// enqueue is the sender's job, done directly so a worker test is about the +// worker. +func enqueue(t *testing.T, store Store, eventType string) int64 { + t.Helper() + sender := &Sender{Store: store} + event := testEvent() + event.Type = eventType + if err := sender.SendWebhook(context.Background(), event); err != nil { + t.Fatalf("enqueue: %v", err) + } + rows, err := store.List(context.Background(), "", 10) + if err != nil { + t.Fatalf("List: %v", err) + } + return rows[0].ID +} + +func TestWorkerDeliversAndRecordsTheAttempt(t *testing.T) { + rec, url := newReceiver(t, http.StatusOK, "") + store := NewMemoryStore() + w, _ := newTestWorker(t, store, url, "") + + n, err := w.RunOnce(context.Background()) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if n != 0 { + t.Errorf("claimed %d rows from an empty store, want 0", n) + } + + id := enqueue(t, store, "account_locked") + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + + if rec.count() != 1 { + t.Fatalf("the endpoint was called %d times, want 1", rec.count()) + } + d, _ := store.Get(id) + if d.Status != StatusDelivered { + t.Errorf("status = %q, want delivered (%s)", d.Status, d.Error) + } + if d.ResponseCode != http.StatusOK { + t.Errorf("response code = %d, want 200", d.ResponseCode) + } + if d.Attempts != 1 { + t.Errorf("attempts = %d, want 1", d.Attempts) + } + if d.DeliveredAt == nil { + t.Error("delivered_at is not set on a delivered row") + } + if d.ClaimedAt != nil { + t.Error("claimed_at is still set after the row was resolved") + } + if d.Error != "" { + t.Errorf("error = %q, want empty on a delivered row", d.Error) + } +} + +// A receiver that keeps failing gets exactly MaxAttempts attempts and then +// stops. Retrying forever is a load generator pointed at a third party; +// giving up silently is half a feature. The row stays readable either way. +func TestWorkerRetriesThenGivesUpAtTheAttemptLimit(t *testing.T) { + rec, url := newReceiver(t, http.StatusInternalServerError, "upstream exploded") + store := NewMemoryStore() + w, clock := newTestWorker(t, store, url, "") + w.MaxAttempts = 3 + + id := enqueue(t, store, "account_locked") + + for attempt := 1; attempt <= w.MaxAttempts; attempt++ { + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce (attempt %d): %v", attempt, err) + } + d, _ := store.Get(id) + if d.Attempts != attempt { + t.Fatalf("after attempt %d, attempts = %d", attempt, d.Attempts) + } + + if attempt < w.MaxAttempts { + if d.Status != StatusPending { + t.Fatalf("after attempt %d, status = %q, want pending for a retry", attempt, d.Status) + } + // Nothing is due yet, so a pass made now must find no work. + if n, _ := w.RunOnce(context.Background()); n != 0 { + t.Fatalf("attempt %d was retried immediately, without waiting out its backoff", attempt) + } + clock.advance(backoff(attempt) + time.Second) + } + } + + d, _ := store.Get(id) + if d.Status != StatusFailed { + t.Errorf("status = %q, want failed once the attempts ran out", d.Status) + } + if d.Attempts != w.MaxAttempts { + t.Errorf("attempts = %d, want exactly the budget of %d", d.Attempts, w.MaxAttempts) + } + if rec.count() != w.MaxAttempts { + t.Errorf("the endpoint was called %d times, want %d", rec.count(), w.MaxAttempts) + } + // The receiver's own words are what makes the row actionable. + if !strings.Contains(d.Error, "500") || !strings.Contains(d.Error, "upstream exploded") { + t.Errorf("error = %q, want the status and the receiver's message", d.Error) + } + + // And a failed row is not picked up again. + clock.advance(24 * time.Hour) + if n, _ := w.RunOnce(context.Background()); n != 0 { + t.Errorf("claimed %d rows, want 0 — a failed delivery is terminal", n) + } +} + +// Doubling, capped, and never negative. +func TestBackoffDoublesAndIsBounded(t *testing.T) { + if got := backoff(1); got != backoffBase { + t.Errorf("backoff(1) = %v, want %v", got, backoffBase) + } + for attempt := 2; attempt <= 6; attempt++ { + if got, want := backoff(attempt), backoffBase<<(attempt-1); got != want { + t.Errorf("backoff(%d) = %v, want %v", attempt, got, want) + } + } + // Well past the cap, including values where an uncapped shift would + // overflow into a negative duration — which would schedule a retry in + // the past and spin. + for _, attempt := range []int{0, -1, 7, 20, 64, 1 << 20} { + if got := backoff(attempt); got <= 0 || got > backoffMax { + t.Errorf("backoff(%d) = %v, want a positive duration no greater than %v", attempt, got, backoffMax) + } + } +} + +// The signature is the receiver's whole basis for believing a delivery came +// from here, so the test verifies it with Verify — the function a receiver +// would use — rather than comparing against a string this test built. +func TestWorkerSignsTheBodyWhenASecretIsSet(t *testing.T) { + rec, url := newReceiver(t, http.StatusOK, "") + store := NewMemoryStore() + w, _ := newTestWorker(t, store, url, "s3cret-shared-with-the-receiver") + + id := enqueue(t, store, "account_locked") + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + + got := rec.last(t) + signature := got.header.Get(SignatureHeader) + if signature == "" { + t.Fatalf("no %s header on a signed delivery", SignatureHeader) + } + if !Verify("s3cret-shared-with-the-receiver", got.body, signature) { + t.Errorf("signature %q does not verify against the body sent", signature) + } + // A different secret must not verify, or the test above would pass for + // the wrong reason. + if Verify("some-other-secret", got.body, signature) { + t.Error("the signature verified under a secret it was not computed with") + } + + // The body is byte-for-byte what the log recorded, which is what makes + // "show me what we signed" answerable. + d, _ := store.Get(id) + if string(d.Payload) != string(got.body) { + t.Errorf("the endpoint received %s but the log records %s", got.body, d.Payload) + } + if got.header.Get("X-Cryden-Event-Type") != "account_locked" { + t.Errorf("event type header = %q", got.header.Get("X-Cryden-Event-Type")) + } + if got.header.Get("X-Cryden-Event-Id") != "evt_01" { + t.Errorf("event id header = %q", got.header.Get("X-Cryden-Event-Id")) + } + if got.header.Get("Content-Type") != "application/json" { + t.Errorf("content type = %q", got.header.Get("Content-Type")) + } +} + +// No secret means no signature header at all — not one computed over an +// empty key, which a receiver might accept as a real signature. +func TestWorkerSendsNoSignatureWithoutASecret(t *testing.T) { + rec, url := newReceiver(t, http.StatusOK, "") + store := NewMemoryStore() + w, _ := newTestWorker(t, store, url, "") + + enqueue(t, store, "account_locked") + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + + if got := rec.last(t).header.Get(SignatureHeader); got != "" { + t.Errorf("%s = %q on an unsigned worker, want the header absent", SignatureHeader, got) + } +} + +// Anything outside 2xx is a failure, including the redirects and 4xx a +// receiver might answer with. 204 is a success — the body is the request's, +// not the response's. +func TestWorkerTreatsOnlyTwoHundredsAsDelivered(t *testing.T) { + for _, tc := range []struct { + status int + want Status + }{ + {http.StatusOK, StatusDelivered}, + {http.StatusCreated, StatusDelivered}, + {http.StatusNoContent, StatusDelivered}, + {http.StatusMovedPermanently, StatusPending}, + {http.StatusBadRequest, StatusPending}, + {http.StatusNotFound, StatusPending}, + {http.StatusTooManyRequests, StatusPending}, + {http.StatusInternalServerError, StatusPending}, + } { + t.Run(http.StatusText(tc.status), func(t *testing.T) { + _, url := newReceiver(t, tc.status, "") + store := NewMemoryStore() + w, _ := newTestWorker(t, store, url, "") + w.MaxAttempts = 5 + + id := enqueue(t, store, "account_locked") + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + if d, _ := store.Get(id); d.Status != tc.want { + t.Errorf("status after %d = %q, want %q", tc.status, d.Status, tc.want) + } + }) + } +} + +// A connection that never completes is a failure with no response code at +// all — 0, which is never a real HTTP status, so "nothing came back" is +// distinguishable from "it said 500". +func TestWorkerRecordsAnUnreachableEndpointAsNoResponse(t *testing.T) { + store := NewMemoryStore() + w, _ := newTestWorker(t, store, "http://127.0.0.1:1/hooks", "") + w.Client = &http.Client{Timeout: 2 * time.Second} + + id := enqueue(t, store, "account_locked") + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + + d, _ := store.Get(id) + if d.Status != StatusPending { + t.Errorf("status = %q, want pending for a retry", d.Status) + } + if d.ResponseCode != 0 { + t.Errorf("response code = %d, want 0 for a connection that never answered", d.ResponseCode) + } + if d.Error == "" { + t.Error("no error was recorded for an unreachable endpoint") + } +} + +// A row claimed by a worker that then went away is retried rather than +// stranded. This is the branch of ClaimDue that ignores the attempt budget, +// and without it a crash mid-delivery would leave a row in_flight forever +// with nothing to finish it. +func TestWorkerReclaimsAStaleInFlightRow(t *testing.T) { + rec, url := newReceiver(t, http.StatusOK, "") + store := NewMemoryStore() + w, clock := newTestWorker(t, store, url, "") + w.MaxAttempts = 5 + + id := enqueue(t, store, "account_locked") + + // A worker claims the row and dies before resolving it. + claimed, err := store.ClaimDue(context.Background(), clock.now(), 10, w.MaxAttempts, w.StaleAfter) + if err != nil { + t.Fatalf("ClaimDue: %v", err) + } + if len(claimed) != 1 { + t.Fatalf("claimed %d rows, want 1", len(claimed)) + } + if d, _ := store.Get(id); d.Status != StatusInFlight { + t.Fatalf("status = %q, want in_flight", d.Status) + } + + // Not yet stale: nothing may claim it, because the first worker might + // still be making the call. + if n, _ := w.RunOnce(context.Background()); n != 0 { + t.Fatal("a freshly claimed row was claimed again — the receiver would see the event twice") + } + + // Past the staleness bound, and the replacement worker delivers it. + clock.advance(w.StaleAfter + time.Second) + if n, _ := w.RunOnce(context.Background()); n != 1 { + t.Fatalf("claimed %d rows after the claim went stale, want 1", n) + } + if d, _ := store.Get(id); d.Status != StatusDelivered { + t.Errorf("status = %q, want delivered", d.Status) + } + if rec.count() != 1 { + t.Errorf("the endpoint was called %d times, want 1", rec.count()) + } +} + +// The other half of the same property: a row that goes stale AFTER its +// attempts have run out must still reach a terminal state, rather than being +// reclaimed forever by a worker that keeps dying. It gets one more real +// attempt — ClaimDue's stale branch ignores the budget on purpose — and that +// attempt is what decides it: delivered if it works, failed if it does not. +// Either way it stops there, never rescheduled. +func TestWorkerResolvesAStaleRowThatIsOutOfAttempts(t *testing.T) { + for _, tc := range []struct { + name string + status int + wantStatus Status + }{ + {"the last attempt succeeds", http.StatusOK, StatusDelivered}, + {"the last attempt fails too", http.StatusInternalServerError, StatusFailed}, + } { + t.Run(tc.name, func(t *testing.T) { + rec, url := newReceiver(t, tc.status, "") + store := NewMemoryStore() + w, clock := newTestWorker(t, store, url, "") + w.MaxAttempts = 2 + + id := enqueue(t, store, "account_locked") + + // Two claims, neither resolved: a worker that died + // mid-delivery twice, which spends the whole budget. + for i := 0; i < w.MaxAttempts; i++ { + clock.advance(w.StaleAfter + time.Second) + if _, err := store.ClaimDue(context.Background(), clock.now(), 10, w.MaxAttempts, w.StaleAfter); err != nil { + t.Fatalf("ClaimDue: %v", err) + } + } + if d, _ := store.Get(id); d.Attempts != w.MaxAttempts { + t.Fatalf("attempts = %d, want the budget spent", d.Attempts) + } + + // Freshly claimed, so not yet reclaimable. + if n, _ := w.RunOnce(context.Background()); n != 0 { + t.Fatal("a freshly claimed row was claimed again") + } + + clock.advance(w.StaleAfter + time.Second) + if n, _ := w.RunOnce(context.Background()); n != 1 { + t.Fatalf("claimed %d stale rows, want 1", n) + } + + d, _ := store.Get(id) + if d.Status != tc.wantStatus { + t.Errorf("status = %q, want %q", d.Status, tc.wantStatus) + } + if rec.count() != 1 { + t.Errorf("the endpoint was called %d times, want the one last attempt", rec.count()) + } + // The bound that makes this safe: exactly one attempt past the + // budget, never a scheduled retry, so no crash can turn into a + // loop. + if d.Attempts != w.MaxAttempts+1 { + t.Errorf("attempts = %d, want exactly one past the budget of %d", d.Attempts, w.MaxAttempts) + } + if d.Status == StatusPending { + t.Error("a stranded row was rescheduled for another retry") + } + }) + } +} + +// BatchSize bounds one pass, so a backlog is worked through a batch at a +// time rather than claimed in one go. +func TestClaimDueHonoursTheBatchSize(t *testing.T) { + store := NewMemoryStore() + clock := newTestClock() + store.Clock = clock.now + + for i := 0; i < 5; i++ { + enqueue(t, store, "account_locked") + } + + first, err := store.ClaimDue(context.Background(), clock.now(), 2, 5, time.Minute) + if err != nil { + t.Fatalf("ClaimDue: %v", err) + } + if len(first) != 2 { + t.Fatalf("claimed %d rows with a batch size of 2, want 2", len(first)) + } + // Claimed rows are in_flight and not yet stale, so the next pass takes + // the NEXT two rather than the same ones again. + second, _ := store.ClaimDue(context.Background(), clock.now(), 2, 5, time.Minute) + if len(second) != 2 { + t.Fatalf("second pass claimed %d rows, want 2", len(second)) + } + seen := map[int64]bool{} + for _, d := range append(first, second...) { + if seen[d.ID] { + t.Fatalf("delivery %d was claimed twice without being resolved", d.ID) + } + seen[d.ID] = true + } + if n := store.Count(StatusInFlight); n != 4 { + t.Errorf("in-flight rows = %d, want 4", n) + } +} + +// A panic in the delivery path must not take the auth API down with it. It +// is recorded as a failure and retried like any other. +func TestWorkerContainsAPanicInTheDeliveryPath(t *testing.T) { + store := NewMemoryStore() + w, _ := newTestWorker(t, store, "http://example.invalid/hooks", "") + w.Client = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + panic("the http transport blew up") + })} + + id := enqueue(t, store, "account_locked") + n, err := w.RunOnce(context.Background()) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if n != 1 { + t.Fatalf("claimed %d rows, want 1", n) + } + + d, _ := store.Get(id) + if d.Status != StatusPending { + t.Errorf("status = %q, want pending so the delivery is retried", d.Status) + } + if !strings.Contains(d.Error, "panic") { + t.Errorf("error = %q, want the panic recorded", d.Error) + } +} + +// The configured URL is where the operator wants their events to go. +// Following a redirect to a different host would deliver the body, and the +// signature over it, wherever that host said. +func TestWorkerRefusesARedirectToAnotherHost(t *testing.T) { + check := defaultClient().CheckRedirect + + original := &http.Request{URL: mustURL(t, "https://hooks.example.com/v1")} + sameHost := &http.Request{URL: mustURL(t, "https://hooks.example.com/v1/")} + otherHost := &http.Request{URL: mustURL(t, "https://attacker.example.net/collect")} + + if err := check(sameHost, []*http.Request{original}); err != nil { + t.Errorf("a same-host redirect was refused: %v", err) + } + if err := check(otherHost, []*http.Request{original}); err == nil { + t.Error("a cross-host redirect was followed") + } +} + +// A malformed WEBHOOK_URL is retried rather than lost: the row would +// otherwise be thrown away for a typo an operator is about to fix. +func TestWorkerRetriesAMalformedURL(t *testing.T) { + store := NewMemoryStore() + w, _ := newTestWorker(t, store, "://not-a-url", "") + + id := enqueue(t, store, "account_locked") + if _, err := w.RunOnce(context.Background()); err != nil { + t.Fatalf("RunOnce: %v", err) + } + if d, _ := store.Get(id); d.Status != StatusPending { + t.Errorf("status = %q, want pending", d.Status) + } +} + +// List is what the admin endpoint is built on: newest first, filtered, and +// bounded. +func TestMemoryStoreListOrdersAndFilters(t *testing.T) { + store := NewMemoryStore() + clock := newTestClock() + store.Clock = clock.now + + for _, eventType := range []string{"first", "second", "third"} { + enqueue(t, store, eventType) + clock.advance(time.Minute) + } + + all, err := store.List(context.Background(), "", 10) + if err != nil { + t.Fatalf("List: %v", err) + } + if len(all) != 3 { + t.Fatalf("listed %d rows, want 3", len(all)) + } + if all[0].EventType != "third" { + t.Errorf("first row = %q, want the newest", all[0].EventType) + } + if all[2].EventType != "first" { + t.Errorf("last row = %q, want the oldest", all[2].EventType) + } + + limited, _ := store.List(context.Background(), "", 2) + if len(limited) != 2 { + t.Errorf("listed %d rows with limit 2", len(limited)) + } + if limited[0].EventType != "third" || limited[1].EventType != "second" { + t.Errorf("limited rows = %q, %q, want the two newest", limited[0].EventType, limited[1].EventType) + } + + if pending, _ := store.List(context.Background(), StatusPending, 10); len(pending) != 3 { + t.Errorf("pending rows = %d, want 3", len(pending)) + } + if delivered, _ := store.List(context.Background(), StatusDelivered, 10); len(delivered) != 0 { + t.Errorf("delivered rows = %d, want 0", len(delivered)) + } +} + +func TestParseStatusRejectsAnythingElse(t *testing.T) { + if got, err := ParseStatus("failed"); err != nil || got != StatusFailed { + t.Errorf("ParseStatus(failed) = %q, %v", got, err) + } + for _, bad := range []string{"", "done", "PENDING", "pending "} { + if _, err := ParseStatus(bad); !errors.Is(err, ErrInvalidStatus) { + t.Errorf("ParseStatus(%q) error = %v, want ErrInvalidStatus", bad, err) + } + } +} + +// A returned row is a copy, so a caller cannot resolve a delivery by editing +// a struct — the same thing a scanned row cannot do to a database. +func TestMemoryStoreHandsOutCopies(t *testing.T) { + store := NewMemoryStore() + id := enqueue(t, store, "account_locked") + + rows, _ := store.List(context.Background(), "", 10) + rows[0].Status = StatusDelivered + rows[0].Payload[0] = 'X' + + got, _ := store.Get(id) + if got.Status != StatusPending { + t.Errorf("status = %q after a caller edited its copy, want pending", got.Status) + } + if got.Payload[0] == 'X' { + t.Error("a caller's edit to its payload buffer reached the stored row") + } +} + +// roundTripFunc is an http.RoundTripper that is a plain function, so a test +// can make the HTTP layer do something a real server cannot. +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func mustURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("parsing %q: %v", raw, err) + } + return u +} From bd2c9909580dd4868c5f43f590ea04975e3531d5 Mon Sep 17 00:00:00 2001 From: Raymond Nicholas Date: Tue, 15 Sep 2026 13:21:10 +0100 Subject: [PATCH 08/10] feat: add cloud logging shipped-events log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The redacted, filtered copy the cloud sink is handed is recorded in this repo's own table — the stand-in for a hosted aggregator, which this repo has no client for. GET /v1/admin/logging/recent (admin) reads it back. Co-Authored-By: Claude Code --- .env.example | 6 + httpapi/logging_handlers.go | 122 +++++++++ httpapi/logging_handlers_test.go | 297 +++++++++++++++++++++ httpapi/router.go | 16 ++ main.go | 44 +++ migrations/011_shipped_log_events.down.sql | 5 + migrations/011_shipped_log_events.up.sql | 63 +++++ shiplog/logger.go | 125 +++++++++ shiplog/logger_test.go | 152 +++++++++++ shiplog/memory.go | 121 +++++++++ shiplog/store.go | 270 +++++++++++++++++++ shiplog/store_test.go | 211 +++++++++++++++ 12 files changed, 1432 insertions(+) create mode 100644 httpapi/logging_handlers.go create mode 100644 httpapi/logging_handlers_test.go create mode 100644 migrations/011_shipped_log_events.down.sql create mode 100644 migrations/011_shipped_log_events.up.sql create mode 100644 shiplog/logger.go create mode 100644 shiplog/logger_test.go create mode 100644 shiplog/memory.go create mode 100644 shiplog/store.go create mode 100644 shiplog/store_test.go diff --git a/.env.example b/.env.example index 1c93e7b..1f4ec72 100644 --- a/.env.example +++ b/.env.example @@ -100,6 +100,12 @@ API_KEY_PREFIX= # typo to debug multiplies a vendor bill and defaulting it to error # throws away the records you were trying to keep. # +# There is no vendor here — this repo ships no SDK, so the shipped copy +# is recorded in the shipped_log_events table, which +# GET /v1/admin/logging/recent (admin) reads back. It is the same bytes a +# hosted aggregator would have received, which is what makes it a stand-in +# for one rather than a second, different log beside it. +# # CLOUD_LOG_REDACTION is "mask" (value replaced with [redacted]) or # "hash" (keyed HMAC digest, so the same address still reads as the same # address across records — "one IP, forty accounts" is the shape diff --git a/httpapi/logging_handlers.go b/httpapi/logging_handlers.go new file mode 100644 index 0000000..ff8ecde --- /dev/null +++ b/httpapi/logging_handlers.go @@ -0,0 +1,122 @@ +package httpapi + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/crydensync/cryden/v2/logger" + + "github.com/crydensync/api/shiplog" +) + +// LoggingHandlers answers the admin view of the shipped-events log. +// +// Read-only, like every other endpoint on this surface (CLAUDE.md's hard +// rule): it lists records and can take no action. There is deliberately no +// endpoint that writes a record, edits LOG_LEVEL, or clears the log — +// changing what gets shipped is configuration, and configuration changes +// go through the same explicit, human-confirmed settings path every other +// one does. +type LoggingHandlers struct { + // Store is this repo's own shipped-events table. Nil when + // CLOUD_LOGGING is unset, which is a wiring fact rather than a server + // fault — the same 404 not_configured shape the OAuth health and + // hash-migration reports use for the same reason. + Store shiplog.Store +} + +// logEventDTO is one record. It is the redacted, filtered copy the cloud +// sink was handed, not the raw record on stdout — the two differ by +// design, and an operator reading this response is reading what would +// have left the building. +type logEventDTO struct { + ID int64 `json:"id"` + Level string `json:"level"` + Message string `json:"message"` + Fields map[string]string `json:"fields,omitempty"` + Sink string `json:"sink"` + + // ShippedAt is when the sink wrote the record, which for a + // synchronous sink is when the engine logged it. Nano-precision on + // the wire, because one login emits many records and at second + // precision they would arrive with identical timestamps and nothing + // downstream could order them. + ShippedAt time.Time `json:"shipped_at"` +} + +// logEventsDTO is the whole response. Levels is included so a console can +// offer the filter without hardcoding the four names, and Level echoes the +// filter in force — an operator looking at a short list needs to know +// whether it is short because of the filter or because the engine has been +// quiet. +type logEventsDTO struct { + Events []logEventDTO `json:"events"` + Count int `json:"count"` + + // Level is the filter that was asked for, absent when none was. It is + // the requested word rather than the resolved minimum: "warn" is what + // an operator typed, and echoing "debug" back for an unfiltered + // listing would suggest a filter that is not in force. + Level string `json:"level,omitempty"` + Levels []string `json:"levels"` +} + +// Recent — admin required (see router.go). Lists the shipped-events log +// newest first, optionally restricted to a level and above. +func (h *LoggingHandlers) Recent(w http.ResponseWriter, r *http.Request) { + if h.Store == nil { + writeErr(w, errAdminStoresUnavailable) + return + } + + limit, err := queryLimit(r) + if err != nil { + writeBadRequest(w, err.Error()) + return + } + + // Unfiltered means everything, so the minimum starts at the least + // severe level rather than at a zero value that happens to be the + // same thing — the distinction matters if the constants ever move. + minLevel := logger.LevelDebug + var requested string + if raw := queryString(r, "level"); raw != "" { + minLevel, err = logger.ParseLevel(raw) + if err != nil { + // 400 naming the four valid values, rather than an empty list + // a caller would read as "the engine logged nothing at this + // level" — the same reasoning the delivery log's status + // filter is built on. + writeBadRequest(w, fmt.Sprintf("level must be one of: %s", strings.Join(shiplog.Levels(), ", "))) + return + } + requested = strings.ToLower(raw) + } + + entries, err := h.Store.List(r.Context(), minLevel, limit) + if err != nil { + writeErr(w, err) + return + } + + out := make([]logEventDTO, 0, len(entries)) + for _, e := range entries { + out = append(out, logEventDTO{ + ID: e.ID, + Level: e.Level.String(), + Message: e.Message, + Fields: e.Fields, + Sink: e.Sink, + ShippedAt: e.ShippedAt, + }) + } + + writeData(w, http.StatusOK, logEventsDTO{ + Events: out, + Count: len(out), + Level: requested, + Levels: shiplog.Levels(), + }) +} diff --git a/httpapi/logging_handlers_test.go b/httpapi/logging_handlers_test.go new file mode 100644 index 0000000..d8397b0 --- /dev/null +++ b/httpapi/logging_handlers_test.go @@ -0,0 +1,297 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/crydensync/cryden/v2" + "github.com/crydensync/cryden/v2/logger" + "github.com/crydensync/cryden/v2/store/memory" + "github.com/crydensync/cryden/v2/token" + + "github.com/crydensync/api/config" + "github.com/crydensync/api/shiplog" +) + +// logEventsResponse mirrors the endpoint's DTO field by field, so a renamed +// or dropped field fails here rather than silently changing the contract an +// operator's console reads. +type logEventsResponse struct { + Data struct { + Events []struct { + ID int64 `json:"id"` + Level string `json:"level"` + Message string `json:"message"` + Fields map[string]string `json:"fields"` + Sink string `json:"sink"` + ShippedAt time.Time `json:"shipped_at"` + } `json:"events"` + Count int `json:"count"` + Level string `json:"level"` + Levels []string `json:"levels"` + } `json:"data"` +} + +type loggingFixture struct { + store *shiplog.MemoryStore + router http.Handler + + adminToken string + userToken string +} + +// newLoggingFixture builds an engine whose operator carries the admin claim +// and a router holding the shipped-events log. Records are written through +// the real sink — shiplog.Logger, composed the way main.go composes it — +// rather than inserted by hand, so what these tests list is what the +// cloud-logging sink actually produced. +func newLoggingFixture(t *testing.T) loggingFixture { + t.Helper() + ctx := context.Background() + + store := shiplog.NewMemoryStore() + var adminID string + engine, err := cryden.New(cryden.Config{ + JWTSecret: "test-secret", + Users: memory.NewUserStore(), + Sessions: memory.NewSessionStore(), + Audit: memory.NewAuditStore(), + Verifications: memory.NewVerificationStore(), + EmailSender: stubMailSender{}, + MagicLinkSender: stubMailSender{}, + AccessTokenClaims: token.ClaimsFunc(func(_ context.Context, userID string) (map[string]any, error) { + if userID == adminID { + return map[string]any{"role": "admin"}, nil + } + return nil, nil + }), + }) + if err != nil { + t.Fatalf("cryden.New on the in-memory stores: %v", err) + } + + admin, err := cryden.SignUp(ctx, engine, "operator@example.com", testPassword, "203.0.113.1") + if err != nil { + t.Fatalf("signup (operator): %v", err) + } + adminID = admin.ID + adminTokens, err := cryden.Login(ctx, engine, "operator@example.com", testPassword, "203.0.113.1", chromeOnMacOS) + if err != nil { + t.Fatalf("login (operator): %v", err) + } + + const userEmail = "dana@example.com" + if _, err := cryden.SignUp(ctx, engine, userEmail, testPassword, "203.0.113.2"); err != nil { + t.Fatalf("signup (user): %v", err) + } + userTokens, err := cryden.Login(ctx, engine, userEmail, testPassword, "203.0.113.2", chromeOnMacOS) + if err != nil { + t.Fatalf("login (user): %v", err) + } + + return loggingFixture{ + store: store, + router: NewRouter(Deps{Engine: engine, Config: config.Config{}, Shipped: store}), + adminToken: adminTokens.AccessToken, + userToken: userTokens.AccessToken, + } +} + +// record writes one entry through the sink the engine would hold, and fails +// the test if it did not land. The sink has no error return — logger.Logger +// has no room for one — so the check is what stops a wiring mistake from +// looking like an endpoint that merely returns nothing. +func (f loggingFixture) record(t *testing.T, level logger.Level, message string, fields map[string]string) { + t.Helper() + before := f.store.Count() + shiplog.NewLogger(f.store).Log(context.Background(), level, message, fields) + if got := f.store.Count(); got != before+1 { + t.Fatalf("recording %q did not reach the store: %d rows, want %d", message, got, before+1) + } +} + +func (f loggingFixture) recent(t *testing.T, token, query string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/v1/admin/logging/recent"+query, nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + f.router.ServeHTTP(rec, req) + return rec +} + +func decodeLogEvents(t *testing.T, rec *httptest.ResponseRecorder) logEventsResponse { + t.Helper() + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } + var resp logEventsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decoding %s: %v", rec.Body.String(), err) + } + return resp +} + +func TestLoggingRecentListsNewestFirst(t *testing.T) { + f := newLoggingFixture(t) + + f.record(t, logger.LevelInfo, "login: completed", map[string]string{"user_id": "u1"}) + f.record(t, logger.LevelWarn, "login: rate limited", map[string]string{"ip": "203.0.113.9"}) + + resp := decodeLogEvents(t, f.recent(t, f.adminToken, "")) + if resp.Data.Count != 2 { + t.Fatalf("count = %d, want 2", resp.Data.Count) + } + if resp.Data.Events[0].Message != "login: rate limited" { + t.Errorf("first row = %q, want the newest", resp.Data.Events[0].Message) + } + + // The vocabulary is reported rather than hardcoded, so a console can + // offer the filter without a second copy of it. + if strings.Join(resp.Data.Levels, ",") != "debug,info,warn,error" { + t.Errorf("levels = %v, want the four names least severe first", resp.Data.Levels) + } + // No filter in force, so nothing is echoed back — "debug" here would + // suggest a filter an operator had not asked for. + if resp.Data.Level != "" { + t.Errorf("level = %q on an unfiltered listing, want empty", resp.Data.Level) + } + + // The record's own fields come back, which is what makes the listing a + // log rather than a list of bare messages. + if resp.Data.Events[0].Fields["ip"] != "203.0.113.9" { + t.Errorf("fields = %v, want the record's own", resp.Data.Events[0].Fields) + } + if resp.Data.Events[0].Sink != shiplog.SinkName { + t.Errorf("sink = %q, want %q", resp.Data.Events[0].Sink, shiplog.SinkName) + } + if resp.Data.Events[0].ShippedAt.IsZero() { + t.Error("shipped_at is zero, want when the sink wrote the record") + } +} + +// level=warn means warn AND WORSE — the same direction the LevelFilter +// above the sink reads the word. A filter that meant "exactly warn" here +// while the shipping filter meant "warn and above" would be one word with +// two meanings in one feature. +func TestLoggingRecentFiltersAtOrAboveTheLevel(t *testing.T) { + f := newLoggingFixture(t) + f.record(t, logger.LevelDebug, "cache miss", nil) + f.record(t, logger.LevelInfo, "login: completed", nil) + f.record(t, logger.LevelWarn, "login: rate limited", nil) + f.record(t, logger.LevelError, "token reuse detected", nil) + + for query, want := range map[string]int{ + "": 4, + "?level=debug": 4, + "?level=info": 3, + "?level=warn": 2, + "?level=error": 1, + "?level=WARNING": 2, // logger.ParseLevel's own leniency, not a second vocabulary + } { + resp := decodeLogEvents(t, f.recent(t, f.adminToken, query)) + if resp.Data.Count != want { + t.Errorf("%s: count = %d, want %d", query, resp.Data.Count, want) + } + } + + // The filter in force is echoed back, in the caller's own word — an + // operator looking at a short list needs to know whether it is short + // because of the filter or because the engine has been quiet. + filtered := decodeLogEvents(t, f.recent(t, f.adminToken, "?level=warn")) + if filtered.Data.Level != "warn" { + t.Errorf("level = %q, want the requested filter echoed back", filtered.Data.Level) + } +} + +// An unrecognized level is a 400 naming the four real values. It must not +// be an empty list, which is indistinguishable from "the engine has been +// quiet" — the one thing a filter must never be able to look like. +func TestLoggingRecentRejectsAnUnknownLevel(t *testing.T) { + f := newLoggingFixture(t) + f.record(t, logger.LevelError, "token reuse detected", nil) + + for _, query := range []string{"?level=fatal", "?level=verbose", "?level=1"} { + rec := f.recent(t, f.adminToken, query) + if rec.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400 (body %s)", query, rec.Code, rec.Body.String()) + continue + } + for _, want := range []string{"debug", "info", "warn", "error"} { + if !strings.Contains(rec.Body.String(), want) { + t.Errorf("%s: body = %s, want the valid values including %q", query, rec.Body.String(), want) + } + } + } +} + +func TestLoggingRecentBoundsTheLimit(t *testing.T) { + f := newLoggingFixture(t) + for _, message := range []string{"one", "two", "three"} { + f.record(t, logger.LevelInfo, message, nil) + } + + limited := decodeLogEvents(t, f.recent(t, f.adminToken, "?limit=2")) + if limited.Data.Count != 2 { + t.Errorf("count = %d with limit=2, want 2", limited.Data.Count) + } + if limited.Data.Events[0].Message != "three" { + t.Errorf("first row = %q, want the newest of the bounded set", limited.Data.Events[0].Message) + } + + // Bounded, not clamped: a caller that asked for 100000 and got 500 back + // has no way to tell that from a log holding 500 records. + for _, query := range []string{"?limit=0", "?limit=501", "?limit=lots"} { + rec := f.recent(t, f.adminToken, query) + if rec.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400 (body %s)", query, rec.Code, rec.Body.String()) + } + } +} + +// A router built without the sink answers 404 rather than 500 — a wiring +// fact, not a server fault, and the same shape every unconfigured feature +// in this API uses. Called directly because this repo has no engine without +// its stores either, and this is the only way to reach the branch. +func TestLoggingRecentWithoutAStoreIsNotFound(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/v1/admin/logging/recent", nil) + rec := httptest.NewRecorder() + + h := &LoggingHandlers{} + h.Recent(rec, req) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404 (body %s)", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "not_configured") { + t.Errorf("body = %s, want the not_configured code", rec.Body.String()) + } +} + +// The shipped copy is the redacted one, and this is the endpoint that shows +// it — so it sits behind the same gate as every other admin report. A +// record can name a user even after redaction, and the keyed digest mode +// exists precisely because the correlation is worth keeping. +func TestLoggingRecentRouteIsGatedByRequireAdmin(t *testing.T) { + f := newLoggingFixture(t) + f.record(t, logger.LevelWarn, "login: rate limited", map[string]string{"ip": "[redacted]"}) + + if rec := f.recent(t, "", ""); rec.Code != http.StatusUnauthorized { + t.Errorf("no token: status = %d, want 401", rec.Code) + } + if rec := f.recent(t, "not-a-real-token", ""); rec.Code != http.StatusUnauthorized { + t.Errorf("garbage token: status = %d, want 401", rec.Code) + } + if rec := f.recent(t, f.userToken, ""); rec.Code != http.StatusForbidden { + t.Errorf("ordinary user: status = %d, want 403 (body %s)", rec.Code, rec.Body.String()) + } + if rec := f.recent(t, f.adminToken, ""); rec.Code != http.StatusOK { + t.Errorf("operator: status = %d, want 200 (body %s)", rec.Code, rec.Body.String()) + } +} diff --git a/httpapi/router.go b/httpapi/router.go index 0fba09b..26bc388 100644 --- a/httpapi/router.go +++ b/httpapi/router.go @@ -8,6 +8,7 @@ import ( "github.com/crydensync/cryden/v2/store" "github.com/crydensync/api/config" + "github.com/crydensync/api/shiplog" "github.com/crydensync/api/usermeta" "github.com/crydensync/api/webhook" ) @@ -51,6 +52,13 @@ type Deps struct { // webhooks, and the handler answers 404 rather than an empty list an // operator would read as "nothing has failed". Hooks webhook.Store + + // Shipped backs the admin shipped-events log — the redacted copy of + // the engine's log records that the cloud sink was handed. Nil unless + // CLOUD_LOGGING is set, for the same reason Hooks is: with the sink + // off, nothing writes rows, and an empty list would be a lie about a + // deployment that ships nothing. + Shipped shiplog.Store } // NewRouter builds the full route table. Called once from main.go. @@ -72,6 +80,7 @@ func NewRouter(d Deps) http.Handler { security := &SecurityHandlers{Audit: d.Audit, Users: d.Users, Config: d.Config} metadata := &MetadataHandlers{Users: d.Users, Meta: d.Meta} hooks := &WebhookHandlers{Store: d.Hooks} + logging := &LoggingHandlers{Store: d.Shipped} mux := http.NewServeMux() @@ -192,5 +201,12 @@ func NewRouter(d Deps) http.Handler { // WebhookHandlers). mux.HandleFunc("GET /v1/admin/webhooks/deliveries", RequireAdmin(engine, hooks.Deliveries)) + // The shipped-events log — the redacted, filtered copy of the engine's + // own log records that the cloud sink was handed, which is what a + // hosted aggregator would have received. Read-only, and the only way + // to see it: cryden keeps no history of what it logged, so this table + // is the history. + mux.HandleFunc("GET /v1/admin/logging/recent", RequireAdmin(engine, logging.Recent)) + return mux } diff --git a/main.go b/main.go index 1b8c5de..1089807 100644 --- a/main.go +++ b/main.go @@ -10,12 +10,14 @@ import ( "github.com/redis/go-redis/v9" "github.com/crydensync/cryden/v2" + "github.com/crydensync/cryden/v2/logger" "github.com/crydensync/cryden/v2/security" "github.com/crydensync/cryden/v2/store/postgres" "github.com/crydensync/api/config" "github.com/crydensync/api/httpapi" "github.com/crydensync/api/operator" + "github.com/crydensync/api/shiplog" "github.com/crydensync/api/templates" "github.com/crydensync/api/usermeta" "github.com/crydensync/api/webhook" @@ -213,6 +215,46 @@ func main() { engineCfg.WebhookEvents = cfg.WebhookEvents } + // Cloud logging. Off by default, and off means Config.Logger stays nil + // and the engine keeps its own console default — there is nothing to + // configure for a deployment that ships no logs anywhere. + // + // The composition is the one cryden's own logger package doc + // prescribes, and the nesting is the whole point: the console logger + // gets every record at full detail, while the shipped copy passes + // through the level filter and the redactor first, so only it loses + // the IP address that makes an incident debuggable. Wrapping the + // fan-out in the redactor instead would strip both copies. + var shippedLog shiplog.Store + if cfg.CloudLogging { + shipped := shiplog.NewLogger(shiplog.NewStore(db)) + shipped.Errors = log.Default() + shippedLog = shipped.Store + + var redacted logger.Logger + if cfg.CloudLogRedaction == config.CloudLogRedactionHash { + // Keyed rather than a bare digest, and keyed with a value of + // its own: the whole IPv4 space is 2^32 values, so an unkeyed + // hash of an address is a lookup table away from being the + // address. cryden's NewHashingRedactor asks for key + // separation explicitly, and config.Load is what enforces + // that the key is present. + var err error + if redacted, err = logger.NewHashingRedactor(shipped, cfg.CloudLogHashKey); err != nil { + log.Fatalf("invalid cloud log redaction: %v", err) + } + } else { + redacted = logger.NewMaskingRedactor(shipped) + } + + engineCfg.Logger = logger.NewMultiLogger( + logger.NewConsoleJSONLogger(), + logger.NewLevelFilter(redacted, cfg.LogLevel), + ) + log.Printf("cloud logging enabled: records at %s and above are redacted (%s) and recorded in shipped_log_events", + cfg.LogLevel, cfg.CloudLogRedaction) + } + engine, err := cryden.New(engineCfg) if err != nil { log.Fatalf("failed to construct cryden engine: %v", err) @@ -248,6 +290,8 @@ func main() { Users: users, Meta: metadata, Hooks: webhookStore, + + Shipped: shippedLog, }) limiter := httpapi.NewEdgeRateLimiter(cfg.EdgeRateLimit, cfg.EdgeRateLimitWindow) handler := httpapi.WithCORS(cfg.CORSOrigins, httpapi.WithEdgeRateLimit(limiter, router)) diff --git a/migrations/011_shipped_log_events.down.sql b/migrations/011_shipped_log_events.down.sql new file mode 100644 index 0000000..b4979d0 --- /dev/null +++ b/migrations/011_shipped_log_events.down.sql @@ -0,0 +1,5 @@ +-- 011_shipped_log_events.down.sql + +DROP INDEX IF EXISTS idx_shipped_log_events_level; +DROP INDEX IF EXISTS idx_shipped_log_events_shipped; +DROP TABLE IF EXISTS shipped_log_events; diff --git a/migrations/011_shipped_log_events.up.sql b/migrations/011_shipped_log_events.up.sql new file mode 100644 index 0000000..d1c23c0 --- /dev/null +++ b/migrations/011_shipped_log_events.up.sql @@ -0,0 +1,63 @@ +-- 011_shipped_log_events.up.sql +-- +-- The "shipped events" log: a record of the log records this +-- deployment's cloud-logging sink was handed. cryden calls a +-- logger.Logger and nothing more — it holds no vendor client, makes no +-- outbound call, and keeps no history of what it logged — so the +-- queryable copy is this repo's own table. +-- +-- What lands here is the REDACTED, FILTERED copy, not the raw one. The +-- sink is wired inside the composition cryden's own package doc +-- prescribes, as +-- +-- logger.NewMultiLogger( +-- logger.NewConsoleJSONLogger(), -- full detail, stdout +-- logger.NewLevelFilter( -- what leaves the box +-- logger.NewMaskingRedactor(shipSink), +-- level, +-- ), +-- ) +-- +-- so the row holds what a hosted aggregator would have received: the +-- IP address replaced by [redacted] or a keyed digest, and everything +-- below LOG_LEVEL never reaching the sink to be written at all. That is +-- the property that makes this table a stand-in for a vendor rather +-- than a second, different log beside one. + +CREATE TABLE shipped_log_events ( + id BIGSERIAL PRIMARY KEY, + + -- One of debug/info/warn/error, written from logger.Level's own + -- String() so a record reads the same here as in the console line + -- and in the API's own level= filter. TEXT rather than an enum, for + -- the same reason webhook_deliveries.event_type is: this repo must + -- not need a migration to learn about a level the engine adds. + level TEXT NOT NULL, + + -- The record's message. cryden's messages are constant strings with + -- every variable part in fields, which is what makes a level filter + -- and a redactor sufficient rather than heuristic. + message TEXT NOT NULL, + + -- The record's structured fields, redacted. Defaulted rather than + -- nullable so a read never has to distinguish "no fields" from + -- "NULL", which a console would render the same way anyway. + fields JSONB NOT NULL DEFAULT '{}'::jsonb, + + -- Which sink wrote the row. Only one value is written today + -- ("database"), and it is recorded anyway: a deployment that adds a + -- second reader of the same Logger — a file, an OpenTelemetry + -- collector — would otherwise have its rows indistinguishable from + -- these in a table it shares. + sink TEXT NOT NULL, + + shipped_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- The admin listing's default view: newest first, unfiltered. +CREATE INDEX idx_shipped_log_events_shipped ON shipped_log_events(shipped_at DESC); + +-- And its filtered view. rank-by-name is not answerable from an index on +-- a TEXT column, so the query passes the set of acceptable level names +-- and this index serves the equality plus the ordering within it. +CREATE INDEX idx_shipped_log_events_level ON shipped_log_events(level, shipped_at DESC); diff --git a/shiplog/logger.go b/shiplog/logger.go new file mode 100644 index 0000000..b8a146a --- /dev/null +++ b/shiplog/logger.go @@ -0,0 +1,125 @@ +package shiplog + +import ( + "context" + "log" + "time" + + "github.com/crydensync/cryden/v2/logger" +) + +// DefaultTimeout bounds one insert. The write happens on the goroutine +// that logged, which is the request path, so this is the most a single +// wedged insert may add to a request — and it is per record, which is why +// the value is small: a request that logs twenty records while the +// database is unreachable spends twenty timeouts, and the level filter in +// front of the sink is what keeps that from being thirty. +const DefaultTimeout = 2 * time.Second + +// Logger is the logger.Logger implementation: it writes every record it +// is handed into the Store. +// +// It implements logger.ContextLogger as well as logger.Logger, which is +// the difference between knowing which request a record came from and +// not. cryden calls Log in preference to the four bare methods when the +// Logger it holds has it — and the MultiLogger it holds here always does +// — so the context of the call being served reaches this sink. Nothing +// reads it today: the insert carries no request id, because a trace key +// belongs to the host app and this repo has not defined one. It is taken +// anyway rather than discarded, because the alternative shape (implement +// only the four methods) makes the context unreachable by construction, +// and a sink that cannot see the request can never correlate with it. +// +// Nothing here panics or returns an error upward, because +// logger.Logger's contract has no room for either. A failed insert is +// reported to Log and the record is lost: a log that could fail the +// operation it was describing would be worse than a log with a hole in +// it. Inside a MultiLogger the hole is survivable — the console copy +// still happened — which is exactly why the composition puts this sink +// beside the console one rather than instead of it. +type Logger struct { + // Store is where records go. A nil Store makes this sink silent, and + // is checked rather than assumed because a nil *Logger inside a + // logger.Logger interface is not a nil interface: NewMultiLogger's + // own doc comment names that trap, and the one setting that produces + // it here is CLOUD_LOGGING being off. + Store Store + + // Timeout bounds one insert. Zero means DefaultTimeout. + Timeout time.Duration + + // Errors receives a line per failed insert. Nil means silent, which is + // what a test wants and what a deployment that would rather not have a + // second failure mode on its stderr can ask for. + // + // Named Errors rather than Log because this type has to implement + // logger.ContextLogger, whose method is Log — a field and a method + // cannot share a name, and the method is the one the interface + // dictates. The webhook worker's field of the same purpose is Log + // only because no such method forced its hand. + Errors *log.Logger +} + +var _ logger.ContextLogger = (*Logger)(nil) + +// NewLogger builds a sink over store with the default timeout. +func NewLogger(store Store) *Logger { + return &Logger{Store: store} +} + +// Log records one event. A nil ctx is treated as context.Background(), as +// logger.ContextLogger requires of every implementation: the four bare +// methods have no context to give and a wrapper standing in for one of +// them passes exactly that. +func (l *Logger) Log(ctx context.Context, level logger.Level, msg string, fields map[string]string) { + if l == nil || l.Store == nil { + return + } + if ctx == nil { + ctx = context.Background() + } + + timeout := l.Timeout + if timeout <= 0 { + timeout = DefaultTimeout + } + // context.WithoutCancel, for the same reason the webhook sender uses + // it: by the time a slow insert runs, the request that logged may be + // gone, and a record about a request is most worth having exactly + // when the request failed. The timeout is what stops that from + // turning into an unbounded write. + insertCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout) + defer cancel() + + err := l.Store.Insert(insertCtx, Entry{ + Level: level, + Message: msg, + Fields: fields, + Sink: SinkName, + }) + if err != nil { + l.logf("shiplog: recording a %s record (%q): %v", level, msg, err) + } +} + +func (l *Logger) Debug(msg string, fields map[string]string) { + l.Log(context.Background(), logger.LevelDebug, msg, fields) +} + +func (l *Logger) Info(msg string, fields map[string]string) { + l.Log(context.Background(), logger.LevelInfo, msg, fields) +} + +func (l *Logger) Warn(msg string, fields map[string]string) { + l.Log(context.Background(), logger.LevelWarn, msg, fields) +} + +func (l *Logger) Error(msg string, fields map[string]string) { + l.Log(context.Background(), logger.LevelError, msg, fields) +} + +func (l *Logger) logf(format string, args ...any) { + if l.Errors != nil { + l.Errors.Printf(format, args...) + } +} diff --git a/shiplog/logger_test.go b/shiplog/logger_test.go new file mode 100644 index 0000000..4dd92d3 --- /dev/null +++ b/shiplog/logger_test.go @@ -0,0 +1,152 @@ +package shiplog + +import ( + "bytes" + "context" + "errors" + "log" + "strings" + "testing" + + "github.com/crydensync/cryden/v2/logger" +) + +// The four bare methods are what the engine calls when the Logger it holds +// is not context-aware, and the whole reason this type exists. Each one has +// to land a row with its own severity — a Debug routed to Info would make +// the level filter above meaningless. +func TestLoggerRecordsEveryLevel(t *testing.T) { + store := NewMemoryStore() + l := NewLogger(store) + ctx := context.Background() + + l.Debug("cache miss", map[string]string{"key": "a"}) + l.Info("login: completed", map[string]string{"user_id": "u1"}) + l.Warn("login: rate limited", map[string]string{"ip": "203.0.113.9"}) + l.Error("token reuse detected", map[string]string{"user_id": "u1"}) + + if n := store.Count(); n != 4 { + t.Fatalf("recorded %d rows, want 4", n) + } + + all, err := store.List(ctx, logger.LevelDebug, 10) + if err != nil { + t.Fatalf("List: %v", err) + } + // Newest first, so the order of the listing is the reverse of the + // order they were logged in. + want := []string{"error", "warn", "info", "debug"} + for i, name := range want { + if got := all[i].Level.String(); got != name { + t.Errorf("row %d level = %q, want %q", i, got, name) + } + } + + // The sink names itself, so a row read out of a table shared with a + // second sink is still attributable to this one. + for _, e := range all { + if e.Sink != SinkName { + t.Errorf("sink = %q, want %q", e.Sink, SinkName) + } + if e.ShippedAt.IsZero() { + t.Errorf("%q was recorded with no timestamp", e.Message) + } + } + + // The fields are the record's own, which is what makes the listing + // useful rather than a list of bare messages. + if all[3].Fields["key"] != "a" { + t.Errorf("debug record fields = %v, want the ones it was logged with", all[3].Fields) + } +} + +// The point of implementing logger.ContextLogger rather than only the four +// methods: ForContext sees the interface and hands over the context of the +// call being served, where a four-method sink would have been returned +// untouched and the context lost at the boundary. +func TestLoggerIsSeenAsAContextLogger(t *testing.T) { + store := NewMemoryStore() + l := NewLogger(store) + + // A cancelled request context — the request that logged has already + // gone. The record must still land: it is most worth having exactly + // when the request failed, which is the same reasoning the webhook + // sender's enqueue is built on. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + bound := logger.ForContext(ctx, l) + if _, ok := bound.(logger.ContextLogger); !ok { + t.Fatal("ForContext did not recognise the sink as a ContextLogger, so the context never reaches it") + } + bound.Warn("login: rate limited", map[string]string{"ip": "203.0.113.9"}) + + if n := store.Count(); n != 1 { + t.Fatalf("recorded %d rows, want 1 — a cancelled request context must not drop the record", n) + } +} + +// A bare nil ctx is what the four methods themselves pass, and +// logger.ContextLogger requires every implementation to treat it as +// context.Background() rather than dereferencing it. +func TestLoggerTreatsANilContextAsBackground(t *testing.T) { + store := NewMemoryStore() + l := NewLogger(store) + + l.Log(nil, logger.LevelInfo, "login: completed", nil) + + if n := store.Count(); n != 1 { + t.Errorf("recorded %d rows with a nil ctx, want 1", n) + } +} + +// The trap logger.NewMultiLogger's own doc comment names: a nil *Logger +// inside a Logger interface is not a nil interface, so a fan-out will call +// it like any other sink. The setting that produces this here is +// CLOUD_LOGGING being off, and a nil dereference inside a log statement +// would take down the login that was only trying to mention something. +func TestLoggerIsSilentWithoutAStore(t *testing.T) { + var nilLogger *Logger + nilLogger.Info("login: completed", nil) + + noStore := &Logger{} + noStore.Info("login: completed", nil) + + // And inside the fan-out the engine actually holds, where the nil + // sink would otherwise be reached on every record. + combined := logger.NewMultiLogger(logger.NewConsoleJSONLogger(), noStore, nilLogger) + combined.Info("login: completed", nil) +} + +// A log that could fail the operation it was describing would be worse than +// a log with a hole in it, so a broken store must not panic and must not +// propagate. It has to be visible somewhere, though, or a sink that has +// never once succeeded looks exactly like a deployment that logs nothing. +func TestLoggerReportsAFailedInsertWithoutFailing(t *testing.T) { + var buf bytes.Buffer + l := &Logger{ + Store: failingStore{err: errors.New("database is down")}, + Errors: log.New(&buf, "", 0), + } + + l.Error("login: failed", map[string]string{"ip": "203.0.113.9"}) + + got := buf.String() + if !strings.Contains(got, "database is down") || !strings.Contains(got, "login: failed") { + t.Errorf("error output = %q, want the store's error and the record that was lost", got) + } + + // Nil Errors is a legitimate choice — a deployment willing to lose the + // record rather than add a second failure mode to stderr — and must + // not panic either. + silent := &Logger{Store: failingStore{err: errors.New("database is down")}} + silent.Error("login: failed", nil) +} + +// failingStore is a Store whose writes always fail, for the error paths. +type failingStore struct{ err error } + +func (s failingStore) Insert(context.Context, Entry) error { return s.err } +func (s failingStore) List(context.Context, logger.Level, int) ([]Entry, error) { + return nil, s.err +} diff --git a/shiplog/memory.go b/shiplog/memory.go new file mode 100644 index 0000000..1a71269 --- /dev/null +++ b/shiplog/memory.go @@ -0,0 +1,121 @@ +package shiplog + +import ( + "context" + "sort" + "sync" + "time" + + "github.com/crydensync/cryden/v2/logger" +) + +// MemoryStore is the in-process Store, for tests and for any embedding +// host that wants the shipped-events log without a database behind it. +// +// It is a faithful double rather than a convenient one, on the two points +// where the two implementations could quietly disagree: +// +// - List means "at or above", and means it the same way the Postgres +// query does: by name against the set levelNames returns, not by a +// rank comparison that happens to agree for the four real levels. A +// double that returned only the exact level would make every filter +// test pass while the real query answered a different question. +// - Fields come back as copies. A caller editing the map it was handed +// cannot reach the stored entry, the way a value read out of JSONB +// cannot reach the row. +// +// What it does not reproduce is the database: there is no JSONB round +// trip here, so a field value that would not survive one is a difference +// this double cannot show. Nothing in flight produces such a value — +// logger.Logger's fields are strings, and every string is valid JSON — but +// the gap is worth naming rather than assuming away. +type MemoryStore struct { + mu sync.Mutex + rows []Entry + next int64 + + // Clock stamps an entry that does not carry its own ShippedAt, so a + // test can make the listing's ordering deterministic instead of + // hoping the wall clock separated two inserts. + Clock func() time.Time +} + +func NewMemoryStore() *MemoryStore { + return &MemoryStore{Clock: func() time.Time { return time.Now().UTC() }} +} + +var _ Store = (*MemoryStore)(nil) + +func (s *MemoryStore) Insert(_ context.Context, e Entry) error { + s.mu.Lock() + defer s.mu.Unlock() + + s.next++ + e.ID = s.next + e.ShippedAt = resolveShippedAt(e, s.Clock()) + e.Sink = sinkOr(e.Sink) + // A copy, so a caller reusing its map cannot rewrite the stored record. + e.Fields = copyFields(e.Fields) + s.rows = append(s.rows, e) + return nil +} + +func (s *MemoryStore) List(_ context.Context, minLevel logger.Level, limit int) ([]Entry, error) { + s.mu.Lock() + defer s.mu.Unlock() + + // Filtering by name against the same set the SQL passes, rather than + // by comparing levels, so the double means literally what the query + // means. The two agree for any Level in range, and this is what makes + // them agree out of range too: Level.String() clamps, so a Level of + // -5 or 99 lands on a real name in both implementations instead of + // being excluded here and included there. + allowed := make(map[string]struct{}, 4) + for _, name := range levelNames(minLevel) { + allowed[name] = struct{}{} + } + + out := make([]Entry, 0, len(s.rows)) + for _, e := range s.rows { + if _, ok := allowed[e.Level.String()]; !ok { + continue + } + out = append(out, copyEntry(e)) + } + // Newest first, ties broken by id descending — the SQL's + // ORDER BY shipped_at DESC, id DESC. + sort.Slice(out, func(i, j int) bool { + if !out[i].ShippedAt.Equal(out[j].ShippedAt) { + return out[i].ShippedAt.After(out[j].ShippedAt) + } + return out[i].ID > out[j].ID + }) + if len(out) > limit { + out = out[:limit] + } + return out, nil +} + +// Count returns how many records have been written. A test helper: no +// production caller needs a total, and no endpoint reports one. +func (s *MemoryStore) Count() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.rows) +} + +func copyEntry(e Entry) Entry { + e.Fields = copyFields(e.Fields) + return e +} + +func copyFields(fields map[string]string) map[string]string { + if fields == nil { + return nil + } + out := make(map[string]string, len(fields)) + for k, v := range fields { + out[k] = v + } + return out +} diff --git a/shiplog/store.go b/shiplog/store.go new file mode 100644 index 0000000..d9f6e0a --- /dev/null +++ b/shiplog/store.go @@ -0,0 +1,270 @@ +// Package shiplog is this repo's implementation of the "shipped events" +// half of cryden's logger seam — the log records a deployment would have +// sent to a hosted aggregator, kept in a table the console can read. +// +// It exists because of the boundary logger's own package doc draws: +// cryden holds a logger.Logger and calls it, and everything that leaves +// the machine belongs to the host app. There is no vendor client in the +// engine and there will not be one. This repo has no vendor client +// either — so "shipped" means "recorded in the log this repo can show an +// operator", the same convention templates/ set for email: a dev +// stand-in for a provider, wired through exactly the seam a real one +// would use, so swapping in a hosted sink later is a change to one line +// of main.go and not to the engine's wiring. +// +// # What lands in the table +// +// The sink sits INSIDE the composition logger's doc comment prescribes: +// +// logger.NewMultiLogger( +// logger.NewConsoleJSONLogger(), // full detail, stdout +// logger.NewLevelFilter( // and the shipped copy +// logger.NewMaskingRedactor(shipSink), +// level, +// ), +// ) +// +// so a row here holds what a vendor would have received — the IP address +// replaced by a marker or a keyed digest, and every record below +// LOG_LEVEL never reaching the sink at all — while stdout keeps the full +// detail that makes an incident debuggable. That placement is the whole +// reason this is a stand-in rather than a second, different log beside +// one: it is the same bytes, minus what would not have left the building. +// +// # The write is synchronous, deliberately +// +// Every record is written on the calling goroutine, which is the request +// path the engine logged from. That is a real cost — one insert per +// record, and a login emits many — and it is not the shape a busy +// deployment wants. It is the shape this one can have: making it +// asynchronous needs a buffer, a flush policy and a shutdown path, and +// this repo has no graceful shutdown anywhere yet (main.go ends at +// log.Fatal(ListenAndServe)). An asynchronous sink whose buffer is never +// flushed is a log that silently drops the last records before a crash, +// which for a log is the failure that matters most. So the honest choice +// is a bounded synchronous write now, and async logging as its own +// change alongside the shutdown path, not smuggled in behind this one. +// +// The level filter in front of the sink is what keeps the volume sane: +// LOG_LEVEL defaults to info and the engine's debug records — the bulk of +// them — never reach here. +// +// # Read-only, like everything else on the admin surface +// +// Nothing here writes through an endpoint. The console reads; whether a +// record should not have been written is a question about a LOG_LEVEL, +// which is configuration, and configuration changes go through the +// settings path like every other one (see CLAUDE.md's hard rule). +package shiplog + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/lib/pq" + + "github.com/crydensync/cryden/v2/logger" +) + +// SinkName is the value recorded in the sink column by the Logger in this +// package. Exported because the admin response reports it back, and a +// console filtering on a string it guessed would be filtering on nothing. +const SinkName = "database" + +// Entry is one shipped record. +// +// Fields is a map[string]string rather than anything richer because that +// is exactly what logger.Logger hands over — the engine's own field +// vocabulary is flat and string-valued, and widening it here would be +// this repo inventing a shape the engine does not produce. +type Entry struct { + ID int64 + Level logger.Level + Message string + Fields map[string]string + Sink string + ShippedAt time.Time +} + +// Store is the persistence seam. Two implementations: PostgresStore and +// MemoryStore, the in-memory double the tests use. +type Store interface { + // Insert records one entry. It is called on the request path, once + // per log record, so it must be a single statement and must not + // dedupe or aggregate: two identical records are two things that + // happened, and a "helpful" unique constraint here would turn a + // retry loop into one row. + // + // A zero ShippedAt is stamped by the store, so a caller that does not + // care about the timestamp does not have to invent one. + Insert(ctx context.Context, e Entry) error + + // List returns entries at or above minLevel, newest first, at most + // limit of them. + // + // "At or above" is the same meaning logger.LevelFilter gives the word + // when it drops records below a threshold — one direction of reading + // for one word, in one feature. level=warn returning only warnings + // while the filter above meant "warn and worse" is exactly the kind + // of second meaning this repo does not want to have. + List(ctx context.Context, minLevel logger.Level, limit int) ([]Entry, error) +} + +var ErrNotFound = errors.New("shipped log event not found") + +// levelNames returns the level names at or above min, in ascending order +// of severity. It is what the Postgres query filters on: an index on a +// TEXT level column answers equality, not rank, so the caller passes the +// set rather than asking SQL to compare severities it has no ordering +// for. +// +// min outside the four constants is clamped, which is what everything in +// logger does with a Level. A caller passing garbage therefore gets +// everything rather than an error, and the same clamping in +// logger.ParseLevel is what stops garbage arriving in the first place. +func levelNames(min logger.Level) []string { + names := make([]string, 0, 4) + for l := logger.LevelDebug; l <= logger.LevelError; l++ { + if l >= min { + names = append(names, l.String()) + } + } + return names +} + +// Levels returns the four level names, least severe first. Exported for +// the same reason webhook.Statuses is: the admin response lists them so a +// console can offer the filter without keeping a second copy of the +// vocabulary that could drift from this one. +func Levels() []string { + return levelNames(logger.LevelDebug) +} + +// parseLevel reads a level back off a stored row. Our own rows always +// hold one of the four canonical names, so the only way this fails is a +// hand-written row — and failing the whole listing over one of those +// would be a log an operator cannot read because of a typo in a row they +// were trying to inspect. So an unrecognized name is filed at the most +// severe end: it surfaces in every filtered view rather than being +// hideable, and the record is never lost. +// +// logger.ParseLevel's error return is not usable as a fallback here — its +// own doc comment says the value it returns on error is the zero value, +// chosen to be wrong in neither direction precisely because it refuses to +// choose. +func parseLevel(name string) logger.Level { + l, err := logger.ParseLevel(name) + if err != nil { + return logger.LevelError + } + return l +} + +// resolveShippedAt is the zero-value rule shared by both stores: an entry +// that does not carry a time is stamped by whichever clock the store +// owns, so Postgres uses the database's and the in-memory double uses the +// test's. +func resolveShippedAt(e Entry, fallback time.Time) time.Time { + if e.ShippedAt.IsZero() { + return fallback + } + return e.ShippedAt +} + +// logColumns is one const so the scan and the query cannot drift apart — +// the same reason webhook's deliveryColumns is one. +const logColumns = `id, level, message, fields, sink, shipped_at` + +// PostgresStore is the durable Store. +type PostgresStore struct { + db *sql.DB +} + +func NewStore(db *sql.DB) *PostgresStore { + return &PostgresStore{db: db} +} + +var _ Store = (*PostgresStore)(nil) + +func (s *PostgresStore) Insert(ctx context.Context, e Entry) error { + // A nil map marshals to the JSON literal "null", which a JSONB column + // with a NOT NULL constraint accepts happily and a reader then has to + // treat as a fourth case. Normalized here so the column has one + // representation of "no fields". + fields := e.Fields + if fields == nil { + fields = map[string]string{} + } + // Marshaled, then handed over as a string: a Go []byte goes to lib/pq + // as bytea hex, which a JSONB column rejects outright. See + // usermeta/store.go, where the same trap cost the same debugging time. + raw, err := json.Marshal(fields) + if err != nil { + return fmt.Errorf("encoding fields for a %s record: %w", e.Level, err) + } + + _, err = s.db.ExecContext(ctx, + `INSERT INTO shipped_log_events (level, message, fields, sink, shipped_at) + VALUES ($1, $2, $3, $4, $5)`, + e.Level.String(), e.Message, string(raw), sinkOr(e.Sink), resolveShippedAt(e, time.Now().UTC()), + ) + return err +} + +func (s *PostgresStore) List(ctx context.Context, minLevel logger.Level, limit int) ([]Entry, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT `+logColumns+` + FROM shipped_log_events + WHERE level = ANY($1) + ORDER BY shipped_at DESC, id DESC + LIMIT $2`, + pq.Array(levelNames(minLevel)), limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + entries := make([]Entry, 0, limit) + for rows.Next() { + e, err := scanEntry(rows) + if err != nil { + return nil, err + } + entries = append(entries, e) + } + return entries, rows.Err() +} + +// scanEntry reads one row. Shared with nothing else today, but kept as a +// function so the column list and the scan order are read together. +func scanEntry(rows *sql.Rows) (Entry, error) { + var ( + e Entry + level string + raw []byte + ) + if err := rows.Scan(&e.ID, &level, &e.Message, &raw, &e.Sink, &e.ShippedAt); err != nil { + return Entry{}, err + } + e.Level = parseLevel(level) + if len(raw) > 0 { + if err := json.Unmarshal(raw, &e.Fields); err != nil { + return Entry{}, fmt.Errorf("decoding fields of shipped log event %d: %w", e.ID, err) + } + } + return e, nil +} + +// sinkOr names the sink on an entry that does not name itself, so a row +// written through this package is always attributable to it. +func sinkOr(sink string) string { + if sink == "" { + return SinkName + } + return sink +} diff --git a/shiplog/store_test.go b/shiplog/store_test.go new file mode 100644 index 0000000..563d525 --- /dev/null +++ b/shiplog/store_test.go @@ -0,0 +1,211 @@ +package shiplog + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/crydensync/cryden/v2/logger" +) + +// newTestStore is a store whose clock a test moves by hand, so a listing's +// ordering is decided rather than hoped for. +func newTestStore() (*MemoryStore, *testClock) { + clock := &testClock{t: time.Date(2026, time.September, 15, 12, 0, 0, 0, time.UTC)} + store := NewMemoryStore() + store.Clock = clock.now + return store, clock +} + +type testClock struct{ t time.Time } + +func (c *testClock) now() time.Time { return c.t } +func (c *testClock) advance(d time.Duration) { c.t = c.t.Add(d) } + +func insert(t *testing.T, store Store, level logger.Level, message string) { + t.Helper() + if err := store.Insert(context.Background(), Entry{Level: level, Message: message}); err != nil { + t.Fatalf("Insert(%q): %v", message, err) + } +} + +// "At or above" is the same direction logger.LevelFilter reads the word, +// and the two have to agree: a sink that shipped warn-and-worse while the +// listing answered "exactly warn" would be a filter whose meaning depended +// on which end you were standing at. +func TestListMeansAtOrAbove(t *testing.T) { + store, _ := newTestStore() + for _, level := range []logger.Level{logger.LevelDebug, logger.LevelInfo, logger.LevelWarn, logger.LevelError} { + insert(t, store, level, level.String()) + } + + for _, tc := range []struct { + min logger.Level + want []string + }{ + {logger.LevelDebug, []string{"debug", "info", "warn", "error"}}, + {logger.LevelInfo, []string{"info", "warn", "error"}}, + {logger.LevelWarn, []string{"warn", "error"}}, + {logger.LevelError, []string{"error"}}, + } { + rows, err := store.List(context.Background(), tc.min, 10) + if err != nil { + t.Fatalf("List(%s): %v", tc.min, err) + } + if len(rows) != len(tc.want) { + t.Errorf("List(%s) returned %d rows, want %d", tc.min, len(rows), len(tc.want)) + continue + } + for i, name := range tc.want { + // Newest first, and the inserts above happened in ascending + // severity, so the expected order is reversed. + if got := rows[len(rows)-1-i].Message; got != name { + t.Errorf("List(%s) row %d = %q, want %q", tc.min, i, got, name) + } + } + } +} + +func TestListOrdersNewestFirstAndBounds(t *testing.T) { + store, clock := newTestStore() + insert(t, store, logger.LevelInfo, "first") + clock.advance(time.Minute) + insert(t, store, logger.LevelInfo, "second") + clock.advance(time.Minute) + insert(t, store, logger.LevelInfo, "third") + + rows, err := store.List(context.Background(), logger.LevelDebug, 10) + if err != nil { + t.Fatalf("List: %v", err) + } + if len(rows) != 3 { + t.Fatalf("listed %d rows, want 3", len(rows)) + } + if rows[0].Message != "third" || rows[2].Message != "first" { + t.Errorf("rows = %q...%q, want the newest first", rows[0].Message, rows[2].Message) + } + + limited, _ := store.List(context.Background(), logger.LevelDebug, 2) + if len(limited) != 2 { + t.Fatalf("listed %d rows with limit 2", len(limited)) + } + if limited[0].Message != "third" || limited[1].Message != "second" { + t.Errorf("limited rows = %q, %q, want the two newest", limited[0].Message, limited[1].Message) + } +} + +// Two records written in the same instant are two things that happened, +// and the tie is broken by id rather than arbitrarily — the SQL's +// ORDER BY shipped_at DESC, id DESC. +func TestListBreaksTiesByID(t *testing.T) { + store, _ := newTestStore() + insert(t, store, logger.LevelInfo, "first") + insert(t, store, logger.LevelInfo, "second") + + rows, _ := store.List(context.Background(), logger.LevelDebug, 10) + if rows[0].Message != "second" || rows[1].Message != "first" { + t.Errorf("rows = %q, %q, want insertion order reversed", rows[0].Message, rows[1].Message) + } +} + +// A returned entry is a copy, so a caller cannot rewrite a stored record +// by editing the map it was handed — the same thing a value read out of +// JSONB cannot do to a row. +func TestListHandsOutCopies(t *testing.T) { + store := NewMemoryStore() + if err := store.Insert(context.Background(), Entry{ + Level: logger.LevelInfo, + Message: "login: completed", + Fields: map[string]string{"user_id": "u1"}, + }); err != nil { + t.Fatalf("Insert: %v", err) + } + + rows, _ := store.List(context.Background(), logger.LevelDebug, 10) + rows[0].Fields["user_id"] = "somebody-else" + + again, _ := store.List(context.Background(), logger.LevelDebug, 10) + if again[0].Fields["user_id"] != "u1" { + t.Errorf("user_id = %q after a caller edited its copy, want the stored value", again[0].Fields["user_id"]) + } +} + +// The store stamps an entry that carries no time, so the sink does not have +// to invent one and the two implementations still agree on what a record +// with no timestamp means. +func TestInsertStampsAnEntryWithNoTime(t *testing.T) { + store, clock := newTestStore() + before := clock.now() + + insert(t, store, logger.LevelWarn, "login: rate limited") + + rows, _ := store.List(context.Background(), logger.LevelDebug, 10) + if !rows[0].ShippedAt.Equal(before) { + t.Errorf("shipped_at = %v, want the store's own clock %v", rows[0].ShippedAt, before) + } + if rows[0].Sink != SinkName { + t.Errorf("sink = %q, want the store to name itself when the caller did not", rows[0].Sink) + } +} + +// The vocabulary the API's level= filter and the console both read, in one +// place and in severity order. +func TestLevelsAreTheFourCanonicalNames(t *testing.T) { + if got := strings.Join(Levels(), ","); got != "debug,info,warn,error" { + t.Errorf("Levels() = %s, want debug,info,warn,error", got) + } +} + +func TestLevelNamesAtOrAbove(t *testing.T) { + for _, tc := range []struct { + min logger.Level + want string + }{ + {logger.LevelDebug, "debug,info,warn,error"}, + {logger.LevelInfo, "info,warn,error"}, + {logger.LevelWarn, "warn,error"}, + {logger.LevelError, "error"}, + // Out of range clamps rather than producing an empty set that + // would silently mean "nothing" for a value that means "all". + {logger.Level(-1), "debug,info,warn,error"}, + {logger.Level(99), ""}, + } { + if got := strings.Join(levelNames(tc.min), ","); got != tc.want { + t.Errorf("levelNames(%d) = %q, want %q", tc.min, got, tc.want) + } + } +} + +// A row written by hand can hold a level this package never writes. Failing +// the whole listing over it would be a log an operator cannot read because +// of a typo in a row they were trying to inspect, so it is filed at the +// most severe end — visible in every filtered view rather than hideable. +func TestParseLevelFilesAnUnknownNameAtTheMostSevereEnd(t *testing.T) { + for _, name := range []string{"fatal", "", "trace", "notice"} { + if got := parseLevel(name); got != logger.LevelError { + t.Errorf("parseLevel(%q) = %s, want error so the record is never lost", name, got) + } + } + // Case and the "warning"/"err" aliases are logger.ParseLevel's own + // leniency, so a hand-written row holding "INFO" reads as info rather + // than being filed as an error. Asserted here because the test above + // would otherwise read as "anything unusual becomes an error". + for raw, want := range map[string]logger.Level{ + "INFO": logger.LevelInfo, + "warning": logger.LevelWarn, + " Warning ": logger.LevelWarn, + } { + if got := parseLevel(raw); got != want { + t.Errorf("parseLevel(%q) = %s, want %s", raw, got, want) + } + } + + // And the names this package writes come back as themselves, which is + // the case that has to work for the endpoint to mean anything. + for _, want := range []logger.Level{logger.LevelDebug, logger.LevelInfo, logger.LevelWarn, logger.LevelError} { + if got := parseLevel(want.String()); got != want { + t.Errorf("parseLevel(%q) = %s, want %s", want, got, want) + } + } +} From cfbf29f9e9689d049503ef96d17116e8704fd499 Mon Sep 17 00:00:00 2001 From: Raymond Nicholas Date: Tue, 15 Sep 2026 13:31:23 +0100 Subject: [PATCH 09/10] docs: document Tier 3 Stage 2 endpoints and tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README gains User metadata, Webhooks and Cloud logging / shipped events sections, plus design notes on the three repo-owned tables, the read-only admin surface and the missing graceful shutdown. spec.yaml goes to 1.3 with the three new admin groups. Its APIKey id description contained unquoted braces inside a flow mapping, so the file had never parsed — fixed here. Co-Authored-By: Claude Code --- README.md | 73 ++++++++ openapi/spec.yaml | 438 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 509 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fc9b0b2..17de229 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,11 @@ DELETE /v1/api-keys/{keyID} (auth required) GET /v1/admin/oauth/health (admin required) GET /v1/admin/security/hash-migration (admin required) +GET /v1/admin/users/{userID}/metadata (admin required) +PUT /v1/admin/users/{userID}/metadata/{key} (admin required) +DELETE /v1/admin/users/{userID}/metadata/{key} (admin required) +GET /v1/admin/webhooks/deliveries (admin required) +GET /v1/admin/logging/recent (admin required) ``` `GET /v1/sessions` answers with *named* sessions: each entry keeps its `id`, `ip`, `user_agent` and `created_at`, and gains `label`, `device` and `location`, all computed on read from the session's own IP and User-Agent — nothing new is stored and no migration exists for it. `label` is the string a "your devices" screen shows (`Chrome on macOS`, or `Unknown device` for a client that sent no User-Agent). `location` is present but empty unless a geolocator is configured, and this repo wires none on purpose: every implementation of that interface calls somebody else's internet service, which is a deployment's decision rather than this repo's. The response shape is documented in `openapi/spec.yaml`. @@ -246,6 +251,70 @@ Every one of these is scoped to the calling user by cryden itself, which derives **No endpoint in this repo authenticates *with* an API key yet.** These three manage them; cryden's `auth.AuthenticateAPIKey` is the other half, and wiring it into a `RequireAPIKey` middleware is a separate change. +## User metadata + +`store.User` in cryden deliberately has no metadata concept — authorization and extra per-user data are a host decision, not an engine one — so per-user metadata is this repo's own table (`migrations/009_user_metadata.*.sql`) and its own package (`usermeta/`). + +Its purpose is **JWT claim mapping**: a console operator attaches a key to a user, and that key appears as a claim in every access token issued for them from then on. `main.go` sets `AccessTokenClaims` to `usermeta.ClaimsProvider(...)`, which merges the user's operator `role` (if any) with every stored metadata key. Because cryden calls that provider at issue time, a metadata change takes effect on that user's next login or refresh — the same latency an operator grant or revoke has, and not retroactive over tokens already issued. + +```json +GET /v1/admin/users/{userID}/metadata +{"data": {"user_id": "...", "metadata": {"plan": "pro", "tenant_id": 41}, "reserved_claim_names": ["aud","exp","iat","iss","jti","nbf","role","sub"]}} + +PUT /v1/admin/users/{userID}/metadata/plan {"value": "pro"} +DELETE /v1/admin/users/{userID}/metadata/plan +``` + +- Writes are **per key**, not a whole-map `PUT`, so two operators editing different fields of one user cannot overwrite each other's work. +- Keys are validated `^[A-Za-z_][A-Za-z0-9_.-]{0,63}$` and refused if they collide with a registered claim name (`sub`, `role`, …) — so a bad key fails when the operator saves it, rather than at every user's next login. The rule lives in `usermeta` rather than in the handler, because it is a data invariant any writer has to pass through. `reserved_claim_names` is reported by `GET` so a console can grey those out instead of letting an operator discover the rule by rejection. +- Values are JSON and stored as `JSONB`, so an object or array is preserved rather than being stringified. Omitting a value and sending an explicit `null` are different things, and the API keeps them different. +- A write for a user that does not exist answers `404 not_found` rather than letting the foreign key fail and surface as a `500`. The path's `{key}` is URL-decoded by `net/http` before it reaches the handler, so a key written `a%2Fb` is judged by the validator above rather than being rejected by URL parsing. + +The claims provider runs **two queries on every login and every refresh** — roughly once per `ACCESS_TOKEN_TTL` per active session. That is the price of claims that are current rather than frozen at signup. + +## Webhooks + +Setting `WEBHOOK_URL` turns on cryden's webhook dispatch: the engine calls this repo's `notify.WebhookSender` for each event in `WEBHOOK_EVENTS` (unset means cryden's own default set, which deliberately excludes `login_success`, `login_failed` and `token_rotated` — the three that fire constantly and say nothing). + +**This repo only enqueues.** cryden calls `SendWebhook` synchronously, in the same goroutine as the login that triggered it, so an HTTP call there would be your receiver's downtime becoming your users' login latency. `SendWebhook` writes one `pending` row to `webhook_deliveries` and returns; a background worker makes the call. The row is the queue — a channel would be faster and would lose everything on restart, and a delivery log that cannot answer "was that lockout announced" for events that vanished before a row was written is not worth having. + +The request body is built once, at enqueue, and stored. Retries send the identical bytes, which is what lets the delivery log answer "show me what we sent that endpoint" for a retry as well as a first attempt. + +| header | meaning | +| --- | --- | +| `X-Cryden-Signature` | `sha256=` + lowercase hex HMAC-SHA256 of the raw body under `WEBHOOK_SECRET`. Absent entirely when no secret is set — never computed over an empty key. | +| `X-Cryden-Event-Type` | the `store.AuditEventType` that fired | +| `X-Cryden-Event-Id` | the engine's idempotency key for this occurrence, **which may be empty** (cryden generates it with `crypto/rand` and deliberately delivers without one rather than dropping the event) | +| `X-Cryden-Delivery-Attempt` | 1-based attempt count, for the receiver's own logs. Not covered by the signature. | + +A receiver in Go can use `webhook.Sign`/`webhook.Verify` rather than reimplementing; in another language it is HMAC-SHA256, key = the shared secret as UTF-8, message = the body byte for byte. **Only the body is signed** — the headers above are informational and a receiver must not make a decision on them. + +Retries use exponential backoff (30s doubling, capped at 30m) up to `WEBHOOK_MAX_ATTEMPTS`, after which the row is `failed` and stays readable. Anything outside `2xx` is a failure, including redirects. A delivery is attempted by exactly one worker: the claim is a single `UPDATE … WHERE id IN (SELECT … FOR UPDATE SKIP LOCKED)` statement, and a row whose worker died mid-delivery is reclaimed once its `claimed_at` goes stale — so raising the worker count later needs no change to the claim. + +`GET /v1/admin/webhooks/deliveries?limit=&status=` lists the log newest first. `response_code` is absent when nothing came back at all (a connection failure, which an operator fixes in a different place from a receiver answering 500). There is deliberately **no** "retry this delivery" endpoint: this surface is read-only (see the design notes), and re-queuing a delivery has consequences for a third party. + +## Cloud logging and shipped events + +`CLOUD_LOGGING` composes a second, redacted, filtered copy of the engine's log records alongside the full-detail JSON line on stdout, in exactly the shape cryden's `logger` package doc prescribes: + +```go +logger.NewMultiLogger( + logger.NewConsoleJSONLogger(), // full detail, stays on stdout + logger.NewLevelFilter( // and the copy that leaves + logger.NewMaskingRedactor(shipSink), // without the personal data + cfg.LogLevel, + ), +) +``` + +Redacting *inside* the fan-out rather than around it is the point: stdout keeps the IP address that makes an incident debuggable, and only the shipped copy loses it. `CLOUD_LOG_REDACTION` picks how — `mask` replaces the value with `[redacted]`, `hash` replaces it with a keyed HMAC digest so the same address still reads as the same address across records ("one IP, forty accounts" is the shape credential stuffing has, and a mask destroys it). `hash` requires `CLOUD_LOG_HASH_KEY`, which should be a value of its own rather than a reuse of `JWT_SECRET` — this key is handed to the component whose job is to hand its output to a third party. + +There is no vendor here: this repo ships no SDK, so "shipped" means "recorded in `shipped_log_events`", which `GET /v1/admin/logging/recent?limit=&level=` reads back. It is the same bytes a hosted aggregator would have received, which is what makes it a stand-in for one rather than a second, different log beside it — swapping in a real client later is a change to one line of `main.go`. + +- `level=warn` means **warn and above**, the same direction the `LevelFilter` above the sink reads the word. One word, one meaning, within one feature. +- An unknown `level` is a `400` naming the four valid values, not an empty list — which is indistinguishable from "the engine has been quiet". +- The write is **synchronous**, on the goroutine that logged. That is a real cost and is not the shape a busy deployment wants; it is the shape this one can have, because an asynchronous sink needs a flush policy and a shutdown path, and this repo has no graceful shutdown anywhere yet. A buffer that is never flushed on exit is a log that silently drops its last records before a crash, which for a log is the failure that matters most. `LOG_LEVEL` (default `info`) is what keeps the volume sane in the meantime, since the engine's debug records never reach the sink. + ## Design notes - `CORS_ORIGINS` is required, no wildcard default — an API handling auth tokens should never allow every origin. @@ -255,6 +324,10 @@ Every one of these is scoped to the calling user by cryden itself, which derives - A paused login is a `200`, not an error: nothing failed, the caller just has one more step. `httpapi/second_factor.go` is the one place that response shape is written. - `DELETE /v1/passkeys/{credentialID}` takes a JSON body (`{"password": "..."}`) — the password is re-confirmation, so a stolen access token alone cannot weaken an account's own auth requirements. - Passkey ceremony options and the browser's credential response travel as raw JSON (an object, not a JSON-encoded string), since that is exactly what `navigator.credentials.create()`/`.get()` produce and consume. +- **Three of this repo's tables are not cryden's and never will be**: `user_metadata`, `webhook_deliveries`, `shipped_log_events`. cryden calls an interface and moves on; it keeps no queryable history of what a sender or a logger did, and `store.User` has no metadata concept on purpose. Each lives in its own package (`usermeta/`, `webhook/`, `shiplog/`) with a Postgres store and an in-memory double behind one interface, mirroring the `store/interfaces.go` + `store/memory` + `store/postgres` split cryden itself uses — which is what makes an endpoint over them testable with no database. +- **The admin surface is read-only by construction.** `GET /v1/admin/webhooks/deliveries` and `GET /v1/admin/logging/recent` report; neither offers a "retry this delivery" button, a "replay this event", or any way to write a log record or a delivery row. That is the same rule cryden's AI admin tools are built under, carried across the repo boundary: an operator reads the state of the system, and every change to it goes through the explicit path that owns that change (or through the receiving system, for a delivery). Adding a write here is a design change, not a convenience. +- `webhook_deliveries.id` is a `BIGSERIAL` surrogate key rather than the natural key you might expect. The event id it corresponds to **can be empty** — cryden generates it with `crypto/rand` and deliberately delivers an event without one rather than dropping it — and a delivery log whose primary key could be blank is a log that loses exactly the rows you would most want to see. The engine's own id is recorded beside it as `event_id` and is used for the receiver's idempotency. +- This repo has **no graceful shutdown**, and as of this tier that is a stated gap rather than an unnoticed one: `main.go` ends at `log.Fatal(http.ListenAndServe(...))`, so the webhook worker's context is never cancelled and the shipped-events sink has no flush-and-exit path. Both were built so that adding one later is a change to `main.go` alone — the worker takes a `context.Context`, which today is `context.Background()`. The sink writes synchronously for the same reason: a buffered sink with no shutdown path drops its last records on a crash. ## License diff --git a/openapi/spec.yaml b/openapi/spec.yaml index 2b611af..027a0d2 100644 --- a/openapi/spec.yaml +++ b/openapi/spec.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: CrydenSync API - version: "1.2" + version: "1.3" description: > A self-hosted HTTP wrapper around the CrydenSync auth engine. Every response follows one of two envelope shapes: {"data": ...} @@ -18,6 +18,15 @@ info: 1.2 is additive: API key management (POST/GET /api-keys and DELETE /api-keys/{keyID}) plus GET /admin/security/hash-migration. No existing path, field or status code changed. + + 1.3 is additive: per-user metadata for JWT claim mapping + (GET/PUT/DELETE /admin/users/{userID}/metadata), the webhook + delivery log (GET /admin/webhooks/deliveries) and the + shipped-events log (GET /admin/logging/recent). Every one is admin + only. No existing path, field or status code changed — but note + that PUT and DELETE /admin/users/{userID}/metadata/{key} are this + API's first endpoints that change what a subsequent token for + another user will contain. See that path's description. servers: - url: http://localhost:8080/v1 description: Local dev @@ -114,7 +123,11 @@ components: value is returned exactly once, by POST /api-keys, and only its SHA-256 hash is stored. properties: - id: { type: string, format: uuid, description: What DELETE /api-keys/{keyID} takes. } + # Quoted because the description contains braces: inside a flow + # mapping `{...}` an unquoted `{keyID}` opens a nested mapping and + # the document stops parsing. Fixed while adding 1.3 — the file had + # never been through a parser. + id: { type: string, format: uuid, description: 'What DELETE /api-keys/{keyID} takes.' } name: { type: string, description: Host-supplied label; not required to be unique or non-empty. } prefix: type: string @@ -190,6 +203,133 @@ components: count only ever rises, while a windowed one falls to zero as the last stragglers log in. + UserMetadata: + type: object + description: > + Every metadata key set on one user, plus the names that may not + be used. The keys here are not merely stored: they are merged + into every access token issued for this user from now on, which + is what the console's claim-mapping screen is for. A change + therefore lands on the user's next login or refresh — the same + latency an operator grant has — and never retroactively over a + token already issued. + properties: + user_id: { type: string, format: uuid } + metadata: + type: object + additionalProperties: true + description: > + Key to value, values being arbitrary JSON preserved exactly + as written — an object stays an object, and an explicit null + is stored as null rather than being dropped. Always + present; an empty object means the user has no keys set. + reserved_claim_names: + type: array + items: { type: string } + description: > + The names a write will be refused for: the seven registered + JWT claims (iss, sub, aud, exp, nbf, iat, jti) and this + API's own "role". Sent so a client can grey these out + instead of letting an operator discover the rule by being + rejected. Read from the engine and this repo's own claim + set, so it stays true as either changes. + + WebhookDelivery: + type: object + description: > + One attempt-series at one event. A row is written before any HTTP + request is made and is never deleted, so this is a record of what + this API decided to send, not a sample of what got through. + properties: + id: { type: integer, description: The delivery's own id, stable across retries. } + event_id: + type: string + description: > + The engine's idempotency key for this occurrence — the value + sent as X-Cryden-Event-Id, and the one a receiver should + deduplicate on. MAY BE EMPTY: the engine generates it with + crypto/rand and deliberately delivers without one rather + than dropping the event, which is exactly why the primary + key is `id` and not this. + event_type: { type: string, description: The audit event that fired, e.g. account_locked. } + user_id: + type: string + format: uuid + description: Absent for an event with no user attached. + ip: + type: string + description: > + Absent when the event carried none. This is the UNREDACTED + address from the delivery payload — the shipping redaction + that applies to the log sink does not apply here, because + this is the record of what was sent to the receiver. + payload: + type: object + description: > + The exact body that was (or will be) POSTed, returned as raw + JSON in the bytes stored at enqueue. Retries send these same + bytes, which is what lets this field answer "show me what we + sent" for a retry as well as a first attempt. The signature + is computed over exactly this. + status: { type: string, enum: [pending, in_flight, delivered, failed] } + attempts: + type: integer + description: > + Attempts STARTED. A row at 3 of the configured maximum tells + an operator the endpoint has been failing; a row that is + `failed` says how many tries it got. + response_code: + type: integer + description: > + Absent when no response arrived at all — a connection + failure or a timeout, which is a different problem from a + receiver answering 500 and one an operator fixes in a + different place. Anything outside 2xx is a failure, + including a redirect. + error: + type: string + description: The receiver's own words where it gave any; absent otherwise. + duration_ms: { type: integer } + created_at: { type: string, format: date-time, description: When the row was enqueued, which is when the triggering request happened. } + next_attempt_at: { type: string, format: date-time, description: When the worker may next claim this row. } + delivered_at: + type: string + format: date-time + nullable: true + description: Absent until a 2xx arrives. Never set on a `failed` row. + + ShippedLogEvent: + type: object + description: > + One record as the shipping sink received it — the REDACTED, + FILTERED copy, not the full-detail line on stdout. The two differ + by design: stdout keeps the IP address that makes an incident + debuggable, and only the copy that leaves the process loses it. + Reading this response is reading what a hosted aggregator would + have been handed. + properties: + id: { type: integer } + level: { type: string, enum: [debug, info, warn, error] } + message: { type: string } + fields: + type: object + additionalProperties: { type: string } + description: > + The record's own key/value fields, already redacted — + "[redacted]" under the masking mode, or an HMAC digest under + the hashing one. Absent for a record logged with none. + sink: + type: string + description: Which sink wrote it, so a row in a table shared with a second sink stays attributable. + shipped_at: + type: string + format: date-time + description: > + When the sink wrote the record, which for a synchronous sink + is when the engine logged it. Nano-precision, because one + login emits many records and at second precision they would + tie and nothing downstream could order them. + responses: BadRequest: description: Malformed request body @@ -674,3 +814,297 @@ paths: items: { $ref: '#/components/schemas/OAuthProviderHealth' } '401': { $ref: '#/components/responses/Unauthorized' } '403': { $ref: '#/components/responses/Forbidden' } + + /admin/users/{userID}/metadata: + get: + summary: Every metadata key set on one user + description: > + Admin only. Per-user metadata is this API's own table, not an + engine feature — cryden's user record deliberately has no + metadata concept, on the grounds that extra per-user data is a + host decision. Its purpose here is JWT claim mapping: the keys + returned are merged into every access token issued for this user + from now on, alongside their operator role if they have one. + Read-only. + security: [{ bearerAuth: [] }] + parameters: + - name: userID + in: path + required: true + schema: { type: string, format: uuid } + responses: + '200': + description: The user's keys and the names that may not be used + content: + application/json: + schema: + type: object + properties: + data: { $ref: '#/components/schemas/UserMetadata' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': + description: > + not_found — no such user, or a userID that is not a UUID at + all. Both answer the same way on purpose: a malformed id + handed to Postgres is a driver error that would surface as a + 500, and "no such user" is the honest description of both. + Also not_configured when the router was built without the + stores this reads. + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } + + /admin/users/{userID}/metadata/{key}: + put: + summary: Set one metadata key on a user + description: > + Admin only. Sets a single key, creating it or replacing it — not + a whole-map replace, so two operators editing different fields of + the same user cannot overwrite each other's work. The user's next + issued token carries the new claim; tokens already issued do not + change, since nothing re-signs them. + + A key is refused with 400 reserved_metadata_key if it names a + registered JWT claim (sub, iss, exp, ...) or this API's own + "role", and with 400 invalid_metadata_key if it does not match + ^[A-Za-z_][A-Za-z0-9_.-]{0,63}$. Both are checked when the key is + saved rather than when a token is next issued, so a bad key fails + once, for the operator who typed it, instead of at every login + for the affected user. `reserved_claim_names` on GET lists the + first set. + security: [{ bearerAuth: [] }] + parameters: + - name: userID + in: path + required: true + schema: { type: string, format: uuid } + - name: key + in: path + required: true + schema: { type: string } + description: > + The claim name to set. Percent-encoded by the client as any + path segment is; a key containing a slash is decoded before + the rule above judges it, so it is rejected by the rule + rather than by URL parsing. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [value] + properties: + value: + description: > + Any JSON value. The field must be present: an + explicit null is a legitimate value to store, while + an absent field is a client bug, and decoding into a + generic value would collapse the two. + responses: + '200': + description: > + The user's whole set after the write, in the same shape GET + returns — so a save is also the refresh, and a client needs + one parser. Read back from storage rather than echoed, which + is what proves the value that landed is the value stored. + content: + application/json: + schema: + type: object + properties: + data: { $ref: '#/components/schemas/UserMetadata' } + '400': + description: > + invalid_metadata_key, reserved_metadata_key, or a body with no + value field. + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': + description: not_found, or not_configured. See GET on this path. + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } + delete: + summary: Remove one metadata key from a user + description: > + Admin only. Removes a single key. The user's next issued token no + longer carries the claim; tokens already issued keep it until + they expire, and nothing revokes them — the same property a role + revoke has. + + A key that is not set is a 404 rather than a bare 200, matching + DELETE /api-keys/{keyID} and DELETE /sessions/{id}: a console + that removed the wrong field should be told, not shown a success + it did not achieve. + security: [{ bearerAuth: [] }] + parameters: + - name: userID + in: path + required: true + schema: { type: string, format: uuid } + - name: key + in: path + required: true + schema: { type: string } + responses: + '200': + description: The user's whole set after the removal. + content: + application/json: + schema: + type: object + properties: + data: { $ref: '#/components/schemas/UserMetadata' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': + description: > + metadata_key_not_found when the user has no such key; + not_found for an unknown or malformed userID; not_configured + when the stores are absent. + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } + + /admin/webhooks/deliveries: + get: + summary: The webhook delivery log, newest first + description: > + Admin only. Lists what this API enqueued for delivery and what + happened to each row. Deliveries are enqueued, never sent + inline: the engine calls the webhook sender synchronously on the + login request path, so this API writes a row and returns, and a + background worker makes the call. Failures are retried with + exponential backoff up to a configured maximum, then recorded as + `failed` and left readable — nothing is ever deleted. + + Read-only by construction. There is deliberately no "retry this + delivery" endpoint: re-queuing a delivery has consequences for a + third party, and this surface reports rather than acts. + security: [{ bearerAuth: [] }] + parameters: + - name: status + in: query + required: false + schema: { type: string, enum: [pending, in_flight, delivered, failed] } + description: > + Restrict to one status. An unrecognized value is a 400 + naming the four valid ones, not an empty list — which is + indistinguishable from "no deliveries", the one thing a + filter must never be confusable with. Absent means every + status. + - name: limit + in: query + required: false + schema: { type: integer, minimum: 1, maximum: 500, default: 50 } + description: > + Out of range or non-numeric is a 400, not a silent clamp — a + caller asking for 100000 and getting 500 back cannot tell + that from a log holding 500 rows. + responses: + '200': + description: The matching rows, newest first + content: + application/json: + schema: + type: object + properties: + data: + type: object + properties: + deliveries: + type: array + items: { $ref: '#/components/schemas/WebhookDelivery' } + count: { type: integer } + status: + type: string + description: The filter in force, absent when none was asked for. + statuses: + type: array + items: { type: string } + description: The four valid values, so a client need not hardcode them. + '400': { $ref: '#/components/responses/BadRequest' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': + description: > + not_configured — the router was built without a delivery log, + which is what an unset WEBHOOK_URL produces. A wiring fact, + not a server fault. + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } + + /admin/logging/recent: + get: + summary: The shipped-events log, newest first + description: > + Admin only. Lists records as the cloud-logging sink received + them: redacted and level-filtered. This deployment ships no + vendor SDK, so "shipped" means recorded in this API's own table, + which is the same bytes a hosted aggregator would have been + handed — swapping in a real client later changes one line of + main.go and nothing here. + + Read-only by construction, like the other admin endpoints. There + is deliberately no endpoint that writes a record, edits the level + threshold, or clears the log: changing what gets shipped is + configuration, and configuration changes go through the same + explicit, human-confirmed settings path every other one does. + security: [{ bearerAuth: [] }] + parameters: + - name: level + in: query + required: false + schema: { type: string, enum: [debug, info, warn, error] } + description: > + The minimum severity, and it means AT OR ABOVE — level=warn + returns warn and error, the same direction the shipping + filter reads the word. One word, one meaning, within one + feature. An unrecognized value is a 400 naming the four valid + ones rather than an empty list. Absent means everything. + - name: limit + in: query + required: false + schema: { type: integer, minimum: 1, maximum: 500, default: 50 } + responses: + '200': + description: The matching records, newest first + content: + application/json: + schema: + type: object + properties: + data: + type: object + properties: + events: + type: array + items: { $ref: '#/components/schemas/ShippedLogEvent' } + count: { type: integer } + level: + type: string + description: > + The filter in force, in the caller's own word, + absent when none was asked for. Present so a + short list can be told apart from a quiet + engine. + levels: + type: array + items: { type: string } + description: The four valid values, least severe first. + '400': { $ref: '#/components/responses/BadRequest' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': + description: > + not_configured — the router was built without a shipped-events + sink, which is what an unset CLOUD_LOGGING produces. + content: + application/json: + schema: { $ref: '#/components/schemas/ErrorResponse' } From 4c9d30a04232bcc26b2180aad766ae76f63c35c3 Mon Sep 17 00:00:00 2001 From: Raymond Nicholas Date: Tue, 15 Sep 2026 13:34:26 +0100 Subject: [PATCH 10/10] docs: record Tier 3 as done and log Stage 2 Marks Tier 3 done in NEXT.md, adds its section to CURRENT-STATE.md and logs the session in PROGRESS.md. The entry says plainly what a green suite does not cover here: no Postgres, so migrations 009-011 have never been applied; the worker's SKIP LOCKED claim is untested; the in-memory double cannot reproduce two workers racing. Co-Authored-By: Claude Code --- docs/development/CURRENT-STATE.md | 97 ++++++++++++++- docs/development/NEXT.md | 25 ++++ docs/development/PROGRESS.md | 188 ++++++++++++++++++++++++++++++ 3 files changed, 307 insertions(+), 3 deletions(-) diff --git a/docs/development/CURRENT-STATE.md b/docs/development/CURRENT-STATE.md index 3df9d47..18acd56 100644 --- a/docs/development/CURRENT-STATE.md +++ b/docs/development/CURRENT-STATE.md @@ -13,7 +13,11 @@ not a router change; Apple is the one that does not fit that shape and has its own `httpapi/apple.go` — see `NEXT.md` Tier 1). Tier 2 added one admin endpoint on top of those, the first in this repo -— see below. +— see below. Tier 3 added three more admin endpoints and this repo's +first three tables of its own, plus the config that lights up Argon2id, +cloud logging and email templates — see below. Every admin endpoint in +this repo is either read-only or an explicit operator action on a named +key; nothing on that surface applies a suggestion by itself. Tier 1 also added the second-factor surface: TOTP enroll/confirm/ disable, passkey registration/list/delete, magic-link request/complete, @@ -217,7 +221,94 @@ cryden copies. `internal/smoketest`'s sessions check also stops accepting "200 with anything in it". -## Tier 3 through 5 +## Tier 3 — config plus real endpoints: DONE + +The first tier that adds things cryden has no concept of. Built in two +stages on `feat/tier3-config-and-endpoints`; `go build`, `go vet`, +`gofmt -l` and `go test ./...` are clean, and `PROGRESS.md` records what +that does and does not cover. + +**Config that lights up an engine feature** (cryden already implements +all of it; this repo supplies values): + +- **Argon2id** (`PASSWORD_HASHER=argon2id` plus the five `ARGON2ID_*` + knobs). `Argon2idParams` starts from `security.DefaultArgon2idParams` + and each env var overrides only the field it names — cryden's + `NewArgon2idHasher` uses a partially-filled struct as a real custom + configuration rather than as "defaults plus overrides", so building + one from only the vars that happened to be set would silently drop + the rest to zero. Switching hasher is safe at any time and needs no + migration: existing bcrypt hashes keep verifying and are rewritten one + successful login at a time. +- **Cloud logging** (`CLOUD_LOGGING`, `LOG_LEVEL`, + `CLOUD_LOG_REDACTION`, `CLOUD_LOG_HASH_KEY`) — see the shipped-events + log below. +- **Email templates** (`EMAIL_TEMPLATE_DIR`, new `templates/` package): + cryden deliberately owns no message copy, so this is entirely this + repo's. `verification.txt` and `magic_link.txt` rendered with + `text/template` (`{{.To}}`, `{{.Token}}`, `{{.URL}}`). Unset keeps the + console senders' built-in lines byte for byte; a set-but-broken + directory is a startup failure. +- **API keys** (`API_KEY_PREFIX`, `POST`/`GET /v1/api-keys`, + `DELETE /v1/api-keys/{keyID}`, all `RequireAuth`): cryden scopes every + one to the calling user by deriving the user ID from the verified + token, so a key belonging to another account, a key that does not + exist and an already-revoked key all answer the same `404 + api_key_not_found`. The raw key is returned once and never again — + cryden stores only its SHA-256 hash. **No endpoint authenticates + *with* an API key yet**; that is a separate change. + +**Endpoints over this repo's own tables:** + +- **`GET /v1/admin/security/hash-migration`** (`RequireAdmin`, + read-only): `store.UserStore.Count` against two `CountByType` calls on + `EventPasswordHashUpgraded` — all-time and windowed — so an operator + can watch a bcrypt-to-Argon2id migration drain. `upgraded_events` + counts events, not users, so the field that actually answers "is this + draining" is `upgraded_events_in_window`, and `estimated_remaining` is + named as an estimate on purpose. +- **Per-user metadata** (`usermeta/`, `migrations/009`): this repo's own + table, because cryden's `store.User` deliberately has no metadata + concept. Its purpose is JWT claim mapping — `main.go` sets + `AccessTokenClaims` to `usermeta.ClaimsProvider(...)`, which merges + the operator `role` with every stored key, so a metadata change lands + on that user's next login or refresh and never retroactively. Admin + `GET`/`PUT`/`DELETE /v1/admin/users/{userID}/metadata[/{key}]`, per key + rather than whole-map so two operators cannot lose each other's work. + Key validation and the reserved-claim rule live in the store, not the + handler, so they hold for any writer. The prefix merge costs **two + queries on every login and every refresh**. +- **Webhook delivery log** (`webhook/`, `migrations/010`): `WEBHOOK_URL` + is the on/off switch. cryden calls `notify.WebhookSender` on the login + request path, so `SendWebhook` writes one `pending` row and returns; + a background worker makes the call. **The row is the queue** — a + channel would lose everything on restart. Exponential backoff 30s + doubling to 30m up to `WEBHOOK_MAX_ATTEMPTS`, then `failed` and left + readable. The body is built at enqueue and stored, so a retry sends + identical bytes and the log can answer "what did we send" for a retry + as well as a first attempt. `GET /v1/admin/webhooks/deliveries` + (admin, read-only, no retry button). `id` is a `BIGSERIAL` surrogate + rather than the event id, which cryden may leave **empty**. +- **Shipped-events log** (`shiplog/`, `migrations/011`): this repo ships + no vendor SDK, so "shipped" means recorded in `shipped_log_events`, + read back by `GET /v1/admin/logging/recent`. `main.go` composes + exactly the shape cryden's `logger` doc prescribes — redacting + *inside* the `MultiLogger` fan-out, so stdout keeps the IP that makes + an incident debuggable and only the copy leaving loses it. `level=` + means "at or above", the same direction the `LevelFilter` reads. + `LOG_LEVEL` (default `info`) keeps the volume sane. + +The three new tables (`009`–`011`) have **never been applied to a real +database** — there is no Postgres in this sandbox. The webhook worker's +claim and backoff behaviour is tested against `httptest` and an +in-memory double, not against Postgres `FOR UPDATE SKIP LOCKED`, and +that double cannot reproduce two workers racing. `PROGRESS.md` says all +of this plainly. + +## Tier 4 and 5 Not started. See `NEXT.md` for the full, ordered, specced-in-detail -queue. +queue. Tier 4 is all behind `RequireAdmin` and stays read-only by +construction, with the decision already made that an AI suggestion +**pre-fills** a settings form and never auto-applies. + diff --git a/docs/development/NEXT.md b/docs/development/NEXT.md index 8031050..5a28e13 100644 --- a/docs/development/NEXT.md +++ b/docs/development/NEXT.md @@ -14,6 +14,8 @@ Tier 1 is done — see the status note under Tier 1 and `PROGRESS.md`'s Tier 2 is done — see the status note under Tier 2 and `PROGRESS.md`'s 2026-09-15 entry. No engine bump this tier, so there were no new cryden migrations to copy. +Tier 3 is done, in two stages on `feat/tier3-config-and-endpoints` — see +the status note under Tier 3 and `PROGRESS.md`'s 2026-09-15 entries. --- @@ -215,6 +217,29 @@ Two details were decided rather than assumed, and are recorded in ## Tier 3 — config plus real endpoints +> **Status: every sub-item below is built, in two stages on +> `feat/tier3-config-and-endpoints`.** Stage 1: Argon2id, cloud-logger +> and email-template config, API keys, hash-migration. Stage 2: +> `user_metadata` with JWT claim mapping, webhooks with a delivery log +> and background worker, cloud-logging shipped-events log. `go build`, +> `go vet`, `gofmt -l` and `go test ./...` are all clean on this branch. +> What is still owed, and said plainly rather than implied: the +> migrations `009`–`011` have **never been applied to a real database** +> (no Postgres in this sandbox), the webhook worker's claim and backoff +> behaviour is tested against `httptest` and an in-memory double +> **rather than against Postgres `FOR UPDATE SKIP LOCKED`**, the +> in-memory double cannot reproduce two workers racing (one mutex), and +> this repo still has **no graceful shutdown** — owed before the +> shipped-events sink could move off the request goroutine. `PROGRESS.md` +> has all of it. +> +> Two deliberate deviations from the spec below, both argued in +> `PROGRESS.md`: `webhook_deliveries` uses a `BIGSERIAL` surrogate +> primary key rather than the event id (which cryden may leave **empty**, +> by design, when its `crypto/rand` generator fails), and the shipped +> copy is recorded in this repo's own table rather than sent to a vendor, +> because this repo ships no vendor SDK. + - **Argon2id, cloud loggers, custom email templates**: config only. Custom email templates specifically need **no engine change at all** — cryden deliberately has no template config (there's a test diff --git a/docs/development/PROGRESS.md b/docs/development/PROGRESS.md index 174ad61..90a545d 100644 --- a/docs/development/PROGRESS.md +++ b/docs/development/PROGRESS.md @@ -490,3 +490,191 @@ Next: Stage 2, once someone can run a build. The check-in the user asked for is the point at which this entry was written; Stage 1's code should be built, vetted, formatted and tested before Stage 2 starts on top of it. + +## 2026-09-15 — Tier 3, Stage 2 (metadata, webhooks, shipped events) + +All four commands were run on `feat/tier3-config-and-endpoints` before +each commit, and all four were clean: + +``` +go build ./... ok +go vet ./... ok +gofmt -l . (no output) +go test -count=1 ./... config 0.031s httpapi 13.308s shiplog 0.006s + templates 0.008s usermeta 0.008s webhook 0.029s +``` + +The Stage 1 entry ended by saying Stage 2 should not start until +someone could run a build. The toolchain here recovered, so it did. + +Four commits, one logical step each: + +- `d43a73d` — `usermeta/`, `migrations/009`, the three admin routes and + the claims-provider merge in `main.go`. +- `b3292c9` — `webhook/` (store, sender, worker), `migrations/010`, the + deliveries endpoint, the `WEBHOOK_*` config and `.env.example` block. +- `bd2c990` — `shiplog/` (store, logger), `migrations/011`, the logging + endpoint, and the `MultiLogger` composition in `main.go`. +- `cfbf29f` — `README.md` and `openapi/spec.yaml` (1.3), then the three + docs files. + +### What was built, and the decisions worth re-reading + +- **A repo-owned store is an interface, a Postgres implementation and an + in-memory double, in one package.** `usermeta`, `webhook` and + `shiplog` each follow cryden's own `store/interfaces.go` + + `store/memory` + `store/postgres` split. That is what makes these + endpoints testable with no Postgres — which matters more than usual + here, because there is none in this sandbox. +- **The webhook delivery row is the queue, not a channel.** cryden calls + `SendWebhook` synchronously on the login request path, so `SendWebhook` + writes one `pending` row and returns; the capacity-1 channel is only a + nudge, and a full channel drops the hint rather than blocking. A + channel would lose everything on restart, and "was that lockout + announced" is unanswerable for an event that vanished before a row + was written. +- **The body is built once, at enqueue, and stored.** Retries resend + identical bytes, so the delivery log answers "what did we send" for a + retry as well as a first attempt, and the signature covers the same + bytes the log shows. +- **`webhook_deliveries.id` is a `BIGSERIAL` surrogate, not the event + id.** The spec said `id UUID PK`. But `notify.WebhookEvent.ID` **may + be empty** — cryden generates it with `crypto/rand` and on generator + failure deliberately delivers without one — and a delivery log whose + primary key can be blank loses exactly the rows an operator most wants. + The engine's id is recorded beside it as `event_id`. This is a + deviation from the plan and is why it is written down. +- **"Shipped" means recorded in this repo's own table.** There is no + vendor SDK here, so `shipped_log_events` holds the same bytes a hosted + aggregator would have received, which is what makes it a stand-in for + one rather than a second, different log beside it. Swapping in a real + client is a change to one line of `main.go`. +- **The shipped-events sink writes synchronously, and that is a + deliberate ceiling.** An asynchronous sink needs a buffer, a flush + policy and a shutdown path, and this repo has no graceful shutdown + anywhere yet. A buffer that is never flushed on exit is a log that + silently drops its last records before a crash — for a log, the + failure that matters most. `LOG_LEVEL` (default `info`) is what keeps + the volume sane meanwhile. +- **`level=` on the logging endpoint means "at or above".** The same + direction `logger.LevelFilter` reads the word, so one word means one + thing within one feature. The in-memory double filters **by name + against the same set the SQL passes**, so it is faithful by + construction even for out-of-range levels, where `Level.String()` + clamps. +- **An unrecognized stored level name is filed at `LevelError`**, not + dropped. Failing a whole listing over one hand-written row would be a + log an operator cannot read because of a typo in a row they were + trying to inspect. +- **`sink` is recorded even though only one value is written today** — + a row read out of a table shared with a second sink stays + attributable. +- **Metadata key validation lives in `usermeta`, not `httpapi`.** The + reserved-claim rule is a data invariant, so it holds for any writer. + `reserved_claim_names` is reported by `GET` so a console can grey + those out rather than let an operator discover the rule by rejection. + Writes are per key, never a whole-map `PUT`, so two operators editing + different fields cannot lose each other's work. +- **`PUT`'s body decodes `value` into a `json.RawMessage`, not an + `any`.** `{"value": null}` and a missing `value` are different things, + and decoding into `any` collapses both to nil. +- **A malformed `userID` is a `404`, not a `500`.** Handed straight to + Postgres, `"not-a-uuid"` is a driver error — "invalid input syntax for + type uuid" — which `mapError` turns into a 500 an operator reads as a + bug in the API rather than as a stale bookmark. + +### Bugs found, and how + +Three real ones, none of which a type check would have caught: + +- **`WEBHOOK_MAX_ATTEMPTS` set without `WEBHOOK_URL` did not fail + startup.** The orphaned-setting check used `os.LookupEnv`, but the + config tests' own `loadForTest` uses `t.Setenv(name, "")` — which + *sets* the variable to empty — so six existing tests failed. The + package's documented convention is that empty counts as unset + everywhere, so the check became `os.Getenv(...) != ""`. Found by + running the suite, which is the only thing that would have. +- **`openapi/spec.yaml` had never parsed as YAML.** `APIKey.id`'s + description — `What DELETE /api-keys/{keyID} takes.` — sat unquoted + inside a flow mapping, so the `{` opened a nested mapping and a parser + stops there. Nothing had ever run the file through one; it was caught + only because the 1.3 additions were validated. Fixed by quoting that + one scalar, with a comment saying why. It is a syntax fix, not a + contract change — no path, field or status code moved, so 1.3's + "additive only" note stands. +- **`webhook.Sender` as a typed nil.** A nil `*webhook.Sender` assigned + to cryden's `notify.WebhookSender` field is non-nil to cryden and + would silently turn on `DefaultWebhookEvents` for a deployment with + `WEBHOOK_URL` unset. `main.go` assigns the field inside the `if + webhookStore != nil` block for exactly that reason, and the store is + declared as the interface rather than the concrete type. The same + class of trap `logger.NewMultiLogger` documents for untyped nils. + +Two test bugs, both found by a red test and both the test's fault: +`TestWebhookDeliveriesFiltersByStatus` resolved rows with `ClaimDue`, +which sweeps *every* due row, so its second row came back `in_flight` +rather than `pending` — the test now resolves before seeding; and +`TestParseLevelFilesAnUnknownNameAtTheMostSevereEnd` asserted that +`"INFO"` and `"warning "` were unknown, when `logger.ParseLevel` is +case-insensitive, trims, and accepts the `warning` alias. The premise +was wrong, not the code. + +One naming collision, the same class as Stage 1's `Deliveries`: +`shiplog.Logger` could not have both a `Log` method (the +`logger.ContextLogger` interface dictates the name) and a `Log` field, +so the field is `Errors`. + +### Verification: what this does NOT cover + +Said plainly, per `CODEX.md`, rather than implied by a green suite: + +- **There is no Postgres and no network in this sandbox.** + `migrations/009`, `010` and `011` have **never been applied to a real + database** — not once, in any environment. They are a copy of a + design, not a verified schema. Everything downstream of them is + tested through the in-memory doubles. +- **The webhook worker's claim and backoff behaviour is not tested + against Postgres.** `ClaimDue`'s single `UPDATE … WHERE id IN (SELECT + … FOR UPDATE SKIP LOCKED)` statement has not been run. What is tested + is the worker's behaviour against `httptest` and `MemoryStore`. +- **The in-memory double cannot reproduce two workers racing.** It is + one mutex, so it proves the worker handles a claimed row correctly and + proves nothing about contention. `SKIP LOCKED` is the reason raising + the worker count later is safe, and that reason is unverified here. +- **`internal/smoketest` still has never been run** against a database, + unchanged from every previous tier's note. +- **WebAuthn still needs a real browser authenticator, and Apple a live + round trip.** Unchanged. +- **The `usermeta` claims path is tested for storage and for the merge, + but the "reaches a freshly issued token" assertion runs on cryden's + in-memory user store**, not on Postgres' `user_metadata` table. +- **`shiplog`'s Postgres `List` has not been run against the JSONB + column it reads.** The `lib/pq` bytea trap (a `[]byte` param is sent + as bytea hex, which a JSONB column rejects, so params go as + `string(raw)`) is handled by reading rather than by a passing test — + the `Insert` path that would exercise it needs a database. + +Newly owed by this tier, alongside the three tables: **the graceful +shutdown the Stage 1 entry already flagged.** The webhook worker takes a +`context.Context` and gets `context.Background()`; the shipped-events +sink writes synchronously precisely because there is nowhere to flush a +buffer on exit. Both become cheap once shutdown exists and neither was +smuggled in behind the other. + +### Noticed while working, not fixed + +- **`openapi/spec.yaml` still predates Tier 1** — unchanged from the + Tier 2 and Stage 1 notes. Stage 2 added only its own schemas, paths and + the 1.3 version bump; the gap is still there. +- **`README.md`'s "Design notes" now carries the repo-wide read-only + rule as prose.** It is in `CLAUDE.md` as a rule; a reviewer reading + only the README previously had no way to know why there is no retry + button. +- **The delivery log and the shipped-events log both answer `404 + not_configured` when their store is nil**, which is a wiring fact. A + client cannot currently tell that apart from "the resource genuinely + does not exist" — the same shape every other unconfigured feature in + this API already uses, so it is consistent rather than new. + +Tier 3 is complete. Next is Tier 4, which stays read-only by +construction with the pre-fill-never-auto-apply decision already made.