forge-alloy.schema.json

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://github.com/CambrianTech/forge-alloy/schema/forge-alloy.schema.json",
  "title": "ForgeAlloy",
  "description": "Portable pipeline specification for model forging",
  "type": "object",
  "required": ["name", "version", "source", "stages", "cycles"],
  "properties": {
    "name": { "type": "string", "description": "Alloy name (e.g., qwen3.5-4b-code-balanced)" },
    "version": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$", "description": "Semver version of this alloy spec" },
    "description": { "type": "string" },
    "author": { "type": "string" },
    "tags": { "type": "array", "items": { "type": "string" } },
    "license": { "type": "string", "default": "apache-2.0" },

    "source": { "$ref": "#/$defs/AlloySource" },
    "stages": { "type": "array", "items": { "$ref": "#/$defs/AlloyStage" }, "minItems": 1 },
    "cycles": { "type": "integer", "minimum": 1, "default": 1 },
    "hardware": { "$ref": "#/$defs/AlloyHardware" },
    "outputs": { "$ref": "#/$defs/AlloyOutputs" },

    "results": { "$ref": "#/$defs/AlloyResults", "description": "Populated after execution — benchmark scores, hardware verification, samples" },

    "sourceAlloyId": { "type": "string", "description": "Parent alloy for re-forge lineage" },
    "forgedModelIds": { "type": "array", "items": { "type": "string" }, "description": "HF model IDs produced" }
  },

  "$defs": {
    "AlloySource": {
      "type": "object",
      "required": ["baseModel", "architecture"],
      "properties": {
        "baseModel": { "type": "string", "description": "HuggingFace model ID (e.g., Qwen/Qwen3.5-4B)" },
        "architecture": { "type": "string", "enum": ["qwen3_5", "qwen3_5_moe", "qwen2", "llama", "mistral", "phi", "gemma", "deepseek"] },
        "revision": { "type": "string" },
        "isMoE": { "type": "boolean", "default": false },
        "totalExperts": { "type": "integer" }
      }
    },

    "AlloyResults": {
      "type": "object",
      "description": "Complete results from executing an alloy pipeline. Empty in recipe alloys, populated after forging.",
      "properties": {
        "completedAt": { "type": "string", "format": "date-time", "description": "When the forge completed" },
        "durationMinutes": { "type": "number", "minimum": 0, "description": "Total forge wall-clock time" },
        "baselinePerplexity": { "type": "number", "description": "Perplexity before forging" },
        "finalPerplexity": { "type": "number", "description": "Perplexity after forging" },
        "improvementPct": { "type": "number", "description": "Improvement percentage (positive = better)" },
        "finalSizeGb": { "type": "number", "description": "Model size after forging in GB" },
        "finalParams": { "type": "string", "description": "Parameter count after forging (e.g., '3.8B')" },
        "benchmarks": {
          "type": "array",
          "items": { "$ref": "#/$defs/BenchmarkResult" },
          "description": "Benchmark results — extensible, each defines its own metrics"
        },
        "hardwareVerified": {
          "type": "array",
          "items": { "$ref": "#/$defs/HardwareProfile" },
          "description": "Verified hardware performance for model card device grid"
        },
        "samples": {
          "type": "array",
          "items": { "$ref": "#/$defs/GenerationSample" },
          "description": "Raw model output — no cherry-picking"
        },
        "integrity": { "$ref": "#/$defs/IntegrityAttestation" }
      }
    },

    "BenchmarkResult": {
      "type": "object",
      "required": ["name", "metrics"],
      "description": "A single benchmark result. Metrics are open-ended — each benchmark reports whatever it wants.",
      "properties": {
        "name": { "type": "string", "description": "Benchmark name (e.g., humaneval, mmlu-pro, imo-proofbench-advanced)" },
        "subset": { "type": "string", "description": "Optional benchmark subset" },
        "metrics": {
          "type": "object",
          "additionalProperties": { "oneOf": [{ "type": "number" }, { "type": "string" }, { "type": "boolean" }] },
          "description": "Open metric bag. Examples: { passing: 63, total: 85, score: 74.1 } for HumanEval, { proofScore: 42.0, difficulty: 'advanced' } for IMO-ProofBench"
        },
        "submittedToLeaderboard": { "type": "boolean", "default": false },
        "resultHash": { "type": "string", "description": "Per-benchmark integrity hash" }
      }
    },

    "HardwareProfile": {
      "type": "object",
      "required": ["device", "format"],
      "description": "Verified performance on a specific device — generates model card device grid.",
      "properties": {
        "device": { "type": "string", "description": "Device name (e.g., MacBook Air 8GB, RTX 5090, iPhone 17)" },
        "format": { "type": "string", "description": "Format used (e.g., Q4_K_M, fp16, mlx-4bit)" },
        "sizeGb": { "type": "number", "description": "Model file size in GB for this format" },
        "tokensPerSec": { "type": "number", "description": "Inference speed in tokens/second" },
        "memoryUsageGb": { "type": "number", "description": "RAM/VRAM usage during inference" },
        "verified": { "type": "boolean", "default": false, "description": "Actually tested (true) or estimated (false)" }
      }
    },

    "GenerationSample": {
      "type": "object",
      "required": ["label", "prompt", "completion"],
      "description": "Raw model output sample — no cherry-picking, no post-processing.",
      "properties": {
        "label": { "type": "string", "description": "Label (e.g., Merge Sort, Binary Search Tree)" },
        "prompt": { "type": "string" },
        "completion": { "type": "string" },
        "baselineCompletion": { "type": "string", "description": "Base model output for comparison" }
      }
    },

    "IntegrityAttestation": {
      "type": "object",
      "required": ["trustLevel", "code", "modelHash", "attestedAt"],
      "description": "Cryptographic attestation modeled after FIDO2/WebAuthn. Self-attested prevents accidental corruption only. Enclave required for tamper-proof.",
      "properties": {
        "trustLevel": {
          "type": "string",
          "enum": ["self-attested", "verified", "enclave"],
          "description": "Trust tier. Self-attested: signer vouches (NOT adversary-proof). Verified: third-party audited. Enclave: TEE hardware-bound."
        },
        "code": { "$ref": "#/$defs/CodeAttestation" },
        "modelHash": { "type": "string", "description": "SHA-256 of FULL model weights (every byte, not partial)" },
        "alloyHash": { "type": "string", "description": "SHA-256 of alloy pipeline (required when signed)" },
        "datasets": {
          "type": "array",
          "items": { "$ref": "#/$defs/DatasetAttestation" },
          "description": "Hashes of benchmark datasets (cover exactly the subset used)"
        },
        "nonce": { "type": "string", "description": "Verifier-provided challenge for replay prevention (like WebAuthn challenge)" },
        "audience": { "type": "string", "description": "Audience binding — prevents cross-marketplace replay" },
        "signature": { "$ref": "#/$defs/AttestationSignature" },
        "certifications": {
          "type": "array",
          "items": { "$ref": "#/$defs/AdapterAttestation" },
          "description": "Third-party adapter attestations — independent certifications from approved adapters. Each adapter signs with its own key."
        },
        "attestedAt": { "type": "string", "format": "date-time" }
      }
    },

    "AdapterAttestation": {
      "type": "object",
      "required": ["adapter", "version", "domain", "attestedAt"],
      "description": "Third-party adapter certification. Like UL: independent authority tests one aspect, signs with own key.",
      "properties": {
        "adapter": { "type": "string", "description": "Adapter identifier (e.g., forge-alloy/humaneval, ul/safety-audit, nvidia/perf-cert)" },
        "version": { "type": "string", "description": "Adapter version (semver)" },
        "domain": { "type": "string", "description": "What this adapter certifies (e.g., benchmark:humaneval, safety:toxicity, hardware:inference)" },
        "result": {
          "type": "object",
          "additionalProperties": true,
          "description": "Adapter-specific results — open-ended metrics"
        },
        "adapterHash": { "type": "string", "description": "SHA-256 of adapter binary/script" },
        "signature": { "$ref": "#/$defs/AttestationSignature" },
        "nonce": { "type": "string", "description": "API-provided nonce for this adapter run" },
        "sourceRepo": { "type": "string", "description": "GitHub URL — open source adapters get higher trust" },
        "commit": { "type": "string", "description": "Git commit of adapter at execution time" },
        "attestedAt": { "type": "string", "format": "date-time" }
      }
    },

    "CodeAttestation": {
      "type": "object",
      "required": ["runner", "version", "binaryHash"],
      "description": "Attestation of the code that produced results — proves the forge runner is genuine.",
      "properties": {
        "runner": { "type": "string", "description": "Runner identifier (e.g., sentinel-ai/forge_model)" },
        "version": { "type": "string", "description": "Runner version (semver)" },
        "binaryHash": { "type": "string", "description": "SHA-256 of the runner binary or entry script" },
        "sourceHash": { "type": "string", "description": "SHA-256 of the source repository at build time" },
        "commit": { "type": "string", "description": "Git commit hash of the runner source" },
        "environment": { "type": "string", "description": "Execution environment description" },
        "environmentHash": { "type": "string", "description": "SHA-256 of the container image or environment" }
      }
    },

    "DatasetAttestation": {
      "type": "object",
      "required": ["name", "hash"],
      "description": "Attestation of a benchmark dataset — proves eval used unmodified data.",
      "properties": {
        "name": { "type": "string", "description": "Dataset name (e.g., humaneval, mmlu-pro)" },
        "version": { "type": "string", "description": "Dataset version or commit hash" },
        "hash": { "type": "string", "description": "SHA-256 of the dataset files" },
        "source": { "type": "string", "description": "Source URL or repository" }
      }
    },

    "AttestationSignature": {
      "type": "object",
      "required": ["algorithm", "publicKey", "value"],
      "description": "Signature over RFC 8785 JCS-canonical attestation payload. Extensible to post-quantum.",
      "properties": {
        "algorithm": {
          "type": "string",
          "enum": ["ES256", "ES384", "EdDSA", "ML-DSA-65", "ML-DSA-87", "SLH-DSA-128s"],
          "description": "Signing algorithm. ES256 default. PQC variants for quantum-safe migration."
        },
        "publicKey": { "type": "string", "description": "Base64url public key — verifiers MUST check against registry" },
        "value": { "type": "string", "description": "Base64url signature over SHA-256(JCS payload)" },
        "keyId": { "type": "string", "description": "Key ID in registry for lookup and revocation" },
        "credentialId": { "type": "string", "description": "WebAuthn credential ID (see passkey signing gap in docs)" },
        "certificateChain": {
          "type": "array",
          "items": { "type": "string" },
          "description": "PEM-encoded certificate chain (leaf first) for verified tier"
        },
        "keyRegistry": { "type": "string", "format": "uri", "description": "Registry URL for key verification and revocation" }
      }
    },

    "AlloyStage": {
      "type": "object",
      "required": ["type"],
      "discriminator": { "propertyName": "type" },
      "oneOf": [
        { "$ref": "#/$defs/PruneStage" },
        { "$ref": "#/$defs/TrainStage" },
        { "$ref": "#/$defs/LoRAStage" },
        { "$ref": "#/$defs/CompactStage" },
        { "$ref": "#/$defs/QuantStage" },
        { "$ref": "#/$defs/EvalStage" },
        { "$ref": "#/$defs/PublishStage" },
        { "$ref": "#/$defs/ExpertPruneStage" },
        { "$ref": "#/$defs/ContextExtendStage" },
        { "$ref": "#/$defs/ModalityStage" }
      ]
    },

    "PruneStage": {
      "type": "object",
      "required": ["type", "strategy", "level"],
      "properties": {
        "type": { "const": "prune" },
        "strategy": { "type": "string", "enum": ["entropy", "magnitude", "gradient", "random"] },
        "level": { "type": "number", "minimum": 0.0, "maximum": 0.9 },
        "minHeadsPerLayer": { "type": "integer", "minimum": 1, "default": 4 },
        "minKvHeadsPerLayer": { "type": "integer", "minimum": 1, "default": 2 },
        "analysisSteps": { "type": "integer", "minimum": 10, "default": 200 }
      }
    },

    "TrainStage": {
      "type": "object",
      "required": ["type", "domain", "steps", "learningRate"],
      "properties": {
        "type": { "const": "train" },
        "domain": { "type": "string" },
        "dataset": { "type": "string" },
        "steps": { "type": "integer", "minimum": 1 },
        "learningRate": { "type": "string" },
        "batchSize": { "type": "integer", "minimum": 1, "maximum": 64, "default": 4 },
        "gradientAccumulation": { "type": "integer", "minimum": 1, "maximum": 16, "default": 1 },
        "scheduler": { "type": "string", "enum": ["cosine", "linear", "constant", "constant_with_warmup", "polynomial"], "default": "cosine" },
        "warmupRatio": { "type": "number", "minimum": 0.0, "maximum": 1.0, "default": 0.03 },
        "weightDecay": { "type": "number", "minimum": 0.0, "maximum": 1.0, "default": 0.01 },
        "maxGradientNorm": { "type": "number", "minimum": 0.0, "maximum": 10.0, "default": 1.0 },
        "precision": { "type": "string", "enum": ["bf16", "fp16", "fp32"], "default": "bf16" },
        "sequenceLength": { "type": "integer", "minimum": 128, "maximum": 131072, "default": 2048 },
        "optimizations": {
          "type": "array",
          "items": { "type": "string", "enum": ["flash_attention", "gradient_checkpointing", "optimizer_8bit", "paged_optimizer", "fused_kernels", "sequence_packing", "dynamic_padding", "activation_offloading"] }
        }
      }
    },

    "LoRAStage": {
      "type": "object",
      "required": ["type", "rank"],
      "properties": {
        "type": { "const": "lora" },
        "rank": { "type": "integer", "minimum": 1, "maximum": 256, "default": 32 },
        "alpha": { "type": "integer", "minimum": 1, "maximum": 512 },
        "dropout": { "type": "number", "minimum": 0.0, "maximum": 1.0, "default": 0.05 },
        "targetModules": { "type": "array", "items": { "type": "string" }, "default": ["q_proj", "k_proj", "v_proj", "o_proj"] },
        "quantize": { "type": "boolean", "default": true },
        "quantizeBits": { "type": "integer", "enum": [4, 8], "default": 4 },
        "dataset": { "type": "string" },
        "epochs": { "type": "integer", "minimum": 1, "maximum": 20, "default": 3 },
        "learningRate": { "type": "string", "default": "1e-4" },
        "batchSize": { "type": "integer", "minimum": 1, "maximum": 64, "default": 4 },
        "mergeAfter": { "type": "boolean", "default": false }
      }
    },

    "CompactStage": {
      "type": "object",
      "required": ["type"],
      "properties": {
        "type": { "const": "compact" },
        "deadThreshold": { "type": "number", "minimum": 0.0, "maximum": 1.0, "default": 0.1 },
        "dormantThreshold": { "type": "number", "minimum": 0.0, "maximum": 1.0, "default": 0.2 },
        "lowThreshold": { "type": "number", "minimum": 0.0, "maximum": 1.0, "default": 0.3 },
        "mediumThreshold": { "type": "number", "minimum": 0.0, "maximum": 1.0, "default": 0.5 },
        "highThreshold": { "type": "number", "minimum": 0.0, "maximum": 1.0, "default": 0.7 },
        "targetSizeGb": { "type": "number", "minimum": 0.1 },
        "enableQuantization": { "type": "boolean", "default": true }
      }
    },

    "QuantStage": {
      "type": "object",
      "required": ["type", "format", "quantTypes"],
      "properties": {
        "type": { "const": "quant" },
        "format": { "type": "string", "enum": ["gguf", "mlx", "safetensors", "onnx"] },
        "quantTypes": {
          "type": "array",
          "items": { "type": "string", "enum": ["Q2_K", "Q3_K_S", "Q3_K_M", "Q3_K_L", "IQ4_XS", "Q4_K_S", "Q4_K_M", "Q5_K_S", "Q5_K_M", "Q6_K", "Q8_0", "F16", "F32"] }
        },
        "deviceTargets": { "type": "array", "items": { "type": "string" } }
      }
    },

    "EvalStage": {
      "type": "object",
      "required": ["type", "benchmarks"],
      "properties": {
        "type": { "const": "eval" },
        "benchmarks": {
          "type": "array",
          "items": {
            "type": "object",
            "required": ["name"],
            "properties": {
              "name": { "type": "string" },
              "subset": { "type": "string" },
              "nShot": { "type": "integer", "minimum": 0 },
              "submitToLeaderboard": { "type": "boolean", "default": false }
            }
          }
        },
        "passingThreshold": { "type": "number", "minimum": 0, "maximum": 100 },
        "compareToBase": { "type": "boolean", "default": true }
      }
    },

    "PublishStage": {
      "type": "object",
      "required": ["type", "org"],
      "properties": {
        "type": { "const": "publish" },
        "org": { "type": "string" },
        "repoNameTemplate": { "type": "string", "default": "{base}-{domain}-forged" },
        "includeAlloy": { "type": "boolean", "default": true },
        "cardFromBenchmarks": { "type": "boolean", "default": true },
        "tags": { "type": "array", "items": { "type": "string" } },
        "private": { "type": "boolean", "default": false }
      }
    },

    "ExpertPruneStage": {
      "type": "object",
      "required": ["type", "keepExperts"],
      "properties": {
        "type": { "const": "expert-prune" },
        "keepExperts": { "type": "integer", "minimum": 1 },
        "selectionStrategy": { "type": "string", "enum": ["activation", "gradient", "random"], "default": "activation" },
        "profileDataset": { "type": "string" },
        "profileSteps": { "type": "integer", "minimum": 1, "default": 100 }
      }
    },

    "ContextExtendStage": {
      "type": "object",
      "required": ["type", "targetLength", "method"],
      "properties": {
        "type": { "const": "context-extend" },
        "targetLength": { "type": "integer", "minimum": 1024 },
        "method": { "type": "string", "enum": ["yarn", "ntk", "linear", "dynamic-ntk"] },
        "trainingDataset": { "type": "string" },
        "trainingSteps": { "type": "integer", "minimum": 1 }
      }
    },

    "ModalityStage": {
      "type": "object",
      "required": ["type", "modality", "encoderModel"],
      "properties": {
        "type": { "const": "modality" },
        "modality": { "type": "string", "enum": ["vision", "audio", "multimodal"] },
        "encoderModel": { "type": "string" },
        "projectionArch": { "type": "string", "enum": ["mlp", "cross-attention", "linear"], "default": "mlp" },
        "freezeBase": { "type": "boolean", "default": true },
        "freezeEncoder": { "type": "boolean", "default": true },
        "trainingDataset": { "type": "string" },
        "trainingSteps": { "type": "integer", "minimum": 1 },
        "projectionDim": { "type": "integer" }
      }
    },

    "AlloyHardware": {
      "type": "object",
      "properties": {
        "minVramGb": { "type": "number", "minimum": 0 },
        "recommendedVramGb": { "type": "number", "minimum": 0 },
        "estimatedDurationMinutes": { "type": "number", "minimum": 0 },
        "supportsCPU": { "type": "boolean", "default": false },
        "testedOn": { "type": "array", "items": { "type": "string" } }
      }
    },

    "AlloyOutputs": {
      "type": "object",
      "properties": {
        "produces": {
          "type": "array",
          "items": {
            "type": "object",
            "required": ["type"],
            "properties": {
              "type": { "type": "string", "enum": ["safetensors", "gguf", "mlx", "lora-adapter", "model-card", "alloy"] },
              "description": { "type": "string" }
            }
          }
        }
      }
    }
  }
}