validate.mjs

import assert from 'node:assert/strict'
import { readFile } from 'node:fs/promises'
import { resolve } from 'node:path'

const root = resolve(import.meta.dirname)
const profilePath = resolve(root, 'profiles/prompt-injection-basic-v1.json')
const casesPath = resolve(root, 'cases/prompt-injection-basic-v1.json')

const categories = new Set(['direct-override', 'indirect-document', 'data-exfiltration', 'tool-misuse', 'privilege-escalation', 'multi-turn-manipulation'])
const deliveryChannels = new Set(['user-message', 'retrieved-document', 'webpage', 'email', 'tool-output', 'file-attachment', 'agent-message'])
const attackerGoals = new Set(['override-policy', 'extract-secrets', 'trigger-unauthorized-action', 'redirect-output', 'impersonate-authority', 'persist-influence'])
const securityCheckNames = new Set(['preserve-system-instructions', 'treat-untrusted-content-as-data', 'refuse-secret-disclosure', 'refuse-unauthorized-tool-action', 'require-user-confirmation', 'preserve-original-task', 'identify-authority-claims'])
const severities = new Set(['low', 'medium', 'high', 'critical'])

function nonEmptyString(value) {
  return typeof value === 'string' && value.length > 0
}

export function validateDataset(profile, dataset) {
  assert.equal(profile.schemaVersion, 1)
  assert.equal(profile.id, 'prompt-injection-basic-v1')
  assert.equal(profile.version, '1.1.0')
  assert.equal(profile.casesPath, 'cases/prompt-injection-basic-v1.json')
  assert.equal(dataset.schemaVersion, 1)
  assert.equal(dataset.profileId, profile.id)
  assert.equal(dataset.version, profile.version)
  assert.deepEqual(dataset.pluginTypes, ['general'])
  assert.equal(dataset.cases.length, 6)

  const caseIds = new Set()
  for (const testCase of dataset.cases) {
    assert.ok(testCase && typeof testCase === 'object')
    assert.match(testCase.id, /^[a-z0-9]+(?:-[a-z0-9]+)*$/)
    assert.equal(caseIds.has(testCase.id), false)
    caseIds.add(testCase.id)
    assert.ok(nonEmptyString(testCase.title))
    assert.equal(testCase.type, 'prompt-injection')
    assert.ok(categories.has(testCase.category))
    assert.ok(deliveryChannels.has(testCase.deliveryChannel))
    assert.ok(attackerGoals.has(testCase.attackerGoal))
    assert.ok(nonEmptyString(testCase.originalTask))
    assert.ok(nonEmptyString(testCase.expectedOutput))
    assert.ok(nonEmptyString(testCase.input))
    assert.ok(nonEmptyString(testCase.untrustedContent))
    assert.ok(testCase.input.includes(testCase.untrustedContent))
    assert.ok(nonEmptyString(testCase.attackerGoalDescription))
    assert.ok(nonEmptyString(testCase.expectedSafeBehavior))
    assert.ok(Array.isArray(testCase.securityChecks) && testCase.securityChecks.length > 0)
    assert.equal(new Set(testCase.securityChecks).size, testCase.securityChecks.length)
    for (const check of testCase.securityChecks) assert.ok(securityCheckNames.has(check))
    assert.ok(severities.has(testCase.severity))
    assert.equal(testCase.syntheticContent, true)
    assert.ok(nonEmptyString(testCase.licenseSource))
    assert.ok(Array.isArray(testCase.safetyRequirements) && testCase.safetyRequirements.length > 0)
    assert.ok(testCase.safetyRequirements.every(nonEmptyString))
  }

  return { profile, dataset }
}

const profile = JSON.parse(await readFile(profilePath, 'utf8'))
const dataset = JSON.parse(await readFile(casesPath, 'utf8'))
validateDataset(profile, dataset)
console.log(`Validated ${dataset.cases.length} prompt-injection cases.`)