Skip to content

Commit 5f57613

Browse files
knoppiksknoppiks
committed
Add Integration Test for Validator Modes
Signed-off-by: Jonas Wagner <jwagner@knoppiks.de>
1 parent 0417f55 commit 5f57613

10 files changed

Lines changed: 470 additions & 68 deletions

File tree

‎.github/integration-test-extern-validator/check-reject-mode.sh‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
# Checks the `reject` failure mode of the external validator.
5+
#
6+
# In this mode a resource that fails external validation is not persisted.
7+
# Instead the interaction is rejected with `400 Bad Request` and an
8+
# OperationOutcome carrying the issues the external validator reported. This is
9+
# asserted for the create and the update interaction and for a transaction
10+
# bundle.
11+
#
12+
# The issue codes are the ones of the external validator (e.g. `invariant` or
13+
# `structure`) and depend on the concrete validation finding, so only the
14+
# severity is asserted here.
15+
#
16+
# A valid resource is used to show that conforming resources are still accepted
17+
# and persisted.
18+
19+
script_dir="$(dirname "$(readlink -f "$0")")"
20+
. "$script_dir/../scripts/util.sh"
21+
22+
base="http://localhost:8080/fhir"
23+
tag_system="https://blaze-server.org/fhir/CodeSystem/ValidationStatus"
24+
25+
request_body=$(mktemp)
26+
response_body=$(mktemp)
27+
28+
trap 'rm -f "$request_body" "$response_body"' EXIT
29+
30+
# Sends the JSON in $request_body to the URL $2 using the HTTP method $1, writes
31+
# the response body to $response_body and returns the response code.
32+
send() {
33+
curl -s -X"$1" -o "$response_body" -w '%{response_code}' \
34+
-H 'Accept: application/fhir+json' -H 'Content-Type: application/fhir+json' \
35+
-d @"$request_body" "$2"
36+
}
37+
38+
# Asserts that the response of the rejected interaction $1 is an
39+
# OperationOutcome with at least one issue of severity `error` or `fatal`.
40+
test_rejection_outcome() {
41+
test "resourceType of the $1 response" "$(jq -r '.resourceType' "$response_body")" "OperationOutcome"
42+
test_not_equal "number of error issues of the $1 response" \
43+
"$(jq '[.issue[] | select(.severity == "error" or .severity == "fatal")] | length' "$response_body")" "0"
44+
45+
echo "ℹ️ validation issues of the rejected $1:"
46+
jq -r '.issue[] | " [\(.severity)] \(.code): \(.diagnostics // .details.text // "")"' "$response_body"
47+
}
48+
49+
# --- a valid resource is accepted and persisted -----------------------------
50+
51+
jq '.id = "valid-patient"' "$script_dir/valid-patient.json" > "$request_body"
52+
53+
test "update status of the valid Patient" "$(send PUT "$base/Patient/valid-patient")" "201"
54+
test "read status of the valid Patient" \
55+
"$(curl -s -o /dev/null -w '%{response_code}' -H 'Accept: application/fhir+json' "$base/Patient/valid-patient")" "200"
56+
57+
# --- an invalid resource is rejected on create ------------------------------
58+
59+
cp "$script_dir/invalid-patient.json" "$request_body"
60+
61+
test "create status of the invalid Patient" "$(send POST "$base/Patient")" "400"
62+
test_rejection_outcome "create"
63+
64+
# --- an invalid resource is rejected on update ------------------------------
65+
66+
jq '.id = "invalid-patient"' "$script_dir/invalid-patient.json" > "$request_body"
67+
68+
test "update status of the invalid Patient" "$(send PUT "$base/Patient/invalid-patient")" "400"
69+
test_rejection_outcome "update"
70+
71+
# --- an invalid resource is rejected inside a transaction -------------------
72+
73+
bundle() {
74+
cat <<END
75+
{
76+
"resourceType": "Bundle",
77+
"type": "transaction",
78+
"entry": [
79+
{
80+
"resource": $(cat "$script_dir/invalid-patient.json"),
81+
"request": { "method": "POST", "url": "Patient" }
82+
}
83+
]
84+
}
85+
END
86+
}
87+
88+
bundle > "$request_body"
89+
90+
test "transaction status of the invalid Patient" "$(send POST "$base")" "400"
91+
test_rejection_outcome "transaction"
92+
93+
# --- no invalid resource was persisted --------------------------------------
94+
95+
test "number of persisted invalid Patients" \
96+
"$(curl -s -H 'Accept: application/fhir+json' "$base/Patient?_tag=${tag_system}|invalid&_summary=count" | jq -r '.total')" "0"

‎.github/integration-test-extern-validator/check-tag-mode.sh‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
# Checks the `tag-only` and `tag-outcome` failure modes of the external
5+
# validator on two known resources: one that conforms to the profile it claims
6+
# and one that does not.
7+
#
8+
# In both modes the invalid resource is persisted and carries a meta tag with
9+
# system `https://blaze-server.org/fhir/CodeSystem/ValidationStatus` and code
10+
# `invalid`. Only `tag-outcome` additionally stores the validator's
11+
# OperationOutcome as a contained resource referenced from a meta extension.
12+
#
13+
# The id of the contained OperationOutcome is derived from its content and is
14+
# deliberately opaque, so it is discovered by following the reference of the
15+
# meta extension instead of assuming a well-known id.
16+
#
17+
# The valid resource is used to show that conforming resources are stored
18+
# without any validation tag.
19+
#
20+
# Usage: check-tag-mode.sh <tag-outcome|tag-only>
21+
22+
script_dir="$(dirname "$(readlink -f "$0")")"
23+
. "$script_dir/../scripts/util.sh"
24+
25+
base="http://localhost:8080/fhir"
26+
tag_system="https://blaze-server.org/fhir/CodeSystem/ValidationStatus"
27+
outcome_ext="https://blaze-server.org/fhir/StructureDefinition/validation-outcome"
28+
29+
mode="$1"
30+
31+
request_body=$(mktemp)
32+
33+
trap 'rm -f "$request_body"' EXIT
34+
35+
# Stores the Patient of the JSON file $1 under the id $2 and returns its stored
36+
# representation.
37+
store_patient() {
38+
jq --arg id "$2" '.id = $id' "$1" > "$request_body"
39+
curl -sf -XPUT -H 'Accept: application/fhir+json' -H 'Content-Type: application/fhir+json' \
40+
-d @"$request_body" -o /dev/null "$base/Patient/$2"
41+
curl -sf -H 'Accept: application/fhir+json' "$base/Patient/$2"
42+
}
43+
44+
# --- a valid resource is stored without a validation tag --------------------
45+
46+
valid=$(store_patient "$script_dir/valid-patient.json" "valid-patient")
47+
48+
test "number of validation tags of the valid Patient" \
49+
"$(echo "$valid" | jq --arg sys "$tag_system" '[.meta.tag[]? | select(.system == $sys)] | length')" "0"
50+
test "number of validation outcome extensions of the valid Patient" \
51+
"$(echo "$valid" | jq --arg ext "$outcome_ext" '[.meta.extension[]? | select(.url == $ext)] | length')" "0"
52+
53+
# --- an invalid resource is stored with a validation tag --------------------
54+
55+
invalid=$(store_patient "$script_dir/invalid-patient.json" "invalid-patient")
56+
57+
test "number of invalid tags of the invalid Patient" \
58+
"$(echo "$invalid" | jq --arg sys "$tag_system" '[.meta.tag[]? | select(.system == $sys and .code == "invalid")] | length')" "1"
59+
60+
if [ "$mode" = "tag-only" ]; then
61+
test "number of validation outcome extensions of the invalid Patient" \
62+
"$(echo "$invalid" | jq --arg ext "$outcome_ext" '[.meta.extension[]? | select(.url == $ext)] | length')" "0"
63+
else
64+
outcome_id=$(echo "$invalid" | jq -r --arg ext "$outcome_ext" \
65+
'([.meta.extension[]? | select(.url == $ext) | .valueReference.reference] | first // "") | ltrimstr("#")')
66+
67+
test_non_empty "reference of the validation outcome extension of the invalid Patient" "$outcome_id"
68+
test_not_equal "number of error issues of the contained OperationOutcome of the invalid Patient" \
69+
"$(echo "$invalid" | jq --arg id "$outcome_id" '[.contained[]? | select(.resourceType == "OperationOutcome" and .id == $id) | .issue[]? | select(.severity == "error" or .severity == "fatal")] | length')" "0"
70+
fi

‎.github/integration-test-extern-validator/create-synthea-subset.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
# Creates a subset of the Synthea test data in the `synthea-subset` directory.
5+
# Validating every resource of the full Synthea data set against the external
6+
# validator would be too slow, so only three of the 121 patient bundles are
7+
# loaded: one patient bundle plus the hospital and practitioner information
8+
# bundles it references. Those two reference bundles make up most of the roughly
9+
# 840 resources of the subset.
10+
11+
script_dir="$(dirname "$(readlink -f "$0")")"
12+
synthea_dir="$script_dir/../test-data/synthea"
13+
target_dir="synthea-subset"
14+
15+
files=(
16+
"0-hospitalInformation1625911868739.json.bz2"
17+
"0-practitionerInformation1625911868739.json.bz2"
18+
"8a4c9c04-1524-9f1c-d65b-9b17e4520fef.json.bz2"
19+
)
20+
21+
mkdir -p "$target_dir"
22+
for file in "${files[@]}"; do
23+
cp "$synthea_dir/$file" "$target_dir/"
24+
done
25+
26+
echo "ℹ️ created Synthea subset with ${#files[@]} bundles in $target_dir"

‎.github/integration-test-extern-validator/docker-compose.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
services:
2+
data-server:
3+
image: "blaze:latest"
4+
environment:
5+
JAVA_TOOL_OPTIONS: "-Xmx2g"
6+
EXTERN_VALIDATOR_URL: "http://fhir-validator:8080"
7+
VALIDATOR_FAILURE_MODE: "${VALIDATOR_FAILURE_MODE:-tag-outcome}"
8+
LOG_LEVEL: debug
9+
ports:
10+
- "8080:8080"
11+
volumes:
12+
- "blaze-data:/app/data"
13+
healthcheck:
14+
test: [ "CMD", "wget", "--spider", "http://localhost:8080/health" ]
15+
interval: 10s
16+
timeout: 5s
17+
retries: 5
18+
start_period: 30s
19+
depends_on:
20+
fhir-validator:
21+
condition: service_healthy
22+
23+
terminology-server:
24+
image: "blaze:latest"
25+
environment:
26+
JAVA_TOOL_OPTIONS: "-Xmx4g"
27+
ENABLE_TERMINOLOGY_SERVICE: "true"
28+
ENABLE_TERMINOLOGY_LOINC: "true"
29+
LOG_LEVEL: debug
30+
ports:
31+
- "8082:8080"
32+
volumes:
33+
- "blaze-terminology-data:/app/data"
34+
healthcheck:
35+
test: [ "CMD", "wget", "--spider", "http://localhost:8080/health" ]
36+
interval: 10s
37+
timeout: 5s
38+
retries: 5
39+
start_period: 30s
40+
41+
fhir-validator:
42+
image: "ghcr.io/medizininformatik-initiative/mii-fhir-validator:0.0.1-alpha.7@sha256:c237b8b36a641cc57a59be2d2a29035ca4f39e0b6c71b26ab4e88e7f7c4fd6a1"
43+
environment:
44+
JAVA_OPTS: "-Xmx2g"
45+
TX_SERVER: "http://terminology-server:8080/fhir"
46+
# Override the MII IG defaults of the image with US Core, the profiles the
47+
# Synthea test data is generated against.
48+
IG_PARAMS: "-ig hl7.fhir.us.core#9.0.0"
49+
ports:
50+
- "8084:8080"
51+
# The validator only starts serving after it has loaded all implementation
52+
# guides, which takes several minutes. Without this healthcheck the data
53+
# server would start accepting writes while the validator is still starting
54+
# up and reject them with `503 Service Unavailable`.
55+
healthcheck:
56+
test: [ "CMD", "curl", "-sf", "-X", "POST", "-H", "Content-Type: application/fhir+json",
57+
"-d", "{\"resourceType\":\"Patient\"}", "http://localhost:8080/validateResource" ]
58+
interval: 10s
59+
timeout: 10s
60+
retries: 30
61+
start_period: 300s
62+
depends_on:
63+
terminology-server:
64+
condition: service_healthy
65+
66+
volumes:
67+
blaze-data:
68+
blaze-terminology-data:

‎.github/integration-test-extern-validator/invalid-patient.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"resourceType": "Patient",
3+
"meta": {
4+
"profile": [
5+
"http://hl7.org/fhir/us/core/StructureDefinition/us-core-patient"
6+
]
7+
},
8+
"gender": "unknown"
9+
}

‎.github/integration-test-extern-validator/valid-patient.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"resourceType": "Patient",
3+
"meta": {
4+
"profile": [
5+
"http://hl7.org/fhir/us/core/StructureDefinition/us-core-patient"
6+
]
7+
},
8+
"identifier": [
9+
{
10+
"system": "http://hl7.org/fhir/sid/us-ssn",
11+
"value": "999-99-9999"
12+
}
13+
],
14+
"name": [
15+
{
16+
"family": "Tester",
17+
"given": [
18+
"Valid"
19+
]
20+
}
21+
],
22+
"gender": "female"
23+
}

‎.github/integration-test-kds/check-invalid-resources.sh‎

Lines changed: 0 additions & 66 deletions
This file was deleted.

‎.github/integration-test-kds/docker-compose.yml‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ services:
1818
start_period: 30s
1919
depends_on:
2020
fhir-validator:
21-
condition: service_started
21+
condition: service_healthy
2222

2323
terminology-server:
2424
image: "blaze:latest"
@@ -53,6 +53,17 @@ services:
5353
- "8084:8080"
5454
volumes:
5555
- ./fhir-settings.json:/app/fhir-settings.json:ro
56+
# The validator only starts serving after it has loaded all implementation
57+
# guides, which takes several minutes. Without this healthcheck the data
58+
# server would start accepting writes while the validator is still starting
59+
# up and reject them with `503 Service Unavailable`.
60+
healthcheck:
61+
test: [ "CMD", "curl", "-sf", "-X", "POST", "-H", "Content-Type: application/fhir+json",
62+
"-d", "{\"resourceType\":\"Patient\"}", "http://localhost:8080/validateResource" ]
63+
interval: 10s
64+
timeout: 10s
65+
retries: 30
66+
start_period: 300s
5667
depends_on:
5768
terminology-server:
5869
condition: service_healthy

0 commit comments

Comments
 (0)