Skip to content

Commit beeec4e

Browse files
BlueSkyXNBlueSkyXN
committed
fix(smoke): presign S3 requests behind private Space auth
1 parent 876a7c2 commit beeec4e

4 files changed

Lines changed: 169 additions & 26 deletions

File tree

‎hfs/nginx.conf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@ http {
5353
"" $proxy_host_header;
5454
}
5555

56+
map $http_authorization $s3_authorization {
57+
~^Bearer\s "";
58+
default $http_authorization;
59+
}
60+
5661
server {
5762
listen 7860;
5863
server_name _;
@@ -122,6 +127,7 @@ http {
122127
proxy_http_version 1.1;
123128

124129
proxy_set_header Host $proxy_host_header;
130+
proxy_set_header Authorization $s3_authorization;
125131
# The private-Space gateway consumes authentication and may inject
126132
# AWS tracing headers after the client has signed the request.
127133
# Strip only those platform headers before MinIO verifies SigV4.

‎scripts/presign-s3-request.py‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#!/usr/bin/env python3
2+
"""Create a short-lived SigV4 query URL for one S3 request."""
3+
4+
from __future__ import annotations
5+
6+
import argparse
7+
import datetime as dt
8+
import hashlib
9+
import hmac
10+
import os
11+
import urllib.parse
12+
13+
14+
def _sign(key: bytes, message: str) -> bytes:
15+
return hmac.new(key, message.encode(), hashlib.sha256).digest()
16+
17+
18+
def _encode(value: str) -> str:
19+
return urllib.parse.quote(value, safe="-_.~")
20+
21+
22+
def presign(method: str, url: str, region: str, expires: int, now: dt.datetime) -> str:
23+
access_key = os.environ.get("AWS_ACCESS_KEY_ID", "")
24+
secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "")
25+
if not access_key or not secret_key:
26+
raise ValueError("AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required")
27+
28+
parsed = urllib.parse.urlsplit(url)
29+
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
30+
raise ValueError("URL must be a credential-free HTTPS endpoint")
31+
if parsed.fragment:
32+
raise ValueError("URL must not contain a fragment")
33+
34+
timestamp = now.astimezone(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
35+
datestamp = timestamp[:8]
36+
scope = f"{datestamp}/{region}/s3/aws4_request"
37+
query = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)
38+
query.extend(
39+
[
40+
("X-Amz-Algorithm", "AWS4-HMAC-SHA256"),
41+
("X-Amz-Credential", f"{access_key}/{scope}"),
42+
("X-Amz-Date", timestamp),
43+
("X-Amz-Expires", str(expires)),
44+
("X-Amz-SignedHeaders", "host"),
45+
]
46+
)
47+
canonical_query = "&".join(
48+
f"{_encode(key)}={_encode(value)}" for key, value in sorted(query)
49+
)
50+
canonical_uri = urllib.parse.quote(parsed.path or "/", safe="/-_.~")
51+
canonical_request = "\n".join(
52+
[
53+
method.upper(),
54+
canonical_uri,
55+
canonical_query,
56+
f"host:{parsed.netloc.lower()}\n",
57+
"host",
58+
"UNSIGNED-PAYLOAD",
59+
]
60+
)
61+
string_to_sign = "\n".join(
62+
[
63+
"AWS4-HMAC-SHA256",
64+
timestamp,
65+
scope,
66+
hashlib.sha256(canonical_request.encode()).hexdigest(),
67+
]
68+
)
69+
date_key = _sign(("AWS4" + secret_key).encode(), datestamp)
70+
region_key = _sign(date_key, region)
71+
service_key = _sign(region_key, "s3")
72+
signing_key = _sign(service_key, "aws4_request")
73+
signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest()
74+
return urllib.parse.urlunsplit(
75+
(parsed.scheme, parsed.netloc, canonical_uri, f"{canonical_query}&X-Amz-Signature={signature}", "")
76+
)
77+
78+
79+
def main() -> int:
80+
parser = argparse.ArgumentParser()
81+
parser.add_argument("--method", required=True)
82+
parser.add_argument("--url", required=True)
83+
parser.add_argument("--region", default="us-east-1")
84+
parser.add_argument("--expires", type=int, default=300)
85+
args = parser.parse_args()
86+
if not 1 <= args.expires <= 900:
87+
parser.error("--expires must be between 1 and 900 seconds")
88+
try:
89+
print(
90+
presign(
91+
args.method,
92+
args.url,
93+
args.region,
94+
args.expires,
95+
dt.datetime.now(dt.timezone.utc),
96+
)
97+
)
98+
except ValueError as exc:
99+
parser.error(str(exc))
100+
return 0
101+
102+
103+
if __name__ == "__main__":
104+
raise SystemExit(main())

‎scripts/smoke-s3-curl.sh‎

Lines changed: 55 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ usage() {
55
cat <<'USAGE'
66
Usage: scripts/smoke-s3-curl.sh
77
8-
Run a credentialed S3 smoke test against the LibreFS HFS endpoint using only
9-
curl's built-in AWS SigV4 support. The script creates a temporary bucket,
8+
Run a credentialed S3 smoke test against the LibreFS HFS endpoint. Public
9+
targets use curl's header SigV4 support; a private HF target uses a short-lived
10+
query signature so the normal Bearer header remains available to the HF gateway.
11+
The script creates a temporary bucket,
1012
uploads one object, verifies signed and anonymous reads, applies a public-read
1113
policy, then removes the temporary resources.
1214
@@ -20,6 +22,7 @@ Optional environment:
2022
S3_SMOKE_BUCKET Default: generated temporary bucket name
2123
S3_SMOKE_OBJECT Default: smoke.txt
2224
S3_SMOKE_PAYLOAD Default: generated timestamp payload
25+
HF_GATEWAY_TOKEN Enables private-Space query signing and HF Bearer auth
2326
USAGE
2427
}
2528

@@ -36,6 +39,8 @@ SECRET_KEY="${MINIO_ROOT_PASSWORD:-${AWS_SECRET_ACCESS_KEY:-}}"
3639
BUCKET="${S3_SMOKE_BUCKET:-librefs-hfs-smoke-$(date -u +%Y%m%d%H%M%S)-$$}"
3740
OBJECT="${S3_SMOKE_OBJECT:-smoke.txt}"
3841
PAYLOAD="${S3_SMOKE_PAYLOAD:-librefs-hfs smoke $(date -u +%Y-%m-%dT%H:%M:%SZ)}"
42+
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
43+
PRESIGNER="$ROOT_DIR/scripts/presign-s3-request.py"
3944

4045
case "$ENDPOINT" in
4146
http://*|https://*) ;;
@@ -72,7 +77,7 @@ done
7277

7378
hf_gateway_args=()
7479
if [[ -n "${HF_GATEWAY_TOKEN:-}" ]]; then
75-
hf_gateway_args=(-H "X-HF-Authorization: Bearer ${HF_GATEWAY_TOKEN}")
80+
hf_gateway_args=(-H "Authorization: Bearer ${HF_GATEWAY_TOKEN}")
7681
fi
7782

7883
tmp_dir="$(mktemp -d)"
@@ -86,33 +91,59 @@ bucket_created=0
8691
object_uploaded=0
8792
policy_applied=0
8893

94+
presigned_url() {
95+
local method="$1"
96+
local url="$2"
97+
AWS_ACCESS_KEY_ID="$ACCESS_KEY" AWS_SECRET_ACCESS_KEY="$SECRET_KEY" \
98+
python3 "$PRESIGNER" --method "$method" --url "$url" --region "$REGION"
99+
}
100+
89101
signed_curl() {
90-
curl --silent --show-error --fail-with-body \
91-
"${hf_gateway_args[@]}" \
92-
--aws-sigv4 "aws:amz:${REGION}:s3" \
93-
--user "${ACCESS_KEY}:${SECRET_KEY}" \
94-
"$@"
102+
local method="$1"
103+
local url="$2"
104+
shift 2
105+
if [[ -n "${HF_GATEWAY_TOKEN:-}" ]]; then
106+
url="$(presigned_url "$method" "$url")"
107+
curl --silent --show-error --fail-with-body \
108+
"${hf_gateway_args[@]}" -X "$method" "$@" "$url"
109+
else
110+
curl --silent --show-error --fail-with-body \
111+
--aws-sigv4 "aws:amz:${REGION}:s3" \
112+
--user "${ACCESS_KEY}:${SECRET_KEY}" \
113+
-X "$method" "$@" "$url"
114+
fi
95115
}
96116

97117
signed_status() {
98-
curl --silent --show-error \
99-
"${hf_gateway_args[@]}" \
100-
--aws-sigv4 "aws:amz:${REGION}:s3" \
101-
--user "${ACCESS_KEY}:${SECRET_KEY}" \
102-
-o "$status_body" \
103-
-w '%{http_code}' \
104-
"$@" || true
118+
local method="$1"
119+
local url="$2"
120+
local request_args=()
121+
if [[ "$method" == HEAD ]]; then
122+
request_args=(--head)
123+
else
124+
request_args=(-X "$method")
125+
fi
126+
if [[ -n "${HF_GATEWAY_TOKEN:-}" ]]; then
127+
url="$(presigned_url "$method" "$url")"
128+
curl --silent --show-error "${hf_gateway_args[@]}" \
129+
"${request_args[@]}" -o "$status_body" -w '%{http_code}' "$url" || true
130+
else
131+
curl --silent --show-error \
132+
--aws-sigv4 "aws:amz:${REGION}:s3" \
133+
--user "${ACCESS_KEY}:${SECRET_KEY}" \
134+
"${request_args[@]}" -o "$status_body" -w '%{http_code}' "$url" || true
135+
fi
105136
}
106137

107138
cleanup() {
108139
if [[ "$policy_applied" -eq 1 ]]; then
109-
signed_curl -X DELETE "$ENDPOINT/$BUCKET?policy" >/dev/null 2>&1 || true
140+
signed_curl DELETE "$ENDPOINT/$BUCKET?policy" >/dev/null 2>&1 || true
110141
fi
111142
if [[ "$object_uploaded" -eq 1 ]]; then
112-
signed_curl -X DELETE "$ENDPOINT/$BUCKET/$OBJECT" >/dev/null 2>&1 || true
143+
signed_curl DELETE "$ENDPOINT/$BUCKET/$OBJECT" >/dev/null 2>&1 || true
113144
fi
114145
if [[ "$bucket_created" -eq 1 ]]; then
115-
signed_curl -X DELETE "$ENDPOINT/$BUCKET" >/dev/null 2>&1 || true
146+
signed_curl DELETE "$ENDPOINT/$BUCKET" >/dev/null 2>&1 || true
116147
fi
117148
rm -rf "$tmp_dir"
118149
}
@@ -138,25 +169,25 @@ EOF_POLICY
138169
trap cleanup EXIT
139170

140171
echo "==> list buckets"
141-
signed_curl "$ENDPOINT/" >/dev/null
172+
signed_curl GET "$ENDPOINT/" >/dev/null
142173

143174
echo "==> ensure temporary bucket does not already exist: $BUCKET"
144-
bucket_status="$(signed_status -I "$ENDPOINT/$BUCKET")"
175+
bucket_status="$(signed_status HEAD "$ENDPOINT/$BUCKET")"
145176
if [[ "$bucket_status" != "404" ]]; then
146177
echo "Refusing to use bucket $BUCKET because HEAD returned HTTP $bucket_status, expected 404." >&2
147178
exit 1
148179
fi
149180

150181
echo "==> create temporary bucket: $BUCKET"
151-
signed_curl -X PUT "$ENDPOINT/$BUCKET" >/dev/null
182+
signed_curl PUT "$ENDPOINT/$BUCKET" >/dev/null
152183
bucket_created=1
153184

154185
echo "==> upload object: $OBJECT"
155-
signed_curl -X PUT --data-binary @"$payload_file" "$ENDPOINT/$BUCKET/$OBJECT" >/dev/null
186+
signed_curl PUT "$ENDPOINT/$BUCKET/$OBJECT" --data-binary @"$payload_file" >/dev/null
156187
object_uploaded=1
157188

158189
echo "==> signed object download"
159-
signed_curl "$ENDPOINT/$BUCKET/$OBJECT" -o "$signed_download"
190+
signed_curl GET "$ENDPOINT/$BUCKET/$OBJECT" -o "$signed_download"
160191
if ! cmp -s "$payload_file" "$signed_download"; then
161192
echo "Signed download content does not match uploaded payload." >&2
162193
exit 1
@@ -170,7 +201,7 @@ if [[ "$private_status" != "403" ]]; then
170201
fi
171202

172203
echo "==> apply public-read bucket policy"
173-
signed_curl -X PUT -H 'Content-Type: application/json' --data-binary @"$policy_file" "$ENDPOINT/$BUCKET?policy" >/dev/null
204+
signed_curl PUT "$ENDPOINT/$BUCKET?policy" -H 'Content-Type: application/json' --data-binary @"$policy_file" >/dev/null
174205
policy_applied=1
175206

176207
echo "==> anonymous read succeeds after policy"

‎scripts/validate-contract.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ from pathlib import Path
7979
8080
for source_path in sys.argv[1:]:
8181
compile(Path(source_path).read_text(encoding="utf-8"), source_path, "exec")
82-
' hfs/ops_service.py hfs/admin_service.py
82+
' hfs/ops_service.py hfs/admin_service.py scripts/presign-s3-request.py
8383

8484
check "README front matter" require_pattern README.md '^sdk: docker$' 'README.md must keep sdk: docker'
8585
require_pattern README.md '^app_port: 7860$' 'README.md must keep app_port: 7860'
@@ -199,7 +199,8 @@ PY
199199
'
200200

201201
check "S3 smoke script contract" require_pattern scripts/smoke-s3-curl.sh '--aws-sigv4' 'S3 smoke test must use curl SigV4 support'
202-
require_pattern scripts/smoke-s3-curl.sh 'X-HF-Authorization: Bearer' 'private candidate S3 smoke must support separate HF gateway auth'
202+
require_pattern scripts/smoke-s3-curl.sh 'presign-s3-request\.py' 'private candidate S3 smoke must use query signing'
203+
require_pattern scripts/smoke-s3-curl.sh 'Authorization: Bearer' 'private candidate S3 smoke must authenticate to the HF gateway'
203204
require_pattern scripts/smoke-s3-curl.sh 'Refusing to use bucket' 'S3 smoke test must refuse existing buckets'
204205

205206
check "storage sampler script contract" test -f scripts/sample-hf-bucket-storage.sh
@@ -217,6 +218,7 @@ require_pattern hfs/nginx.conf 'proxy_pass http://127\.0\.0\.1:9001/;' 'Console
217218
require_pattern hfs/nginx.conf 'proxy_pass http://127\.0\.0\.1:9000;' 'S3 API must stay at the root path'
218219
require_pattern hfs/nginx.conf 'proxy_set_header X-HF-Authorization "";' 'S3 proxy must strip the private Space gateway header before SigV4 verification'
219220
require_pattern hfs/nginx.conf 'proxy_set_header X-Amzn-Trace-Id "";' 'S3 proxy must strip platform trace headers added after SigV4 signing'
221+
require_pattern hfs/nginx.conf 'proxy_set_header Authorization \$s3_authorization;' 'S3 proxy must strip only HF Bearer auth while preserving header SigV4'
220222
require_pattern hfs/nginx.conf 'proxy_hide_header X-Frame-Options;' 'Console proxy must hide upstream X-Frame-Options'
221223

222224
check "ops/admin service contract" require_pattern hfs/ops_service.py 'SECRET_KEYS' 'ops service must summarize secret presence only'

0 commit comments

Comments
 (0)