Skip to content

Commit 52957e6

Browse files
kusssalkusssal
authored
Merge pull request #4177 from tianpeng-dev/fix/metadata-validation-ssrf-targets
fix(metadata-validation): block private fetch targets
2 parents b599117 + 563a3e6 commit 52957e6

3 files changed

Lines changed: 230 additions & 9 deletions

File tree

‎govtool/metadata-validation/src/app.service.test.ts‎

Lines changed: 98 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,19 @@ import { Test, TestingModule } from '@nestjs/testing';
22
import { HttpService } from '@nestjs/axios';
33
import { of, throwError } from 'rxjs';
44
import * as blake from 'blakejs';
5+
import { lookup } from 'node:dns/promises';
56

67
import { AppService } from './app.service';
78
import { ValidateMetadataDTO } from '@dto';
89
import { MetadataValidationStatus } from '@enums';
910
import { MetadataStandard } from '@types';
10-
import { validateMetadataStandard, parseMetadata } from '@utils';
11+
import { validateMetadataStandard, parseMetadata, getStandard } from '@utils';
1112
import { AxiosResponse, AxiosRequestHeaders } from 'axios';
1213

1314
jest.mock('@utils');
15+
jest.mock('node:dns/promises', () => ({
16+
lookup: jest.fn(),
17+
}));
1418

1519
describe('AppService', () => {
1620
let service: AppService;
@@ -31,19 +35,20 @@ describe('AppService', () => {
3135

3236
service = module.get<AppService>(AppService);
3337
httpService = module.get<HttpService>(HttpService);
38+
(lookup as jest.Mock).mockResolvedValue([{ address: '93.184.216.34' }]);
3439
});
3540

3641
it('should validate metadata correctly', async () => {
3742
const url = 'http://example.com';
3843
const hash = 'correctHash';
3944
const validateMetadataDTO: ValidateMetadataDTO = { hash, url };
40-
const data = {
45+
const body = {
4146
body: 'testBody',
4247
headers: {},
4348
};
4449
const parsedMetadata = { parsed: 'metadata' };
4550
const response: AxiosResponse = {
46-
data,
51+
data: JSON.stringify(body),
4752
status: 200,
4853
statusText: 'OK',
4954
headers: {},
@@ -53,6 +58,7 @@ describe('AppService', () => {
5358
},
5459
};
5560
jest.spyOn(httpService, 'get').mockReturnValueOnce(of(response));
61+
(getStandard as jest.Mock).mockReturnValueOnce(MetadataStandard.CIP108);
5662
(validateMetadataStandard as jest.Mock).mockResolvedValueOnce(undefined);
5763
(parseMetadata as jest.Mock).mockReturnValueOnce(parsedMetadata);
5864
jest.spyOn(blake, 'blake2bHex').mockReturnValueOnce(hash);
@@ -65,12 +71,16 @@ describe('AppService', () => {
6571
metadata: parsedMetadata,
6672
});
6773
expect(validateMetadataStandard).toHaveBeenCalledWith(
68-
data,
74+
body.body,
6975
MetadataStandard.CIP108,
7076
);
71-
expect(parseMetadata).toHaveBeenCalledWith(
72-
data.body,
73-
MetadataStandard.CIP108,
77+
expect(parseMetadata).toHaveBeenCalledWith(body.body);
78+
expect(httpService.get).toHaveBeenCalledWith(
79+
url,
80+
expect.objectContaining({
81+
httpAgent: expect.any(Object),
82+
httpsAgent: expect.any(Object),
83+
}),
7484
);
7585
});
7686

@@ -98,13 +108,13 @@ describe('AppService', () => {
98108
const url = 'http://example.com';
99109
const hash = 'incorrectHash';
100110
const validateMetadataDTO: ValidateMetadataDTO = { hash, url };
101-
const data = {
111+
const body = {
102112
body: 'testBody',
103113
};
104114
const parsedMetadata = { parsed: 'metadata' };
105115

106116
const response: AxiosResponse = {
107-
data,
117+
data: JSON.stringify(body),
108118
status: 200,
109119
statusText: 'OK',
110120
headers: {},
@@ -114,6 +124,7 @@ describe('AppService', () => {
114124
},
115125
};
116126
jest.spyOn(httpService, 'get').mockReturnValueOnce(of(response));
127+
(getStandard as jest.Mock).mockReturnValueOnce(MetadataStandard.CIP108);
117128
(validateMetadataStandard as jest.Mock).mockResolvedValueOnce(undefined);
118129
(parseMetadata as jest.Mock).mockReturnValueOnce(parsedMetadata);
119130
jest.spyOn(blake, 'blake2bHex').mockReturnValueOnce('differentHash');
@@ -126,4 +137,82 @@ describe('AppService', () => {
126137
metadata: parsedMetadata,
127138
});
128139
});
140+
141+
it('should block loopback metadata URLs before fetching', async () => {
142+
const validateMetadataDTO: ValidateMetadataDTO = {
143+
hash: 'hash',
144+
url: 'http://127.0.0.1:3000/api',
145+
};
146+
147+
const result = await service.validateMetadata(validateMetadataDTO);
148+
149+
expect(result).toEqual({
150+
status: MetadataValidationStatus.URL_BLOCKED,
151+
valid: false,
152+
metadata: undefined,
153+
});
154+
expect(httpService.get).not.toHaveBeenCalled();
155+
});
156+
157+
it('should block hostnames that resolve to private addresses', async () => {
158+
(lookup as jest.Mock).mockResolvedValueOnce([{ address: '10.0.0.5' }]);
159+
160+
const validateMetadataDTO: ValidateMetadataDTO = {
161+
hash: 'hash',
162+
url: 'https://metadata.internal.example/metadata.json',
163+
};
164+
165+
const result = await service.validateMetadata(validateMetadataDTO);
166+
167+
expect(result).toEqual({
168+
status: MetadataValidationStatus.URL_BLOCKED,
169+
valid: false,
170+
metadata: undefined,
171+
});
172+
expect(httpService.get).not.toHaveBeenCalled();
173+
});
174+
175+
it('should block private addresses during the HTTP agent lookup', async () => {
176+
const url = 'http://example.com';
177+
const hash = 'correctHash';
178+
const body = {
179+
body: 'testBody',
180+
};
181+
const response: AxiosResponse = {
182+
data: JSON.stringify(body),
183+
status: 200,
184+
statusText: 'OK',
185+
headers: {},
186+
config: {
187+
headers: {} as AxiosRequestHeaders,
188+
url,
189+
},
190+
};
191+
jest.spyOn(httpService, 'get').mockReturnValueOnce(of(response));
192+
(getStandard as jest.Mock).mockReturnValueOnce(MetadataStandard.CIP108);
193+
(validateMetadataStandard as jest.Mock).mockResolvedValueOnce(undefined);
194+
jest.spyOn(blake, 'blake2bHex').mockReturnValueOnce(hash);
195+
196+
await service.validateMetadata({ hash, url });
197+
198+
const requestConfig = (httpService.get as jest.Mock).mock.calls[0][1];
199+
const agentLookup = requestConfig.httpAgent.options.lookup;
200+
(lookup as jest.Mock).mockResolvedValueOnce({
201+
address: '127.0.0.1',
202+
family: 4,
203+
});
204+
205+
await expect(
206+
new Promise((resolve, reject) => {
207+
agentLookup('example.com', {}, (error: Error | null) => {
208+
if (error) {
209+
reject(error);
210+
return;
211+
}
212+
213+
resolve(undefined);
214+
});
215+
}),
216+
).rejects.toThrow(MetadataValidationStatus.URL_BLOCKED);
217+
});
129218
});

‎govtool/metadata-validation/src/app.service.ts‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ import { Injectable, Logger } from '@nestjs/common';
22
import { catchError, finalize, firstValueFrom } from 'rxjs';
33
import { HttpService } from '@nestjs/axios';
44
import * as blake from 'blakejs';
5+
import { lookup } from 'node:dns/promises';
6+
import { isIP, LookupFunction } from 'node:net';
7+
import { Agent as HttpAgent } from 'node:http';
8+
import { Agent as HttpsAgent } from 'node:https';
59

610
import { ValidateMetadataDTO } from '@dto';
711
import { LoggerMessage, MetadataValidationStatus } from '@enums';
@@ -12,6 +16,129 @@ import { /* MetadataStandard, */ ValidateMetadataResult } from '@types';
1216
export class AppService {
1317
constructor(private readonly httpService: HttpService) {}
1418

19+
private readonly safeHttpAgent = new HttpAgent({
20+
lookup: this.createSafeLookup(),
21+
});
22+
23+
private readonly safeHttpsAgent = new HttpsAgent({
24+
lookup: this.createSafeLookup(),
25+
});
26+
27+
private isBlockedIPv4(address: string): boolean {
28+
const parts = address.split('.').map(Number);
29+
const [first, second] = parts;
30+
31+
return (
32+
first === 0 ||
33+
first === 10 ||
34+
first === 127 ||
35+
(first === 100 && second >= 64 && second <= 127) ||
36+
(first === 169 && second === 254) ||
37+
(first === 172 && second >= 16 && second <= 31) ||
38+
(first === 192 && second === 0 && parts[2] === 0) ||
39+
(first === 192 && second === 168) ||
40+
(first === 198 && (second === 18 || second === 19)) ||
41+
first >= 224
42+
);
43+
}
44+
45+
private isBlockedIPv6(address: string): boolean {
46+
const normalized = address.toLowerCase();
47+
const ipv4MappedPrefix = '::ffff:';
48+
49+
if (normalized.startsWith(ipv4MappedPrefix)) {
50+
const mappedAddress = normalized.slice(ipv4MappedPrefix.length);
51+
if (isIP(mappedAddress) === 4) {
52+
return this.isBlockedIPv4(mappedAddress);
53+
}
54+
}
55+
56+
return (
57+
normalized === '::' ||
58+
normalized === '::1' ||
59+
normalized.startsWith('fc') ||
60+
normalized.startsWith('fd') ||
61+
normalized.startsWith('fe80:') ||
62+
normalized.startsWith('::ffff:0:')
63+
);
64+
}
65+
66+
private isBlockedAddress(address: string): boolean {
67+
const version = isIP(address);
68+
69+
if (version === 4) {
70+
return this.isBlockedIPv4(address);
71+
}
72+
73+
if (version === 6) {
74+
return this.isBlockedIPv6(address);
75+
}
76+
77+
return false;
78+
}
79+
80+
private async assertAllowedMetadataUrl(url: string): Promise<void> {
81+
let parsedUrl: URL;
82+
83+
try {
84+
parsedUrl = new URL(url);
85+
} catch (error) {
86+
throw MetadataValidationStatus.URL_NOT_FOUND;
87+
}
88+
89+
if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
90+
throw MetadataValidationStatus.URL_BLOCKED;
91+
}
92+
93+
const hostname = parsedUrl.hostname.toLowerCase();
94+
if (hostname === 'localhost' || hostname.endsWith('.localhost')) {
95+
throw MetadataValidationStatus.URL_BLOCKED;
96+
}
97+
98+
if (this.isBlockedAddress(hostname)) {
99+
throw MetadataValidationStatus.URL_BLOCKED;
100+
}
101+
102+
let resolvedAddresses: { address: string }[];
103+
try {
104+
resolvedAddresses = await lookup(hostname, { all: true, verbatim: true });
105+
} catch (error) {
106+
throw MetadataValidationStatus.URL_NOT_FOUND;
107+
}
108+
109+
if (
110+
resolvedAddresses.some(({ address }) => this.isBlockedAddress(address))
111+
) {
112+
throw MetadataValidationStatus.URL_BLOCKED;
113+
}
114+
}
115+
116+
private createSafeLookup(): LookupFunction {
117+
return (hostname, options, callback) => {
118+
lookup(hostname, options)
119+
.then((result) => {
120+
const addresses = Array.isArray(result) ? result : [result];
121+
122+
if (
123+
addresses.some(({ address }) => this.isBlockedAddress(address))
124+
) {
125+
callback(new Error(MetadataValidationStatus.URL_BLOCKED), '', 0);
126+
return;
127+
}
128+
129+
if (Array.isArray(result)) {
130+
callback(null, result);
131+
return;
132+
}
133+
134+
callback(null, result.address, result.family);
135+
})
136+
.catch((error) => {
137+
callback(error as Error, '', 0);
138+
});
139+
};
140+
}
141+
15142
async validateMetadata({
16143
hash,
17144
url,
@@ -27,9 +154,13 @@ export class AppService {
27154
}
28155

29156
try {
157+
await this.assertAllowedMetadataUrl(url);
158+
30159
const { data: rawData } = await firstValueFrom(
31160
this.httpService
32161
.get(url, {
162+
httpAgent: this.safeHttpAgent,
163+
httpsAgent: this.safeHttpsAgent,
33164
headers: {
34165
// Required to not being blocked by APIs that require a User-Agent
35166
'User-Agent': 'GovTool/Metadata-Validation-Tool',

‎govtool/metadata-validation/src/enums/ValidationError.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
export enum MetadataValidationStatus {
2+
URL_BLOCKED = 'URL_BLOCKED',
23
URL_NOT_FOUND = 'URL_NOT_FOUND',
34
INVALID_JSONLD = 'INVALID_JSONLD',
45
INVALID_HASH = 'INVALID_HASH',

0 commit comments

Comments
 (0)