Skip to content

Commit 2fa0d11

Browse files
stijnpotters1stijnpotters1
committed
implemented tests for new zip handling logic
1 parent a1008d5 commit 2fa0d11

3 files changed

Lines changed: 564 additions & 539 deletions

File tree

‎src/main/java/org/frankframework/insights/release/ReleaseArtifactService.java‎

Lines changed: 161 additions & 160 deletions
Original file line numberDiff line numberDiff line change
@@ -22,164 +22,165 @@
2222
@Slf4j
2323
public class ReleaseArtifactService {
2424

25-
private static final Path ARCHIVE_DIR = Paths.get("release-archive");
26-
private static final String GITHUB_ZIP_URL_FORMAT =
27-
"https://github.com/frankframework/frankframework/archive/refs/tags/%s.zip";
28-
29-
private static final int BUFFER_SIZE = 4096;
30-
private static final long MAX_ARCHIVE_SIZE = 1024L * 1024 * 1024;
31-
private static final int MAX_ENTRIES = 1024;
32-
private static final double COMPRESSION_RATIO_LIMIT = 10.0;
33-
34-
@Transactional
35-
public Path prepareReleaseArtifacts(Release release) throws IOException {
36-
Path releaseDir = ARCHIVE_DIR.resolve(release.getName());
37-
38-
if (releaseDirectoryExists(releaseDir, release)) {
39-
return releaseDir;
40-
}
41-
42-
String zipUrl = buildZipUrl(release);
43-
Path zipFile = downloadReleaseZip(releaseDir, zipUrl, release);
44-
unpackAndCleanup(zipFile, releaseDir);
45-
46-
return releaseDir;
47-
}
48-
49-
private boolean releaseDirectoryExists(Path releaseDir, Release release) throws IOException {
50-
if (Files.isDirectory(releaseDir)) {
51-
try (Stream<Path> stream = Files.list(releaseDir)) {
52-
if (stream.findFirst().isPresent()) {
53-
log.info("Source code for release {} already exists, skipping download.", release.getName());
54-
return true;
55-
}
56-
}
57-
}
58-
return false;
59-
}
60-
61-
private String buildZipUrl(Release release) throws IOException {
62-
String tagName = release.getTagName();
63-
if (tagName == null || tagName.isBlank()) {
64-
throw new IOException("Release " + release.getName() + " is missing a tagName.");
65-
}
66-
return String.format(GITHUB_ZIP_URL_FORMAT, tagName);
67-
}
68-
69-
private Path downloadReleaseZip(Path releaseDir, String zipUrl, Release release) throws IOException {
70-
log.info("Downloading source for {} from {}", release.getName(), zipUrl);
71-
Files.createDirectories(releaseDir);
72-
Path zipFile = releaseDir.resolve(release.getName() + ".zip");
73-
74-
try (InputStream in = URI.create(zipUrl).toURL().openStream()) {
75-
Files.copy(in, zipFile, StandardCopyOption.REPLACE_EXISTING);
76-
}
77-
return zipFile;
78-
}
79-
80-
private void unpackAndCleanup(Path zipFile, Path releaseDir) throws IOException {
81-
log.debug("Unpacking archive for {}", zipFile.getFileName());
82-
unzip(zipFile, releaseDir);
83-
Files.delete(zipFile);
84-
log.debug("Successfully downloaded and unpacked source for {}", zipFile.getFileName());
85-
}
86-
87-
/**
88-
* Securely unzips a file using the java.nio.file.FileSystem API for robust traversal.
89-
* This approach is clearer for static analysis tools and avoids manual entry iteration.
90-
*/
91-
private void unzip(Path zipFile, Path destDir) throws IOException {
92-
Path normalizedDestDir = destDir.toAbsolutePath().normalize();
93-
AtomicInteger entryCount = new AtomicInteger(0);
94-
AtomicLong totalUncompressedSize = new AtomicLong(0);
95-
96-
try (FileSystem zipFs = FileSystems.newFileSystem(zipFile, (ClassLoader) null);
97-
ZipFile zf = new ZipFile(zipFile.toFile())) {
98-
99-
Path root = zipFs.getPath("/");
100-
try (Stream<Path> stream = Files.walk(root)) {
101-
stream.forEach(path -> {
102-
try {
103-
if (entryCount.incrementAndGet() > MAX_ENTRIES) {
104-
throw new IOException("Archive contains too many entries.");
105-
}
106-
processPath(path, normalizedDestDir, totalUncompressedSize, zf);
107-
} catch (IOException e) {
108-
throw new RuntimeException(e); // Will be caught by the outer catch block
109-
}
110-
});
111-
}
112-
} catch (RuntimeException e) {
113-
// Un-wrap the IOException from the lambda
114-
if (e.getCause() instanceof IOException) {
115-
throw (IOException) e.getCause();
116-
}
117-
throw e;
118-
}
119-
}
120-
121-
/**
122-
* Processes a single path from the zip file system, handling directories and files.
123-
*/
124-
private void processPath(Path pathInZip, Path destDir, AtomicLong totalUncompressedSize, ZipFile zf) throws IOException {
125-
Path validatedPath = validateAndStripPath(pathInZip, destDir);
126-
if (validatedPath == null) {
127-
return; // Skip top-level directory
128-
}
129-
130-
if (Files.isDirectory(pathInZip)) {
131-
Files.createDirectories(validatedPath);
132-
} else {
133-
extractAndValidateFile(pathInZip, validatedPath, totalUncompressedSize, zf);
134-
}
135-
}
136-
137-
/**
138-
* Validates and prepares the destination path, preventing path traversal.
139-
*/
140-
private Path validateAndStripPath(Path pathInZip, Path destDir) throws IOException {
141-
if (pathInZip.getNameCount() <= 1) {
142-
return null; // Ignore the root and GitHub's top-level directory
143-
}
144-
Path strippedPath = pathInZip.subpath(1, pathInZip.getNameCount());
145-
Path resolvedPath = destDir.resolve(strippedPath.toString()).normalize();
146-
if (!resolvedPath.startsWith(destDir)) {
147-
throw new IOException("Bad zip entry: " + pathInZip + " (Path Traversal attempt)");
148-
}
149-
return resolvedPath;
150-
}
151-
152-
/**
153-
* Extracts a file while validating its size and compression ratio.
154-
*/
155-
private void extractAndValidateFile(Path pathInZip, Path destFile, AtomicLong totalUncompressedSize, ZipFile zf) throws IOException {
156-
Files.createDirectories(destFile.getParent());
157-
ZipEntry entry = zf.getEntry(pathInZip.toString().substring(1)); // ZipFile needs entry name without leading '/'
158-
if (entry == null) {
159-
throw new IOException("Could not find ZipEntry for path: " + pathInZip);
160-
}
161-
162-
long totalEntrySize = 0;
163-
byte[] buffer = new byte[BUFFER_SIZE];
164-
try (InputStream in = zf.getInputStream(entry);
165-
var out = Files.newOutputStream(destFile)) {
166-
int nBytes;
167-
while ((nBytes = in.read(buffer)) > 0) {
168-
out.write(buffer, 0, nBytes);
169-
totalEntrySize += nBytes;
170-
171-
if (totalUncompressedSize.addAndGet(nBytes) > MAX_ARCHIVE_SIZE) {
172-
throw new IOException("Archive is too large when uncompressed.");
173-
}
174-
175-
if (entry.getCompressedSize() > 0) {
176-
double ratio = (double) totalEntrySize / entry.getCompressedSize();
177-
if (ratio > COMPRESSION_RATIO_LIMIT) {
178-
throw new IOException("Compression ratio for entry " + entry.getName() + " is too high.");
179-
}
180-
}
181-
}
182-
}
183-
}
25+
private static final Path ARCHIVE_DIR = Paths.get("release-archive");
26+
private static final String GITHUB_ZIP_URL_FORMAT =
27+
"https://github.com/frankframework/frankframework/archive/refs/tags/%s.zip";
28+
29+
private static final int BUFFER_SIZE = 4096;
30+
private static final long MAX_ARCHIVE_SIZE = 1024L * 1024 * 1024;
31+
private static final int MAX_ENTRIES = 1024;
32+
private static final double COMPRESSION_RATIO_LIMIT = 10.0;
33+
34+
@Transactional
35+
public Path prepareReleaseArtifacts(Release release) throws IOException {
36+
Path releaseDir = ARCHIVE_DIR.resolve(release.getName());
37+
38+
if (releaseDirectoryExists(releaseDir, release)) {
39+
return releaseDir;
40+
}
41+
42+
String zipUrl = buildZipUrl(release);
43+
Path zipFile = downloadReleaseZip(releaseDir, zipUrl, release);
44+
unpackAndCleanup(zipFile, releaseDir);
45+
46+
return releaseDir;
47+
}
48+
49+
private boolean releaseDirectoryExists(Path releaseDir, Release release) throws IOException {
50+
if (Files.isDirectory(releaseDir)) {
51+
try (Stream<Path> stream = Files.list(releaseDir)) {
52+
if (stream.findFirst().isPresent()) {
53+
log.info("Source code for release {} already exists, skipping download.", release.getName());
54+
return true;
55+
}
56+
}
57+
}
58+
return false;
59+
}
60+
61+
private String buildZipUrl(Release release) throws IOException {
62+
String tagName = release.getTagName();
63+
if (tagName == null || tagName.isBlank()) {
64+
throw new IOException("Release " + release.getName() + " is missing a tagName.");
65+
}
66+
return String.format(GITHUB_ZIP_URL_FORMAT, tagName);
67+
}
68+
69+
private Path downloadReleaseZip(Path releaseDir, String zipUrl, Release release) throws IOException {
70+
log.info("Downloading source for {} from {}", release.getName(), zipUrl);
71+
Files.createDirectories(releaseDir);
72+
Path zipFile = releaseDir.resolve(release.getName() + ".zip");
73+
74+
try (InputStream in = URI.create(zipUrl).toURL().openStream()) {
75+
Files.copy(in, zipFile, StandardCopyOption.REPLACE_EXISTING);
76+
}
77+
return zipFile;
78+
}
79+
80+
private void unpackAndCleanup(Path zipFile, Path releaseDir) throws IOException {
81+
log.debug("Unpacking archive for {}", zipFile.getFileName());
82+
unzip(zipFile, releaseDir);
83+
Files.delete(zipFile);
84+
log.debug("Successfully downloaded and unpacked source for {}", zipFile.getFileName());
85+
}
86+
87+
/**
88+
* Securely unzips a file using the java.nio.file.FileSystem API for robust traversal.
89+
* This approach is clearer for static analysis tools and avoids manual entry iteration.
90+
*/
91+
private void unzip(Path zipFile, Path destDir) throws IOException {
92+
Path normalizedDestDir = destDir.toAbsolutePath().normalize();
93+
AtomicInteger entryCount = new AtomicInteger(0);
94+
AtomicLong totalUncompressedSize = new AtomicLong(0);
95+
96+
try (FileSystem zipFs = FileSystems.newFileSystem(zipFile, (ClassLoader) null);
97+
ZipFile zf = new ZipFile(zipFile.toFile())) {
98+
99+
Path root = zipFs.getPath("/");
100+
try (Stream<Path> stream = Files.walk(root)) {
101+
stream.forEach(path -> {
102+
try {
103+
if (entryCount.incrementAndGet() > MAX_ENTRIES) {
104+
throw new IOException("Archive contains too many entries.");
105+
}
106+
processPath(path, normalizedDestDir, totalUncompressedSize, zf);
107+
} catch (IOException e) {
108+
throw new RuntimeException(e); // Will be caught by the outer catch block
109+
}
110+
});
111+
}
112+
} catch (RuntimeException e) {
113+
// Un-wrap the IOException from the lambda
114+
if (e.getCause() instanceof IOException) {
115+
throw (IOException) e.getCause();
116+
}
117+
throw e;
118+
}
119+
}
120+
121+
/**
122+
* Processes a single path from the zip file system, handling directories and files.
123+
*/
124+
private void processPath(Path pathInZip, Path destDir, AtomicLong totalUncompressedSize, ZipFile zf)
125+
throws IOException {
126+
Path validatedPath = validateAndStripPath(pathInZip, destDir);
127+
if (validatedPath == null) {
128+
return; // Skip top-level directory
129+
}
130+
131+
if (Files.isDirectory(pathInZip)) {
132+
Files.createDirectories(validatedPath);
133+
} else {
134+
extractAndValidateFile(pathInZip, validatedPath, totalUncompressedSize, zf);
135+
}
136+
}
137+
138+
/**
139+
* Validates and prepares the destination path, preventing path traversal.
140+
*/
141+
private Path validateAndStripPath(Path pathInZip, Path destDir) throws IOException {
142+
if (pathInZip.getNameCount() <= 1) {
143+
return null; // Ignore the root and GitHub's top-level directory
144+
}
145+
Path strippedPath = pathInZip.subpath(1, pathInZip.getNameCount());
146+
Path resolvedPath = destDir.resolve(strippedPath.toString()).normalize();
147+
if (!resolvedPath.startsWith(destDir)) {
148+
throw new IOException("Bad zip entry: " + pathInZip + " (Path Traversal attempt)");
149+
}
150+
return resolvedPath;
151+
}
152+
153+
/**
154+
* Extracts a file while validating its size and compression ratio.
155+
*/
156+
private void extractAndValidateFile(Path pathInZip, Path destFile, AtomicLong totalUncompressedSize, ZipFile zf)
157+
throws IOException {
158+
Files.createDirectories(destFile.getParent());
159+
ZipEntry entry = zf.getEntry(pathInZip.toString().substring(1)); // ZipFile needs entry name without leading '/'
160+
if (entry == null) {
161+
throw new IOException("Could not find ZipEntry for path: " + pathInZip);
162+
}
163+
164+
long totalEntrySize = 0;
165+
byte[] buffer = new byte[BUFFER_SIZE];
166+
try (InputStream in = zf.getInputStream(entry);
167+
var out = Files.newOutputStream(destFile)) {
168+
int nBytes;
169+
while ((nBytes = in.read(buffer)) > 0) {
170+
out.write(buffer, 0, nBytes);
171+
totalEntrySize += nBytes;
172+
173+
if (totalUncompressedSize.addAndGet(nBytes) > MAX_ARCHIVE_SIZE) {
174+
throw new IOException("Archive is too large when uncompressed.");
175+
}
176+
177+
if (entry.getCompressedSize() > 0) {
178+
double ratio = (double) totalEntrySize / entry.getCompressedSize();
179+
if (ratio > COMPRESSION_RATIO_LIMIT) {
180+
throw new IOException("Compression ratio for entry " + entry.getName() + " is too high.");
181+
}
182+
}
183+
}
184+
}
185+
}
184186
}
185-

0 commit comments

Comments
 (0)