Make extension packages portable

Author: mehmetozguldev

extensions/official/icons-market/extension.json

@@ -10,8 +10,8 @@
   "activationEvents": ["onIconTheme:market-icons"],
   "installation": {
     "downloadUrl": "https://athas.dev/extensions/packages/icon-theme/market/athas.icon-theme.market.tar.gz",
-    "size": 1325,
-    "checksum": "c60ac2bfa9d89b60ceff84dcd0fe6021e08c6c562094571b66e3d7286a52fdd5"
+    "size": 1161,
+    "checksum": "063f10fe0faeb317c15f386c5440360f2803057070f9d3a4cda3b4c145a64d5e"
   },
   "license": "MIT",
   "repository": {

extensions/official/theme-market/extension.json

@@ -146,8 +146,8 @@
   ],
   "installation": {
     "downloadUrl": "https://athas.dev/extensions/packages/theme/market/athas.theme.market.tar.gz",
-    "size": 1756,
-    "checksum": "eef547f490c83628a8fa1bd0e45d521e4bcb0c782ecbc807bb5ed2651ea90f5d"
+    "size": 1567,
+    "checksum": "c82ff12c6bfaa3fb54a1fe3540ca9e454b8024828a71bebefba3624df4167a38"
   },
   "license": "MIT",
   "repository": {

src/features/extensions/scripts/extension-workspace.ts

@@ -1,6 +1,9 @@
-import { readdir } from "node:fs/promises";
-import { writeFile } from "node:fs/promises";
+import { createGzip } from "node:zlib";
+import { createWriteStream } from "node:fs";
+import { readdir, readFile, stat, writeFile } from "node:fs/promises";
 import { basename, join, relative, resolve } from "node:path";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
 
 export type ExtensionManifestRecord = Record<string, unknown>;
 
@@ -136,3 +139,80 @@ export async function writeExtensionManifest(
 ) {
   await writeFile(manifestPath, `${stringifyManifest(manifest)}\n`);
 }
+
+async function listPackageFiles(root: string) {
+  const files: string[] = [];
+
+  async function walk(directory: string) {
+    for (const entry of await readdir(directory, { withFileTypes: true })) {
+      if (entry.name === ".DS_Store") continue;
+
+      const absolutePath = join(directory, entry.name);
+      if (entry.isDirectory()) {
+        await walk(absolutePath);
+      } else if (entry.isFile()) {
+        files.push(relative(root, absolutePath));
+      }
+    }
+  }
+
+  await walk(root);
+  return files.sort((a, b) => a.localeCompare(b));
+}
+
+function writeOctalField(header: Buffer, offset: number, length: number, value: number) {
+  const octal = value.toString(8).padStart(length - 1, "0");
+  header.write(octal, offset, length - 1, "ascii");
+  header[offset + length - 1] = 0;
+}
+
+function writeTarHeader(path: string, size: number, mode: number) {
+  const header = Buffer.alloc(512, 0);
+  const normalizedPath = path.replace(/\\/g, "/");
+
+  if (Buffer.byteLength(normalizedPath) > 100) {
+    throw new Error(`Packaged extension path is too long for portable tar: ${normalizedPath}`);
+  }
+
+  header.write(normalizedPath, 0, 100, "utf8");
+  writeOctalField(header, 100, 8, mode & 0o777);
+  writeOctalField(header, 108, 8, 0);
+  writeOctalField(header, 116, 8, 0);
+  writeOctalField(header, 124, 12, size);
+  writeOctalField(header, 136, 12, 1577836800);
+  header.fill(" ", 148, 156);
+  header[156] = "0".charCodeAt(0);
+  header.write("ustar", 257, 6, "ascii");
+  header.write("00", 263, 2, "ascii");
+
+  let checksum = 0;
+  for (const byte of header) checksum += byte;
+  const checksumText = checksum.toString(8).padStart(6, "0");
+  header.write(checksumText, 148, 6, "ascii");
+  header[154] = 0;
+  header[155] = 0x20;
+
+  return header;
+}
+
+export async function writeStableTarGz(root: string, packagePath: string) {
+  const chunks: Buffer[] = [];
+
+  for (const file of await listPackageFiles(root)) {
+    const absolutePath = join(root, file);
+    const fileStats = await stat(absolutePath);
+    const contents = await readFile(absolutePath);
+    chunks.push(writeTarHeader(file, contents.length, fileStats.mode));
+    chunks.push(contents);
+
+    const padding = (512 - (contents.length % 512)) % 512;
+    if (padding > 0) {
+      chunks.push(Buffer.alloc(padding, 0));
+    }
+  }
+
+  chunks.push(Buffer.alloc(1024, 0));
+  const output = createWriteStream(packagePath);
+  const gzip = createGzip({ level: 9 });
+  await pipeline(Readable.from(chunks), gzip, output);
+}

src/features/extensions/scripts/package-database-sidecars.ts

@@ -13,6 +13,7 @@ import {
   getGeneratedCdnPath,
   listExtensionFolders,
   writeExtensionManifest,
+  writeStableTarGz,
 } from "./extension-workspace";
 
 const cdnBaseUrl = process.env.EXTENSIONS_CDN_BASE_URL || "https://athas.dev/extensions";
@@ -72,10 +73,7 @@ async function createPackage(params: {
     await chmod(targetBinary, 0o755);
 
     await mkdir(dirname(params.packagePath), { recursive: true });
-    await $`find ${tempDir} -exec touch -t 202001010000 {} +`;
-    await $`find . -type f -print | LC_ALL=C sort | tar --no-xattrs --owner=0 --group=0 --numeric-owner -cf - -C ${tempDir} -T - | gzip -n > ${params.packagePath}`.cwd(
-      tempDir,
-    );
+    await writeStableTarGz(tempDir, params.packagePath);
   } finally {
     await rm(tempDir, { recursive: true, force: true });
   }

src/features/extensions/scripts/package-extensions.ts

@@ -12,6 +12,7 @@ import {
   getGeneratedCdnPath,
   listExtensionFolders,
   writeExtensionManifest,
+  writeStableTarGz,
 } from "./extension-workspace";
 
 const cdnBaseUrl = process.env.EXTENSIONS_CDN_BASE_URL || "https://athas.dev/extensions";
@@ -48,10 +49,7 @@ async function createStablePackage(
       `${JSON.stringify(packagedManifest, null, 2)}\n`,
     );
 
-    await $`find ${tempDir} -exec touch -t 202001010000 {} +`;
-    await $`find . -type f -print | LC_ALL=C sort | tar --no-xattrs --owner=0 --group=0 --numeric-owner -cf - -C ${tempDir} -T - | gzip -n > ${packagePath}`.cwd(
-      tempDir,
-    );
+    await writeStableTarGz(tempDir, packagePath);
   } finally {
     await rm(tempDir, { recursive: true, force: true });
   }