fix(install): ship headers order data instead of fetching it on install

package.json

@@ -62,12 +62,13 @@
     "lint": "standard-markdown README.md && standard",
     "postinstall": "node scripts/postinstall.js",
     "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
-    "prerelease": "npm run contributors",
+    "prerelease": "npm run contributors && npm run update:headers-order && npm test",
     "pretest": "npm run lint",
     "release": "standard-version -a",
     "release:github": "github-generate-release",
     "release:tags": "git push --follow-tags origin HEAD:master",
-    "test": "c8 ava"
+    "test": "c8 ava",
+    "update:headers-order": "node scripts/update-headers-order.js && git add src/headers-order.json"
   },
   "license": "MIT",
   "ava": {

scripts/postinstall.js

@@ -1,26 +1,23 @@
 'use strict'
 
-const { writeFile } = require('fs/promises')
+const { main } = require('./update-headers-order')
 
-const URL =
-  'https://raw.githack.com/apify/fingerprint-suite/master/packages/header-generator/src/data_files/headers-order.json'
-
-const isStrict = process.argv.includes('--strict')
-
-fetch(URL)
-  .then(res => {
-    if (!res.ok) throw new Error(`HTTP error! status: ${res.status}`)
-    return res.json()
-  })
-  .then(headersOrder =>
-    writeFile('src/headers-order.json', JSON.stringify(headersOrder, null, 2))
-  )
-  .catch(error => {
-    if (isStrict) {
-      console.error(error)
-      process.exit(1)
-    }
+const install = async ({ strict = false, ...options } = {}) => {
+  try {
+    await main(options)
+  } catch (error) {
+    if (strict) throw error
     console.warn(
       `https-tls: couldn't update headers order, using the bundled one (${error.message})`
     )
+  }
+}
+
+if (require.main === module) {
+  install({ strict: process.argv.includes('--strict') }).catch(error => {
+    console.error(error)
+    process.exitCode = 1
   })
+}
+
+module.exports = { install }

scripts/update-headers-order.js

@@ -0,0 +1,57 @@
+'use strict'
+
+const fs = require('fs/promises')
+const path = require('path')
+
+const SOURCE_URL =
+  'https://raw.githubusercontent.com/apify/fingerprint-suite/master/packages/header-generator/src/data_files/headers-order.json'
+
+const DESTINATION = path.join(__dirname, '../src/headers-order.json')
+
+const REQUIRED_BROWSERS = ['chrome', 'firefox', 'safari']
+
+const isHeaderList = value =>
+  Array.isArray(value) &&
+  value.length > 0 &&
+  value.every(header => typeof header === 'string')
+
+const assertHeadersOrder = headersOrder => {
+  const browsers = [
+    ...new Set([...REQUIRED_BROWSERS, ...Object.keys(headersOrder ?? {})])
+  ]
+  const invalid = browsers.filter(
+    browser => !isHeaderList(headersOrder?.[browser])
+  )
+  if (invalid.length > 0) {
+    throw new TypeError(`Unexpected headers order for: ${invalid.join(', ')}`)
+  }
+  return headersOrder
+}
+
+const writeAtomically = async (destination, content, io = fs) => {
+  const temporary = `${destination}.${process.pid}.tmp`
+  try {
+    await io.writeFile(temporary, content)
+    await io.rename(temporary, destination)
+  } catch (error) {
+    // A cleanup failure must not replace the write or rename error.
+    await io.rm(temporary, { force: true }).catch(() => {})
+    throw error
+  }
+}
+
+const main = async ({ destination = DESTINATION, url = SOURCE_URL } = {}) => {
+  const response = await fetch(url)
+  if (!response.ok) throw new Error(`HTTP ${response.status} for ${url}`)
+  const headersOrder = assertHeadersOrder(await response.json())
+  await writeAtomically(destination, JSON.stringify(headersOrder, null, 2))
+}
+
+if (require.main === module) {
+  main().catch(error => {
+    console.error(error)
+    process.exitCode = 1
+  })
+}
+
+module.exports = { assertHeadersOrder, main, writeAtomically }

test/package.js

@@ -0,0 +1,45 @@
+'use strict'
+
+const { execFileSync } = require('child_process')
+const test = require('ava').default
+const path = require('path')
+
+const pkg = require('../package.json')
+
+const INSTALL_LIFECYCLE_SCRIPTS = ['preinstall', 'install', 'postinstall']
+
+const packedFiles = () => {
+  const [{ files }] = JSON.parse(
+    execFileSync('npm', ['pack', '--dry-run', '--json'], {
+      cwd: path.join(__dirname, '..'),
+      encoding: 'utf8'
+    })
+  )
+  return files.map(file => file.path)
+}
+
+test('install refreshes header order and falls back to the bundled file', t => {
+  const installScripts = INSTALL_LIFECYCLE_SCRIPTS.filter(
+    name => name in pkg.scripts
+  )
+  t.deepEqual(installScripts, ['postinstall'])
+  t.is(pkg.scripts.postinstall, 'node scripts/postinstall.js')
+})
+
+test('the published package ships the headers order and the install script', t => {
+  const files = packedFiles()
+  t.true(files.includes('src/headers-order.json'))
+  t.true(files.includes('scripts/postinstall.js'))
+  t.true(files.includes('scripts/update-headers-order.js'))
+})
+
+test('release refreshes header order before testing and stages it only after a successful update', t => {
+  t.is(
+    pkg.scripts.prerelease,
+    'npm run contributors && npm run update:headers-order && npm test'
+  )
+  t.is(
+    pkg.scripts['update:headers-order'],
+    'node scripts/update-headers-order.js && git add src/headers-order.json'
+  )
+})

test/postinstall.js

@@ -0,0 +1,85 @@
+'use strict'
+
+const { spawnSync } = require('child_process')
+const fs = require('fs')
+const { mkdtemp, readFile, rm, writeFile } = require('fs/promises')
+const os = require('os')
+const path = require('path')
+
+const test = require('ava').default
+
+const { install } = require('../scripts/postinstall')
+
+const SCRIPT = path.join(__dirname, '../scripts/postinstall.js')
+const SHIPPED = path.join(__dirname, '../src/headers-order.json')
+const ORIGINAL = 'original\n'
+const REFUSED = 'http://127.0.0.1:1/headers-order.json'
+
+const temporaryDirectory = () =>
+  mkdtemp(path.join(os.tmpdir(), 'headers-order-install-'))
+
+const runCli = (args, stubSource) => {
+  const directory = fs.mkdtempSync(
+    path.join(os.tmpdir(), 'headers-order-install-cli-')
+  )
+  const stub = path.join(directory, 'stub.js')
+  fs.writeFileSync(stub, `'use strict'\n${stubSource}\n`)
+  const before = fs.readFileSync(SHIPPED)
+  try {
+    const result = spawnSync(process.execPath, ['--require', stub, SCRIPT, ...args], {
+      encoding: 'utf8',
+      timeout: 10000
+    })
+    return {
+      result,
+      unchanged: before.equals(fs.readFileSync(SHIPPED))
+    }
+  } finally {
+    fs.rmSync(directory, { recursive: true, force: true })
+  }
+}
+
+test('a failed refresh keeps the bundled file', async t => {
+  const directory = await temporaryDirectory()
+  t.teardown(() => rm(directory, { recursive: true, force: true }))
+  const destination = path.join(directory, 'headers-order.json')
+  await writeFile(destination, ORIGINAL)
+
+  await install({ destination, url: REFUSED })
+
+  t.is(await readFile(destination, 'utf8'), ORIGINAL)
+})
+
+test('a strict refresh failure is an error', async t => {
+  const directory = await temporaryDirectory()
+  t.teardown(() => rm(directory, { recursive: true, force: true }))
+  const destination = path.join(directory, 'headers-order.json')
+  await writeFile(destination, ORIGINAL)
+
+  await t.throwsAsync(install({ destination, strict: true, url: REFUSED }))
+  t.is(await readFile(destination, 'utf8'), ORIGINAL)
+})
+
+test('install exits 0 when the download fails', t => {
+  const { result, unchanged } = runCli(
+    [],
+    "global.fetch = async () => { throw new TypeError('fetch failed') }"
+  )
+
+  t.is(result.status, 0)
+  t.true(result.stderr.includes('using the bundled one'))
+  t.true(result.stderr.includes('fetch failed'))
+  t.true(unchanged)
+})
+
+test('install exits 1 when the download fails and strict is set', t => {
+  const { result, unchanged } = runCli(
+    ['--strict'],
+    "global.fetch = async () => { throw new TypeError('fetch failed') }"
+  )
+
+  t.is(result.status, 1)
+  t.true(result.stderr.includes('fetch failed'))
+  t.false(result.stderr.includes('using the bundled one'))
+  t.true(unchanged)
+})