diff --git a/.github/workflows/build-linux-big-endian.yml b/.github/workflows/build-linux-big-endian.yml index 190cc5af3f..391e5fe001 100644 --- a/.github/workflows/build-linux-big-endian.yml +++ b/.github/workflows/build-linux-big-endian.yml @@ -1,12 +1,14 @@ name: Build and test Linux big-endian # Cross-builds the console version for s390x (big-endian) and runs it under -# qemu-user: the algorithm self-tests and one of the Tests/*.hc volumes -# (created on little-endian). +# qemu-user: the algorithm self-tests, a read of known data from one of the +# Tests/*.hc volumes without mounting, and a mount of one of them (all +# created on little-endian). # # The full suite (manual run with "full" selected) also builds amd64, then -# tests the Tests/*.hc volumes on both builds and volume compatibility -# between them in both directions, in separate jobs that run concurrently. +# reads and tests all Tests/*.hc volumes on both builds and volume +# compatibility between them in both directions, in separate jobs that run +# concurrently. on: push: @@ -25,6 +27,8 @@ on: - 'src/Makefile' - 'Tests/test.*.hc' - 'Tests/test_byte_order_interop.py' + - 'Tests/test_volume_read.py' + - 'Tests/volume_read_test.cpp' - '.github/workflows/build-linux-big-endian.yml' pull_request: branches: [ "master" ] @@ -42,6 +46,8 @@ on: - 'src/Makefile' - 'Tests/test.*.hc' - 'Tests/test_byte_order_interop.py' + - 'Tests/test_volume_read.py' + - 'Tests/volume_read_test.cpp' - '.github/workflows/build-linux-big-endian.yml' # Manual runs, e.g. on a fork before opening a pull request. workflow_dispatch: @@ -128,6 +134,19 @@ jobs: - name: Algorithm self-tests run: src/Main/veracrypt --text --test + # Needs Volume.a and Platform.a, so it runs here and not in the jobs that + # only receive the binary. One volume keeps the automatic check short. + - name: Read known data from the test volumes + run: | + set -- --build-dir src --require-thread-pool + if [ "${{ matrix.arch }}" = s390x ]; then + export CXX=s390x-linux-gnu-g++ NM=s390x-linux-gnu-nm + fi + if [ "${{ env.FULL }}" != true ]; then + set -- "$@" --fixture test.sha512.hc + fi + python3 Tests/test_volume_read.py "$@" + # One volume keeps this check short; the full suite opens all of them. - name: Volume created on little-endian if: env.FULL != 'true' diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 994bfe73de..bef10cfffb 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -22,6 +22,9 @@ on: - 'src/Volume/**' - 'src/Makefile' - 'Tests/test_linux_privilege_helpers.py' + - 'Tests/test.*.hc' + - 'Tests/test_volume_read.py' + - 'Tests/volume_read_test.cpp' - '.github/workflows/build-linux.yml' pull_request: branches: [ "master" ] @@ -44,6 +47,9 @@ on: - 'src/Volume/**' - 'src/Makefile' - 'Tests/test_linux_privilege_helpers.py' + - 'Tests/test.*.hc' + - 'Tests/test_volume_read.py' + - 'Tests/volume_read_test.cpp' - '.github/workflows/build-linux.yml' concurrency: @@ -129,6 +135,12 @@ jobs: name: veracrypt-console-debs-${{ matrix.os }} path: /tmp/VeraCrypt_Packaging/Console/Packaging/veracrypt-console-*.* + # Decrypts the Tests/*.hc volumes through Volume::ReadSectors, without + # mounting, using the archives that the last build above (the console + # version) left in src, and checks known data. + - name: Read known data from the test volumes + run: python3 Tests/test_volume_read.py --build-dir src --require-thread-pool + - name: Install and test VeraCrypt GUI .deb packages run: | sudo apt install -y /tmp/VeraCrypt_Packaging/GUI/Packaging/veracrypt-*.deb diff --git a/Tests/test_volume_read.py b/Tests/test_volume_read.py new file mode 100644 index 0000000000..02419c529d --- /dev/null +++ b/Tests/test_volume_read.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Read known data from the Tests/test.*.hc volumes without mounting. + +Usage: + python3 Tests/test_volume_read.py --build-dir src [--require-thread-pool] + [--fixture test.sha512.hc ...] + +Compiles Tests/volume_read_test.cpp against Volume/Volume.a and +Platform/Platform.a of a finished Linux build in --build-dir, then decrypts +the data area of each volume through Volume::ReadSectors (in one call, as +multi-sector ranges at nonzero offsets, with the encryption thread pool, and +one sector at a time). The master keys and data offsets decoded from a +real header must recover the volume's plaintext. For every volume, the decrypted data area must: + + - start with a FAT boot sector, + - contain DUMMY.TXT with the content "Dummy" at the cluster given by the + root directory, + - match a known SHA-256 digest. + +Without --fixture, every Tests/test.*.hc volume is checked, and a volume +without a digest here (or a digest without a volume) is an error, so a new +fixture cannot be left out by accident. The thread pool does not start with +fewer than two processors; that part is then reported as skipped, or fails +with --require-thread-pool (as in CI). + +Supported builds: the standard Linux build (Crypto/ backend, not +WOLFCRYPT_BACKEND), native or cross-compiled. The test program is compiled +with the defines and libraries listed in build() below, not with the flags +the archives were built with; only the libstdc++ ABI is detected from the +archive. Other configurations, e.g. a wolfCrypt build, which also needs +WOLFCRYPT_BACKEND and the wolfSSL library, are not supported. + +Unlike mounting the volumes, this needs no root, FUSE or installed package, +so it also runs on a cross build: set CXX (and NM) to the cross tools, e.g. +CXX=s390x-linux-gnu-g++ NM=s390x-linux-gnu-nm. The test program is then run +through qemu-user binfmt support, which needs the target's shared libraries +(e.g. libstdc++6:s390x from a multiarch install, as in the CI workflow). +""" + +import argparse +import hashlib +import os +import shlex +import struct +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PASSWORD = "test" +EXPECTED_FILE = ("DUMMY TXT", b"Dummy") + +# SHA-256 of the decrypted data area of each volume. +DIGESTS = { + "test.blake2s.hc": + "21cc4f2f8db13fadca82bebb2ad574184dcbb441d447a5dee7363ceac729e0c0", + "test.sha256.hc": + "2b15dbe4544a853ca15311dd1dc83162c6cc06c6c470d98ca427b26093a9fb41", + "test.sha512.hc": + "eef460d147e613d70df39a8e6e83b624fc1d258d8ae35875caf6579a05cba95b", + "test.streebog.hc": + "4c890502857a6360ed059c4d1dc736179fe29bffc9af72bfbff63bb691d34d82", + "test.whirlpool.hc": + "cb194e676678b7c40eead878df102eee0c93d453ca9d0a9e4eab7a3847edd1e0", +} + + +class CheckFailed(Exception): + pass + + +def old_abi(archive): + """Tell whether the archive was built with _GLIBCXX_USE_CXX11_ABI=0. + + src/Makefile selects the old ABI for WXSTATIC builds with GCC 5 or later; + the test program must use the same one to link. + """ + nm = shlex.split(os.environ.get("NM", "nm")) + symbols = subprocess.run(nm + ["--defined-only", str(archive)], + stdout=subprocess.PIPE, text=True, + check=True).stdout + return "cxx11" not in symbols + + +def build(build_dir, workdir): + volume_lib = build_dir / "Volume" / "Volume.a" + platform_lib = build_dir / "Platform" / "Platform.a" + for lib in (volume_lib, platform_lib): + if not lib.is_file(): + sys.exit(f"{lib} not found: build VeraCrypt in {build_dir} first") + program = workdir / "volume_read_test" + command = shlex.split(os.environ.get("CXX", "c++")) + [ + "-std=c++11", "-O2", "-DTC_UNIX", "-DTC_LINUX", "-DARGON2_NO_THREADS", + "-D_FILE_OFFSET_BITS=64", + "-I" + str(build_dir), "-I" + str(build_dir / "Crypto"), + "-I" + str(build_dir / "Crypto" / "Argon2" / "include"), + "-I" + str(build_dir / "PKCS11"), + ] + if old_abi(volume_lib): + command.append("-D_GLIBCXX_USE_CXX11_ABI=0") + command += [str(ROOT / "Tests" / "volume_read_test.cpp"), str(volume_lib), + str(platform_lib), "-pthread", "-ldl", "-o", str(program)] + subprocess.run(command, check=True, timeout=300) + return program + + +def find_file(data, name): + """Return a small file (one cluster at most) from the root directory of a + FAT12/16 image.""" + if data[510:512] != b"\x55\xaa": + raise CheckFailed("no FAT boot sector signature") + (sector_size, cluster_sectors, reserved_sectors, fat_count, root_entries, + _, _, fat_sectors) = struct.unpack_from(" len(data): + raise CheckFailed("FAT root directory beyond the data area") + for offset in range(root, root + root_entries * 32, 32): + entry = data[offset:offset + 32] + if entry[0] == 0: + break + # Skip volume labels, directories and long file name entries. + if entry[:11].decode("latin-1") == name and not entry[11] & 0x18: + cluster, size = struct.unpack_from(" cluster_sectors * sector_size \ + or start + size > len(data): + raise CheckFailed(f"{name.strip()}: bad cluster {cluster} " + f"or size {size}") + return data[start:start + size] + raise CheckFailed(f"{name.strip()} not found in the root directory") + + +def check_volume(program, fixture, workdir, require_thread_pool): + kdf = fixture.name.split(".")[1] + output = workdir / (fixture.name + ".plain") + command = [str(program), str(fixture), PASSWORD, kdf, str(output)] + if require_thread_pool: + command.append("--require-thread-pool") + result = subprocess.run(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, timeout=900, + check=False) + if result.returncode != 0: + raise CheckFailed(f"exit {result.returncode}: " + f"{result.stderr.strip() or result.stdout.strip()}") + data = output.read_bytes() + output.unlink() + name, content = EXPECTED_FILE + found = find_file(data, name) + if found != content: + raise CheckFailed(f"{name.strip()} contains {found!r}, " + f"expected {content!r}") + digest = hashlib.sha256(data).hexdigest() + if digest != DIGESTS[fixture.name]: + raise CheckFailed(f"data area SHA-256 {digest}, " + f"expected {DIGESTS[fixture.name]}") + return result.stdout.strip() + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--build-dir", type=Path, required=True, + help="VeraCrypt source directory of a finished build " + "(normally src)") + parser.add_argument("--fixture", action="append", default=[], + metavar="NAME", + help="check only this volume from Tests/, given by " + "file name, e.g. test.sha512.hc (repeatable; " + "default: all)") + parser.add_argument("--require-thread-pool", action="store_true", + help="fail if the encryption thread pool does not " + "start (instead of skipping the threaded reads)") + args = parser.parse_args() + + if not args.fixture: + present = {p.name for p in (ROOT / "Tests").glob("test.*.hc")} + if present != set(DIGESTS): + missing = sorted(present - set(DIGESTS)) + stale = sorted(set(DIGESTS) - present) + problems = [] + if missing: + problems.append(f"no digest for {', '.join(missing)}") + if stale: + problems.append(f"no volume for {', '.join(stale)}") + sys.exit("Tests/test.*.hc and the digests in this script do not " + "match: " + "; ".join(problems)) + + names = args.fixture or sorted(DIGESTS) + fixtures = [] + for name in names: + path = ROOT / "Tests" / name + if not path.is_file(): + sys.exit(f"Missing fixture: {path}") + if name not in DIGESTS: + sys.exit(f"No digest for {name} in this script") + fixtures.append(path) + + failures = 0 + with tempfile.TemporaryDirectory(prefix="vc-volume-read-") as tmp: + workdir = Path(tmp) + program = build(args.build_dir.resolve(), workdir) + for fixture in fixtures: + try: + info = check_volume(program, fixture, workdir, + args.require_thread_pool) + print(f"ok {fixture.name}: {info}", flush=True) + except (CheckFailed, subprocess.TimeoutExpired) as e: + print(f"FAIL {fixture.name}: {e}", flush=True) + failures += 1 + + if failures: + sys.exit(f"Failed: {failures} check(s)") + print("All checks passed", flush=True) + + +if __name__ == "__main__": + main() diff --git a/Tests/volume_read_test.cpp b/Tests/volume_read_test.cpp new file mode 100644 index 0000000000..3d93f7dabd --- /dev/null +++ b/Tests/volume_read_test.cpp @@ -0,0 +1,155 @@ +// Decrypt the data area of a volume through the volume layer, without +// mounting. Usage: +// volume_read_test VOLUME PASSWORD KDF OUTPUT [--require-thread-pool] +// +// The volume is opened read-only with PIM 0 and the given KDF (as with +// --hash). CPU features are detected as in the application, so the +// accelerated cipher code is used where available. The data area is read in +// one call, then as ranges of several sectors at nonzero offsets (odd sector +// counts, so the work is split unevenly), again with the encryption thread +// pool running, and one sector at a time; every read must match the first. +// The pool does not start with fewer than two processors: that part is then +// reported as skipped, or fails with --require-thread-pool. The plaintext is +// written to OUTPUT, where Tests/test_volume_read.py checks it. +#include "Platform/Platform.h" +#include "Platform/Finally.h" +#include "Volume/EncryptionThreadPool.h" +#include "Volume/Pkcs5Kdf.h" +#include "Volume/Volume.h" +#include "Crypto/cpu.h" +#include +#include +#include +#include + +using namespace VeraCrypt; + +// Same matching as --hash on the command line: the KDF name, the hash name +// or its alternative name, or "argon2"/"argon2id". +static shared_ptr FindKdf (const string &name) +{ + string lowerName = StringConverter::ToLower (name); + foreach (shared_ptr kdf, Pkcs5Kdf::GetAvailableAlgorithms()) + { + if (StringConverter::ToLower (StringConverter::ToSingle (kdf->GetName())) == lowerName) + return kdf; + if (kdf->IsArgon2()) + { + if (lowerName == "argon2" || lowerName == "argon2id") + return kdf; + continue; + } + if (StringConverter::ToLower (StringConverter::ToSingle (kdf->GetHash()->GetName())) == lowerName + || StringConverter::ToLower (StringConverter::ToSingle (kdf->GetHash()->GetAltName())) == lowerName) + return kdf; + } + throw ParameterIncorrect (SRC_POS); +} + +static void Require (bool condition, const char *message) +{ + if (!condition) + throw std::runtime_error (message); +} + +// Read several multi-sector ranges at nonzero offsets and compare them with +// the data area read in one call. +static void CheckRanges (Volume &volume, const SecureBuffer &data, const char *message) +{ + const size_t sectorSize = volume.GetSectorSize(); + const uint64 sectors = data.Size() / sectorSize; + Require (sectors >= 34, "data area too small for the range checks"); + const uint64 ranges[][2] = { + { 1, 3 }, { 7, 5 }, { sectors / 2 - 4, 9 }, { sectors - 17, 17 } + }; + + for (size_t i = 0; i < sizeof (ranges) / sizeof (ranges[0]); i++) + { + const uint64 offset = ranges[i][0] * sectorSize; + const size_t length = (size_t) (ranges[i][1] * sectorSize); + Require (ranges[i][0] > 0 && ranges[i][0] + ranges[i][1] <= sectors, "data area too small for the range checks"); + + SecureBuffer buffer (length); + volume.ReadSectors (buffer, offset); + Require (memcmp (buffer.Ptr(), data.Ptr() + offset, length) == 0, message); + } +} + +int main (int argc, char **argv) +{ + const bool requireThreadPool = (argc == 6 && strcmp (argv[5], "--require-thread-pool") == 0); + if (argc != 5 && !requireThreadPool) + { + std::cerr << "usage: volume_read_test VOLUME PASSWORD KDF OUTPUT [--require-thread-pool]\n"; + return 2; + } + + // The volume path is converted to a wide string with mbsrtowcs, which + // needs the character set of the environment (e.g. for "répertoire"). + // Without it (e.g. the locale is not installed) only ASCII paths work. + const bool localeSet = setlocale (LC_CTYPE, "") != nullptr; + + try + { +#ifdef CRYPTOPP_CPUID_AVAILABLE + DetectX86Features (); +#endif +#if CRYPTOPP_BOOL_ARMV8 + DetectArmFeatures (); +#endif + shared_ptr password (new VolumePassword ((const uint8 *) argv[2], strlen (argv[2]))); + Volume volume; + volume.Open (VolumePath (string (argv[1])), false, password, 0, FindKdf (argv[3]), + shared_ptr (), false, VolumeProtection::ReadOnly); + + const uint64 size = volume.GetSize(); + const size_t sectorSize = volume.GetSectorSize(); + Require (size != 0 && size % sectorSize == 0, "unexpected data area size"); + + SecureBuffer data ((size_t) size); + volume.ReadSectors (data, 0); + CheckRanges (volume, data, "read of several sectors differs"); + + EncryptionThreadPool::Start(); + finally_do ({ EncryptionThreadPool::Stop(); }); + + const bool threaded = EncryptionThreadPool::IsRunning(); + if (threaded) + { + SecureBuffer all ((size_t) size); + volume.ReadSectors (all, 0); + Require (memcmp (all.Ptr(), data.Ptr(), (size_t) size) == 0, "read with the thread pool differs"); + CheckRanges (volume, data, "read of several sectors with the thread pool differs"); + } + else + Require (!requireThreadPool, "the encryption thread pool did not start (fewer than two processors?)"); + + SecureBuffer sector (sectorSize); + for (uint64 offset = 0; offset < size; offset += sectorSize) + { + volume.ReadSectors (sector, offset); + Require (memcmp (sector.Ptr(), data.Ptr() + offset, sectorSize) == 0, "read one sector at a time differs"); + } + + std::ofstream output (argv[4], std::ios::binary); + output.write ((const char *) data.Ptr(), (std::streamsize) data.Size()); + output.close(); + Require (!output.fail(), "cannot write the output file"); + + std::cout << StringConverter::ToSingle (volume.GetEncryptionAlgorithm()->GetName()) + << " " << size << " bytes" << (threaded ? "" : ", thread pool reads skipped (pool not started)") + << (localeSet ? "" : ", locale from the environment unavailable, using C") << "\n"; + volume.Close(); + } + catch (std::exception &e) + { + std::cerr << "error: " << StringConverter::ToSingle (StringConverter::ToExceptionString (e)) << "\n"; + return 1; + } + catch (...) + { + std::cerr << "error: unknown exception\n"; + return 1; + } + return 0; +}