Merge development work

- documented basic things that are included in the engine for assisted development
- added tests for Array, String and fixed various bugs
- added tests for PPMem
- completely replaced FFMpeg with WebM playback (AV1 / Opus)
- rewritten EgfMan into sysLib and ImGui, dropping wxWidgets version of the tool
- fixed Github CI issues and made tests runnable
- various bug fixes

Author: SoapyMan

.github/workflows/linux_build.yml

@@ -28,16 +28,23 @@ on:
 
 env:
   PROJECT_NAME: auto_tests
-  BUILD_CONFIGURATION: Release
   BUILD_PLATFORM: x64
+  TESTS: ds_tests ppmem_tests scripting_tests
 
   EXE_SUFFIX:
-  BUILD_CONFIGURATION_LOWER: retail
 
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - configuration: Debug
+            make_config: debug
+          - configuration: Release
+            make_config: release
 
     steps:
     - uses: actions/checkout@v3
@@ -62,6 +69,16 @@ jobs:
       run: ./utils/premake5 --file=premake5-tests.lua gmake2
 
     - name: make
-      working-directory: ./build
-      run: make config=${{env.BUILD_CONFIGURATION_LOWER}}_${{env.BUILD_PLATFORM}} -j10
-            
+      working-directory: ./build/${{env.PROJECT_NAME}}
+      run: make config=${{matrix.make_config}}_${{env.BUILD_PLATFORM}} -j10
+
+    - name: Run tests
+      working-directory: ./build/${{env.PROJECT_NAME}}/bin/${{env.BUILD_PLATFORM}}/${{matrix.configuration}}
+      timeout-minutes: 10
+      run: |
+        export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)
+        failed=""
+        for test in $TESTS; do
+          ./$test || failed="$failed $test"
+        done
+        if [ -n "$failed" ]; then echo "::error::Failed tests:$failed"; exit 1; fi

.github/workflows/msbuild.yml

@@ -28,16 +28,22 @@ on:
 
 env:
   PROJECT_NAME: auto_tests
-  BUILD_CONFIGURATION: Release
   BUILD_PLATFORM: x64
-  WINSDK_VER: "10.0.19041.0"
+  TESTS: ds_tests ppmem_tests scripting_tests
+  # Tracy needs SDK 10.0.26100 or newer (RelationProcessorDie, ETW system provider GUIDs);
+  # it is preinstalled on the windows-2022 runner image
+  WINSDK_VER: "10.0.26100.0"
 
 permissions:
   contents: read
 
 jobs:
   build:
     runs-on: windows-2022
+    strategy:
+      fail-fast: false
+      matrix:
+        configuration: [ Debug, Release ]
 
     steps:      
     - uses: actions/checkout@v3
@@ -50,11 +56,6 @@ jobs:
       with:
         modules-to-cache: 7Zip4PowerShell
 
-    - name: Setup Windows 10 SDK Action
-      uses: GuillaumeFalourd/setup-windows10-sdk-action@v2
-      with:
-        sdk-version: 19041 # TODO: env
-
     - name: Run PS commands
       shell: pwsh
       run: .\utils\windows_dev_prepare.ps1
@@ -70,5 +71,16 @@ jobs:
       working-directory: ${{env.GITHUB_WORKSPACE}}
       # Add additional options to the MSBuild command line here (like platform or verbosity level).
       # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
-      run: msbuild /m /p:Configuration=${{env.BUILD_CONFIGURATION}} /p:Platform=${{env.BUILD_PLATFORM}} build\${{env.PROJECT_NAME}}.sln
+      run: msbuild /m /p:Configuration=${{matrix.configuration}} /p:Platform=${{env.BUILD_PLATFORM}} build\${{env.PROJECT_NAME}}\${{env.PROJECT_NAME}}.sln
 
+    - name: Run tests
+      working-directory: build\${{env.PROJECT_NAME}}\bin\${{env.BUILD_PLATFORM}}\${{matrix.configuration}}
+      shell: pwsh
+      timeout-minutes: 10
+      run: |
+        $failed = @()
+        foreach ($test in $env:TESTS.Split(' ')) {
+          & ".\$test.exe"
+          if ($LASTEXITCODE -ne 0) { $failed += $test }
+        }
+        if ($failed) { Write-Error "Failed tests: $failed"; exit 1 }

.gitignore

@@ -20,14 +20,15 @@ android-project/build/
 android-project/.gradle/
 android-project/.idea
 
+build/*
+samples/*/build
+
 src_dependency/wxWidgets/*
 src_dependency/openal-soft/*
 src_dependency/SDL2/*
 !src_dependency/wxWidgets/premake5.lua
 !src_dependency/openal-soft/premake5.lua
 !src_dependency/SDL2/premake5.lua
-src_dependency/ffmpeg/*
-!src_dependency/ffmpeg/premake5.lua
 src_dependency/slang/*
 !src_dependency/slang/premake5.lua
 src_dependency_android/SDL2/*

AGENTS.md

@@ -0,0 +1,50 @@
+# EQ2 Engine agent instructions
+
+This is the canonical policy for every coding agent working in this repository. Tool-specific
+instruction files must import or point here instead of copying these rules. If guidance conflicts,
+the user's current request wins, followed by this file, then the subsystem documentation.
+
+## Agent prerequisites and human ownership
+
+- **Agents must not create or submit pull requests, or reply to reviewer questions.** Prepare local
+  changes and validation results for the human author instead.
+- The human author must fully understand every submitted code change, including its behavior,
+  evidence, compatibility constraints and validation. The human creates the pull request, answers
+  review questions and takes responsibility for the submission; agent assistance does not transfer
+  that responsibility.
+  
+## Mandatory preflight
+
+Before editing:
+
+1. Read this file completely.
+2. Run `git status --short`. This is a shared, usually dirty worktree; assume every existing change
+   belongs to the user or another agent.
+3. Read `docs/building.md`, `docs/codestyle.md` and the relevant parts of `docs/architecture.md`.
+4. Check whether another agent is modifying the same files. Partition work or coordinate before
+   editing overlapping code.
+5. At the start of each new task in a long-running session, and always right before committing,
+   check whether this file or the docs you rely on changed since you read them
+   (`git status --short -- AGENTS.md docs/`, `git log -1 --format=%h -- AGENTS.md docs/`).
+   Re-read any that changed and follow the current version; other agents may update the rules
+   in parallel sessions.
+
+## Commits
+
+- Commit only when the user asks. Write commit messages following `docs/commit_messages.md`.
+- Before running `git commit`, prepare the commit following "Review before committing" in
+  `docs/commit_messages.md` (check the index, stage only the task files or use
+  `git commit --only` when others have staged files), show the user the file list and the full
+  commit message, and commit only after the user confirms.
+
+## Validation
+
+- TODO
+- run appropriate tests if test projects are available for covered source parts.
+
+## Documentation
+
+- TODO
+
+
+

CLAUDE.md

@@ -0,0 +1,6 @@
+# Claude Code repository instructions
+
+@AGENTS.md
+
+`AGENTS.md` is the canonical policy for this repository. Read and follow it in full before inspecting
+or changing source; do not maintain a separate copy of its rules here.

TODO.md

@@ -0,0 +1,86 @@
+# TODO
+
+Issues noticed while writing the documentation in `docs/`. None of them has been fixed yet; each
+needs review by the code owner before changing.
+
+## Possible bugs
+
+- [ ] `core/e2core/Localize.cpp:213` — `RemoveTokensFile` calls `m_tokens.remove(it)` and then
+  `++it` on the removed iterator (use after free). `Map::remove` returns the next iterator; the
+  loop should continue from it.
+- [ ] `core/e2core/Localize.cpp:302` — `AddToken(const char*, const wchar_t*)` passes the wide
+  string to `LocalizeConvertSymbols` through `(char*)`. The function parses it as a narrow string
+  and stops on the second byte of the first character, so `\n` / `\t` are not converted for this
+  overload.
+- [ ] `core/e2core/Localize.cpp:302`, `:314` — both `AddToken` overloads cast away `const` and
+  modify the caller's string in place (undefined behavior for literals).
+- [ ] `core/e2core/Localize.cpp:160` — when `RegionalSettings` is missing, the message says
+  "force english" but `m_language` stays empty and the `$LOCALIZE$` search path is already removed.
+- [ ] `public/materialsystem1/IMaterialSystem.h:235` — template `SetupDrawDefaultUP(topology,
+  const VERT* verts, ...)` passes `&verts` (address of the local pointer) instead of `verts`.
+- [ ] `public/render/IDebugOverlay.h:62-63` — `DDNodeBase::nameHash` and `frameindex` are not
+  initialized. Without `Name()` the primitive may be treated as named by
+  `CDebugOverlay::CheckNodeLifetime` (`nameHash != 0`) and replaced by an unrelated primitive.
+- [ ] `framework/core/platform/eqthread.h:101`, `:162` — `CScopedMutex` and `CScopedWriteLocker`
+  with `blocking = false` unlock in the destructor even if the lock was not acquired.
+- [ ] `framework/core/platform/eqatomic.h:62` — `Atomic::CompareExchangeWeak` takes `oldVal` by
+  value, so the updated expected value is not returned to the caller.
+- [ ] `framework/core/platform/eqjobmanager.cpp:260` — `CEqJobManager::AllJobsCompleted` dequeues a
+  job and enqueues it back to check the queue, which changes the queue order (and is racy with
+  workers).
+- [ ] `framework/ds/Map.h:168`, `:499` — `clear()` and `remove()` read `item->next` after calling
+  `item->~Item()`. Works in practice, but is undefined behavior; read `next` before destroying.
+
+- [ ] `shared/font/FontCache.cpp:253` — `GetFamily` returns the default family for unknown names,
+  so `GetFont(..., defaultIfNotFound = false)` never returns `nullptr` once a default font exists.
+
+## Code quality
+
+- [ ] `shared/font/FontCache.cpp:47` — font description loading uses the deprecated
+  `KV_LoadFromFile`.
+- [ ] `shared/studio/StudioGeom.cpp:708` — `.skin` loading uses the deprecated `KV_LoadFromFile`.
+- [ ] `tools/fcompress/fcompress_main.cpp:390`, `:520` — config and KeyValues conversion use the
+  deprecated `KV_LoadFromFile` / `KV_GetValue*`.
+- [ ] `public/egf/*.h`, `shared/egf`, tools — headers and comments still call EGF "Equilibrium
+  Geometry Format" / "Equilibrium Graphics File"; the format is now named Entity Geometry Format.
+
+- [ ] `framework/core/ConCommand.h:17` — `CMD_ARG_CONCAT` uses an undefined `i` and appends the
+  separator once after the loop instead of between arguments. Appears unused.
+- [ ] `core/e2core/profiler_core.cpp:158` — `ptrace_stop` description is "Performance trace start"
+  (copied from `ptrace_start`).
+- [ ] `framework/core/platform/eqthread.h:222`, `:237` — the usage comment mentions a non-existent
+  `StartWorker()` and its worker example calls `StartThread` instead of `StartWorkerThread`.
+- [ ] `public/core/IEqParallelJobs.h:27` — comment mentions `JOB_FLAG_DELETE`, which does not exist
+  (the job is created with `DeleteOnFinish`).
+- [ ] `framework/math/Utility.cpp:36`, `:50` — `PointToScreen` returns `true` when the point is
+  **behind** the camera; the name suggests the opposite. Consider renaming the result or
+  documenting it in `Utility.h`.
+- [ ] `core/e2core/Localize.cpp:113` — `Init` lower-cases `m_language` again after
+  `SetLanguageName` already did it.
+- [ ] `core/e2core/Localize.cpp` — style does not match `docs/codestyle.md` (C-style casts,
+  `_FindToken` reserved identifier, missing `override`, `pszDefaultToken = 0`, mixed `if(` /
+  `if (`). Only relevant if the file is refactored.
+
+## Features
+
+- [ ] `tools/egfca` — generate `.skin` files (additional model skins) from the compile script, so
+  they do not have to be written by hand.
+
+## Build scripts
+
+- [ ] `shared/premake5.lua:189` — `sysLib` lists `gamesys/**`, which does not exist in `shared/`.
+- [ ] `samples/GPUDriven/premake5.lua:26` — uses `egfLib`, which is defined only when
+  `ENABLE_TOOLS` is on.
+- [ ] `utils/.premake_modules/usage` — the custom `usage` / `uses` module conflicts with the
+  built-in usages of newer premake5 releases (they work differently), so the build is pinned to
+  the premake5 binaries in `utils/`. Resolve the conflict (rename the module API or migrate to
+  the built-in usages) to allow updating premake5.
+
+## Documentation to verify
+
+- [ ] `docs/keyvalues.md` — V3 array elements separated by whitespace (`[bottom hcenter]`) and the
+  one-line `embed1 { embeddedFlags flag1 flag3; }` form in V2 were not verified against the parser.
+- [ ] `docs/public_filesystem.md` — whether `CFileSystemFind::GetPath()` returns a full path or only
+  the entry name.
+- [ ] `docs/framework_math.md` — `QuadTree.h` description (Barnes-Hut style) is inferred from method
+  names.

core/e2core/FileSystem.cpp

@@ -12,7 +12,7 @@
 #include <Windows.h>
 #include <direct.h>	// _mkdir()
 #else
-#include <unistd.h> // rmdir()
+#include <unistd.h> // rmdir(), getcwd()
 #define USE_PATH_TO_FILE_MAPPING
 #endif
 
@@ -684,6 +684,70 @@ EqStringRef CFileSystem::GetAbsolutePath(ESearchPath search, const char* dirOrFi
 	return fullPath;
 }
 
+EqStringRef CFileSystem::GetAbsoluteFullPath(ESearchPath search, const char* dirOrFileName) const
+{
+	constexpr int MAX_PATH_LEN = 1024;
+
+	EqString fullPath = GetAbsolutePath(search, dirOrFileName);
+
+#ifdef _WIN32
+
+	EqWString fullWPath;
+	AnsiUnicodeConverter(fullWPath, fullPath);
+
+	wchar_t fullAbsWPath[MAX_PATH_LEN]{ 0 };
+	GetFullPathNameW(fullWPath, MAX_PATH_LEN, fullAbsWPath, nullptr);
+
+	AnsiUnicodeConverter(fullPath, fullAbsWPath);
+#else
+	// like GetFullPathName: lexical normalization, path does not have to exist
+	if (fullPath.Length() && fullPath[0] != CORRECT_PATH_SEPARATOR)
+	{
+		char cwdPath[MAX_PATH_LEN];
+		if (getcwd(cwdPath, sizeof(cwdPath)))
+			fullPath = fnmPathCombine(cwdPath, fullPath);
+	}
+
+	Array<EqStringRef> pathParts(PP_SL);
+	int partStart = 0;
+	for (int i = 0; i <= fullPath.Length(); ++i)
+	{
+		if (i < fullPath.Length() && fullPath[i] != CORRECT_PATH_SEPARATOR)
+			continue;
+
+		const EqStringRef part(fullPath.ToCString() + partStart, i - partStart);
+		partStart = i + 1;
+
+		if (part.Length() == 0 || part == ".")
+			continue;
+
+		if (part == "..")
+		{
+			if (pathParts.numElem())
+				pathParts.popBack();
+			continue;
+		}
+
+		pathParts.append(part);
+	}
+
+	EqString normalizedPath;
+	for (const EqStringRef part : pathParts)
+	{
+		normalizedPath.Append(CORRECT_PATH_SEPARATOR);
+		normalizedPath.Append(part);
+	}
+
+	if (!normalizedPath.Length())
+		normalizedPath.Append(CORRECT_PATH_SEPARATOR);
+
+	fullPath = normalizedPath;
+#endif
+
+	// fullPath is local, return it from temp string storage
+	return EqStringRef::GetTempString(fullPath.ToCString(), fullPath.Length());
+}
+
 bool CFileSystem::FileRemove(const char* filename, ESearchPath search ) const
 {
 	const EqStringRef path = GetAbsolutePath(search, filename);

core/e2core/FileSystem.h

@@ -133,6 +133,7 @@ class CFileSystem : public IFileSystem
 	bool						InitNextPath(FSFindData* findData) const;
 
 	EqStringRef					GetAbsolutePath(ESearchPath search, const char* dirOrFileName) const;
+	EqStringRef					GetAbsoluteFullPath(ESearchPath search, const char* dirOrFileName) const;
 	EqStringRef					GetSearchPath(ESearchPath search, int directoryId = -1) const;
 
 	using SPWalkFunc = EqFunction<bool(EqStringRef filePath, ESearchPath searchPath, const FSSearchPathInfo& spInfo, int spFlags)>;