chore: Add fix for SSM timeout as well.

Signed-off-by: Karthik Bekal Pattathana <133984042+karthikbekalp@users.noreply.github.com>

Author: karthikbekalp

.github/workflows/integ_windows.yml

@@ -109,35 +109,86 @@ jobs:
             --query 'Reservations[0].Instances[0].InstanceId' `
             --profile license --region $region --output text).Trim()
           if (-not $bastion -or $bastion -eq "None") { Write-Error "Bastion host not found"; exit 1 }
+          Write-Host "::add-mask::$bastion"
 
           # Start SSM port forwarding to the bastion (params via file to avoid
-          # quoting issues). Redirect stdout to a log file so we can capture
-          # the SessionId for deterministic teardown.
+          # quoting issues). Retry the complete session once because a failed
+          # AWS CLI process cannot recover through additional local port probes.
           $paramsFile = Join-Path $env:RUNNER_TEMP "ssm_params.json"
           @{portNumber=@($rlmPort);localPortNumber=@($rlmPort)} | ConvertTo-Json -Compress | Out-File -FilePath $paramsFile -Encoding ascii
-          $ssmLog = Join-Path $env:RUNNER_TEMP "ssm.log"
-          Start-Process -FilePath "aws" -ArgumentList "ssm start-session --target $bastion --document-name DccInteg-PortForwardToLicenseServer --parameters file://$paramsFile --profile license --region $region" -NoNewWindow -RedirectStandardOutput $ssmLog
-
-          # Wait for port (retry up to 60s)
           $ready = $false
-          for ($i = 0; $i -lt 12; $i++) {
-            Start-Sleep -Seconds 5
-            $t = Test-NetConnection -ComputerName 127.0.0.1 -Port $rlmPort -WarningAction SilentlyContinue
-            if ($t.TcpTestSucceeded) { $ready = $true; break }
-          }
-          if (-not $ready) { Write-Error "SSM port forward not up after 30s"; if (Test-Path $ssmLog) { Get-Content $ssmLog }; exit 1 }
-          Write-Host "License tunnel up via SSM"
-
-          # Capture the SSM session id so the teardown step can terminate the
-          # session on the (shared, long-lived) license host explicitly, rather
-          # than relying on SSM's disconnect detection to reap it.
           $ssmSessionId = ""
-          if (Test-Path $ssmLog) {
-            $m = Select-String -Path $ssmLog -Pattern 'SessionId: ([A-Za-z0-9._-]+)' | Select-Object -First 1
-            if ($m) { $ssmSessionId = $m.Matches[0].Groups[1].Value }
+          $lastStatus = "not started"
+
+          for ($attempt = 1; $attempt -le 2; $attempt++) {
+            $ssmOutLog = Join-Path $env:RUNNER_TEMP "ssm-$attempt.out.log"
+            $ssmErrLog = Join-Path $env:RUNNER_TEMP "ssm-$attempt.err.log"
+            Write-Host "Starting SSM port forward (attempt $attempt/2)"
+            $ssmProcess = Start-Process -FilePath "aws" `
+              -ArgumentList "ssm start-session --target $bastion --document-name DccInteg-PortForwardToLicenseServer --parameters file://$paramsFile --profile license --region $region" `
+              -NoNewWindow -PassThru `
+              -RedirectStandardOutput $ssmOutLog -RedirectStandardError $ssmErrLog
+
+            # Wait up to 30 seconds for the local listener.
+            for ($i = 0; $i -lt 6; $i++) {
+              Start-Sleep -Seconds 5
+              if ($ssmProcess.HasExited) { break }
+              $client = [System.Net.Sockets.TcpClient]::new()
+              try {
+                $client.Connect("127.0.0.1", [int]$rlmPort)
+                $ready = $true
+                break
+              } catch {
+                # The listener is not ready yet.
+              } finally {
+                $client.Dispose()
+              }
+            }
+
+            # Capture and mask the session id. It is retained for teardown but
+            # never printed in the public workflow log.
+            $attemptSessionId = ""
+            if (Test-Path $ssmOutLog) {
+              $m = Select-String -Path $ssmOutLog -Pattern 'SessionId: ([A-Za-z0-9._-]+)' | Select-Object -First 1
+              if ($m) {
+                $attemptSessionId = $m.Matches[0].Groups[1].Value
+                Write-Host "::add-mask::$attemptSessionId"
+              }
+            }
+
+            if ($ready) {
+              $ssmSessionId = $attemptSessionId
+              break
+            }
+
+            $lastStatus = if ($ssmProcess.HasExited) { "exit code $($ssmProcess.ExitCode)" } else { "process still running" }
+            Write-Warning "SSM attempt $attempt/2 did not open the local port ($lastStatus)"
+
+            # Tear down a partial session before retrying.
+            if ($attemptSessionId) {
+              aws ssm terminate-session --session-id $attemptSessionId --profile license --region $region 2>$null | Out-Null
+              $global:LASTEXITCODE = 0
+            }
+            if (-not $ssmProcess.HasExited) {
+              Stop-Process -Id $ssmProcess.Id -Force -ErrorAction SilentlyContinue
+            }
+            Get-Process -Name "session-manager-plugin" -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue
+
+            if ($attempt -lt 2) {
+              Write-Host "Retrying SSM port forward in 5 seconds"
+              Start-Sleep -Seconds 5
+            }
           }
+
           "SSM_SESSION_ID=$ssmSessionId" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
-          if ($ssmSessionId) { Write-Host "SSM session id: $ssmSessionId" } else { Write-Host "SSM session id: <not captured>" }
+
+          if (-not $ready) {
+            Write-Error "SSM port forward failed after 2 attempts ($lastStatus)"
+            exit 1
+          }
+
+          Write-Host "License tunnel up via SSM"
+          if ($ssmSessionId) { Write-Host "SSM session id captured for teardown" } else { Write-Host "SSM session id was not captured" }
 
           hatch run integ:test
 

PR_DESCRIPTION.md

@@ -0,0 +1,101 @@
+### What was the problem/requirement? (What/Why)
+
+The Windows integration workflow exposed two startup failure modes:
+
+1. Cinema 4D 2025 encountered a rare native startup crash and exited with
+   `0xC0000005` (`STATUS_ACCESS_VIOLATION`) before its submitter accessibility
+   application appeared. The test continued waiting for UI Automation, which
+   obscured the native process failure.
+2. A later run never opened the local SSM license-forwarding port. The workflow
+   did not retain the AWS CLI process or capture its stderr, so it could only
+   report a port timeout. `Test-NetConnection` also made the nominal timeout
+   substantially longer than the message indicated.
+
+Failed jobs:
+
+- Cinema 4D startup crash:
+  https://github.com/aws-deadline/deadline-cloud-for-cinema-4d/actions/runs/31621623631/job/94197613785
+- SSM port-forward startup failure:
+  https://github.com/aws-deadline/deadline-cloud-for-cinema-4d/actions/runs/31669886824/job/94352428414
+
+### What was the solution? (How)
+
+The Windows integration test now monitors the launched Cinema 4D process before
+each UI Automation scan:
+
+- If the accessibility application appears, the test continues normally.
+- If Cinema 4D exits first, the test reports its decimal and hexadecimal exit
+  codes and restarts Cinema 4D once.
+- The failed process is cleaned up before the second launch.
+- Each attempt logs its PID and has a separate plugin diagnostic log.
+- A warning records the first startup failure even when the second launch
+  succeeds.
+- If the second launch also exits early, the test fails with the native exit
+  code.
+
+Only an early process exit is retried. Accessibility timeouts and failures after
+startup are not retried.
+
+The integration tests now run in-process with uncaptured output so Cinema 4D
+and xa11y progress is visible immediately. If a test runs for ten minutes,
+pytest's faulthandler dumps every Python thread and exits rather than waiting
+for the GitHub Actions job timeout.
+
+The Windows SSM setup now:
+
+- Retains the `aws ssm start-session` process and reports an early exit code.
+- Uses a direct TCP connection to check the local listener six times at
+  five-second intervals.
+- If the first session does not open the port, cleans up the partial process and
+  session, waits five seconds, and starts one fresh SSM session.
+- After both attempts fail, reports whether the AWS CLI exited without printing
+  raw SSM logs.
+- Captures the SSM session ID for deterministic teardown.
+
+### What is the impact of this change?
+
+This change only affects the integration test harness and Windows integration
+workflow. It makes the suite resilient to a rare, transient Cinema 4D startup
+crash, retries a failed SSM tunnel once, and reports whether the AWS CLI exited
+if both tunnel attempts fail.
+
+There is no change to the Cinema 4D submitter, adaptor, customer workflows, or
+production behavior.
+
+### How was this change tested?
+
+- Unit tests: `358 passed, 6 skipped`
+- `hatch run lint`: passed
+- Workflow YAML parsing: passed
+- `git diff --check`: passed
+
+- Have you run the unit tests?
+
+  Yes. `hatch run test` completed with `358 passed, 6 skipped`.
+
+- Have you run the integration tests? (Add your integration test report below)
+
+  Not locally. The full integration suite requires a Windows environment with
+  Cinema 4D installed, GitHub OIDC credentials, and access to the license
+  infrastructure. End-to-end validation must run in the versioned Windows
+  integration workflow.
+
+- Have you made changes to the submitter?
+
+  No. The changes are limited to the integration test harness, its Hatch
+  command, the Windows workflow, and test documentation.
+
+### Was this change documented?
+
+The modified integration-test functions include updated docstrings, and
+`test/AGENTS.md` documents the in-process execution and hang diagnostics. No
+README, schema, or customer-facing documentation changes are required.
+
+### Is this a breaking change?
+
+No. This change is limited to integration test behavior and does not modify any
+public contract or customer-facing functionality.
+
+----
+
+*By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.*

hatch.toml

@@ -43,7 +43,7 @@ pre-install-commands = [
 PYTHONIOENCODING = "utf-8"
 
 [envs.integ.scripts]
-test = "pytest --no-cov {args:test/integ} -vvv --numprocesses=1"
+test = "pytest --no-cov {args:test/integ} -vvv --numprocesses=0 -s -o faulthandler_timeout=600 -o faulthandler_exit_on_timeout=true"
 
 [envs.installer.scripts]
 build-installer = "python {root}/scripts/build_installer_cli.py --installer-source-path {root}/installer/DeadlineCloudForCinema4dSubmitter.xml {args:}"

test/AGENTS.md

@@ -280,13 +280,13 @@ macOS. All six jobs run in parallel. Local runs default to Cinema 4D 2026.
 Set `C4D_VERSION` for an older version and set `C4D_LOCATION` only when its
 installation is outside that version's default path.
 
-The `integ:test` script hardcodes the `test/integ` path and
-`--numprocesses=1`. Beware: any args you pass *replace* the path (hatch
-`{args:test/integ}` falls back to the global `testpaths = ["test"]`), so
-`hatch run integ:test -k physical` would scan the whole `test/` tree. To
-filter or run in-process (e.g. to see C4D/xa11y stdout, which xdist hides), call
-pytest directly with an explicit path — the test spawns its own subprocesses
-regardless of `--numprocesses`:
+The `integ:test` script runs in-process with output capture disabled so C4D and
+xa11y progress reaches CI as it happens. If one test runs for 10 minutes, pytest
+dumps all Python thread stacks and exits instead of waiting for the CI job
+timeout. Beware: any args you pass *replace* the `test/integ` path (hatch
+`{args:test/integ}` falls back to the global `testpaths = ["test"]`), so `hatch
+run integ:test -k physical` would scan the whole `test/` tree. To filter, call
+pytest with an explicit path:
 
 ```bash
 hatch -e integ run pytest --no-cov test/integ/test_cinema4d.py \

test/integ/test_cinema4d.py

@@ -284,6 +284,18 @@ def _find_submitter_app_while_process_runs(
     last_error: Exception | None = None
 
     while time.monotonic() < deadline:
+        # Avoid entering Windows UIA after C4D has already exited. Enumerating
+        # every top-level app can block on an unresponsive accessibility
+        # provider, which would prevent both this process check and the outer
+        # startup timeout from running.
+        returncode = proc.poll()
+        if returncode is not None:
+            unsigned_returncode = returncode & 0xFFFFFFFF
+            raise _Cinema4DStartupError(
+                "Cinema 4D exited before its accessibility app appeared "
+                f"(exit code {returncode} / 0x{unsigned_returncode:08X})"
+            )
+
         try:
             app = next(
                 (
@@ -299,15 +311,6 @@ def _find_submitter_app_while_process_runs(
         if app is not None:
             return app
 
-        # Check after the app scan so an app registered at the same instant the
-        # process exits is not incorrectly reported as a startup crash.
-        returncode = proc.poll()
-        if returncode is not None:
-            unsigned_returncode = returncode & 0xFFFFFFFF
-            raise _Cinema4DStartupError(
-                "Cinema 4D exited before its accessibility app appeared "
-                f"(exit code {returncode} / 0x{unsigned_returncode:08X})"
-            )
         time.sleep(0.25)
 
     message = (
@@ -599,7 +602,9 @@ def _export_job_bundle_via_submitter(
                 deadline_farm["env_overlay"],
                 extra_env=extra_env,
             )
+            log(f"launching Cinema 4D (attempt {attempt + 1}/2)")
             proc = _launch_cinema4d(cinema4d_gui_exe, scene_path, env)
+            log(f"Cinema 4D launched (attempt {attempt + 1}/2, pid={proc.pid})")
             try:
                 staged_bundle = _drive_submitter_ui(proc, history_dir, configure=configure)
                 _copy_bundle_files(staged_bundle, job_bundle_generated)