@@ -1,5 +1,6 @@
 name: Profile
 
+# Independent profiling requests use priority-scheduled runner slots.
 on: # zizmor: ignore[concurrency-limits]
   workflow_dispatch:
     inputs:
@@ -17,6 +18,11 @@ on: # zizmor: ignore[concurrency-limits]
         required: false
         type: string
         default: '64'
+      moe-debug:
+        description: "Enable MoE debug patch and log (MOE_DEBUG_LOG)"
+        required: false
+        type: boolean
+        default: false
       ref:
         description: "Ref (branch/sha) to checkout"
         required: false
@@ -30,16 +36,20 @@ env:
   INFERENCEX_E2E_ROOT: ${{ github.workspace }}
   PORT: '8888'
   INFMAX_CONTAINER_WORKSPACE: '/workspace'
+  DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1'
+  MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600'
   ENROOT_IMPORT_TIME_LIMIT: '120'
   SALLOC_TIME_LIMIT: '480'
+  KEEP_LOGS: '0'
   EVAL_ONLY: 'false'
   RUN_EVAL: 'false'
   IS_AGENTIC: '0'
   IS_MULTINODE: 'false'
+  SCENARIO_SUBDIR: fixed_seq_len/
   HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
   HF_HUB_CACHE: '/mnt/hf_hub_cache/'
   RANDOM_RANGE_RATIO: '0.8'
-  REQUIRE_POWER: '0'
+  PERFETTO_RELAY_URL: https://semianalysisai.github.io/InferenceX-trace-storage
   PYTHONDONTWRITEBYTECODE: '1'
   PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache
 
@@ -98,10 +108,12 @@ jobs:
             python -P -m infx.workflows.ci_priority \
             --policy "${PRIORITY_ROOT}/configs/ci-priority.yaml" \
             --event-name "${GITHUB_EVENT_NAME}" \
-            --queue-namespace "${GITHUB_RUN_ID}:${GITHUB_RUN_ATTEMPT}")
+            --queue-namespace "${GITHUB_RUN_ID}:${GITHUB_RUN_ATTEMPT}" \
+            --criteria-json "${PRIORITY_CRITERIA}")
           echo "raw=$CONFIG_JSON" >> "$GITHUB_OUTPUT"
 
         env:
+          PRIORITY_CRITERIA: ${{ inputs.moe-debug && '["patchwork"]' || '' }}
           INPUTS_CONFIG_FILE: ${{ inputs.config-file }}
           INPUTS_CONFIG_KEY: ${{ inputs.config-key }}
           INPUTS_CONC: ${{ inputs.conc }}
@@ -198,8 +210,6 @@ jobs:
       ) }}
     env:
       EXP_NAME: ${{ matrix.config.exp-name }}
-      SRT_RECIPE: ${{ matrix.config.srt-recipe }}
-      RECIPE_FINGERPRINT: ${{ matrix.config.recipe-fingerprint }}
       MODEL: ${{ matrix.config.model }}
       MODEL_PREFIX: ${{ matrix.config.model-prefix }}
       ISL: ${{ matrix.config.isl }}
@@ -217,6 +227,8 @@ jobs:
       CONC: ${{ matrix.config.conc }}
       SPEC_DECODING: ${{ matrix.config.spec-decoding }}
       DISAGG: ${{ matrix.config.disagg }}
+      MOE_DEBUG: '0'
+      MOE_DEBUG_LOG: ${{ (inputs.moe-debug) && '/workspace/moe_debug.tp0.log' || '' }}
     steps:
       - name: Slurm cleanup
         run: |
@@ -231,6 +243,8 @@ jobs:
             done
           fi
 
+      # A cancelled or crashed job can leave git state in this persistent
+      # workspace that breaks checkout; re-cloning fixes it and is idempotent.
       - name: Checkout code
         id: checkout
         continue-on-error: true
@@ -239,7 +253,6 @@ jobs:
           fetch-depth: 0
           ref: ${{ inputs.ref || github.sha }}
           clean: false
-          submodules: true
           persist-credentials: false
 
       - name: Wipe workspace after failed checkout
@@ -251,6 +264,7 @@ jobs:
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with: *checkout-inputs
 
+      # With clean: false above, nested checkouts also persist between jobs.
       - name: Checkout result tooling
         id: checkout-tooling
         continue-on-error: true
@@ -299,31 +313,59 @@ jobs:
         run: |
           "$INFERENCEX_LAUNCH_PYTHON" -P -m infx.launch cleanup
 
-      - name: Launch job script
+      - name: Launch + Profile (single-node sglang/vllm)
+        id: run
         env:
           RUNNER_NAME: ${{ runner.name }}
-          JOB_CHECK_RUN_ID: ${{ job.check_run_id }}
-          RESULT_FILENAME_BASE: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_tp${{ env.TP }}-pp${{ env.PP_SIZE }}-dcp${{ env.DCP_SIZE }}-pcp${{ env.PCP_SIZE }}-ep${{ env.EP_SIZE }}-dpa${{ env.DP_ATTENTION }}_disagg-${{ env.DISAGG }}_spec-${{ env.SPEC_DECODING }}_conc${{ env.CONC }}_a${{ github.run_attempt }}_${{ runner.name }}
+          PROFILE: '1'
+          SGLANG_TORCH_PROFILER_DIR: /workspace/
+          VLLM_TORCH_PROFILER_DIR: /workspace/
+          VLLM_RPC_TIMEOUT: '1800000'
         shell: bash
         run: |
           if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
           export GITHUB_WORKSPACE="$PWD"
           echo "INFERENCEX_E2E_ROOT=$PWD" >> "$GITHUB_ENV"
           set -eo pipefail
-          RESULT_FILENAME=$(python3 -m infx.results.result_filename)
-          export RESULT_FILENAME
-          echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV"
           export GPU_COUNT=$((TP * PP_SIZE * PCP_SIZE))
           echo "GPU_COUNT=${GPU_COUNT}" >> "$GITHUB_ENV"
+          ep_val="${EP_SIZE:-1}"
+          res_name="${EXP_NAME}_${PRECISION}_${FRAMEWORK}_tp${TP}-pp${PP_SIZE}-dcp${DCP_SIZE}-pcp${PCP_SIZE}_ep${ep_val}_dpa_${DP_ATTENTION}_conc${CONC}_${RUNNER_NAME}"
+          export RESULT_FILENAME="${res_name}"
+          echo "RESULT_FILENAME=${res_name}" >> "$GITHUB_ENV"
 
           source benchmarks/runtime_settings.sh
           "$INFERENCEX_LAUNCH_PYTHON" -m infx.launch run
 
-          if [ ! -f "${RESULT_FILENAME}.json" ]; then
-            echo "Run failed: Benchmark result ${RESULT_FILENAME}.json not found." >&2
+          if [ ! -f "${res_name}.json" ]; then
+            echo "Run failed: Benchmark result ${res_name}.json not found." >&2
             exit 1
           fi
 
+          trace_path="profile_${res_name}.trace.json.gz"
+          if [ -f "$trace_path" ]; then
+            echo "trace=$PWD/$trace_path" >> "$GITHUB_OUTPUT"
+            if [ "${FRAMEWORK}" = "sglang" ]; then
+              # Try to locate corresponding TP-0 traces produced by SGLang profiler
+              merged_latest=$(ls -t profiles/merged-*.trace.json.gz 2>/dev/null | head -n1 || true)
+              if [ -n "${merged_latest}" ] && [ -f "${merged_latest}" ]; then
+                ts_name="${merged_latest##*/}"
+                ts_name="${ts_name#merged-}"
+                ts_name="${ts_name%.trace.json.gz}"
+                tp0_decode="profiles/${ts_name}-TP-0-DECODE.trace.json.gz"
+                tp0_extend="profiles/${ts_name}-TP-0-EXTEND.trace.json.gz"
+                if [ -f "${tp0_decode}" ]; then
+                  echo "tp0_decode=$PWD/${tp0_decode}" >> "$GITHUB_OUTPUT"
+                fi
+                if [ -f "${tp0_extend}" ]; then
+                  echo "tp0_extend=$PWD/${tp0_extend}" >> "$GITHUB_OUTPUT"
+                fi
+              fi
+            fi
+          else
+            echo "Profile trace not found: $trace_path" >&2
+          fi
+
       - name: Process result (json -> agg)
         env:
           RUNNER_TYPE: ${{ matrix.config.runner }}
@@ -333,3 +375,108 @@ jobs:
           source "$PYTHONPATH/benchmarks/check_env.sh"
           check_env_vars INFERENCEX_RESULTS_PYTHON
           "$INFERENCEX_RESULTS_PYTHON" -P -m infx.results.fixed_sequence
+
+      - name: Upload profile as artifact
+        if: ${{ steps.run.outputs.trace != '' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: profile_${{ env.RESULT_FILENAME }}
+          path: ${{ steps.run.outputs.trace }}
+          if-no-files-found: ignore
+
+      - name: Upload TP-0-DECODE trace as artifact
+        if: ${{ steps.run.outputs.tp0_decode != '' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: profile_${{ env.RESULT_FILENAME }}_TP0_DECODE
+          path: ${{ steps.run.outputs.tp0_decode }}
+          if-no-files-found: ignore
+
+      - name: Upload TP-0-EXTEND trace as artifact
+        if: ${{ steps.run.outputs.tp0_extend != '' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: profile_${{ env.RESULT_FILENAME }}_TP0_EXTEND
+          path: ${{ steps.run.outputs.tp0_extend }}
+          if-no-files-found: ignore
+
+      - name: Upload MoE debug log as artifact
+        if: ${{ env.MOE_DEBUG == '1' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: moe_debug_${{ env.RESULT_FILENAME }}
+          path: ${{ env.INFERENCEX_E2E_ROOT }}/moe_debug.tp0.log
+          if-no-files-found: ignore
+
+      - name: Checkout storage repo
+        id: checkout-storage
+        if: ${{ steps.run.outputs.trace != '' }}
+        continue-on-error: true
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with: &storage-inputs
+          repository: SemiAnalysisAI/InferenceX-trace-storage
+          path: storage
+          ref: master
+          # Storage-repository deploy key for authorized profiling dispatches.
+          ssh-key: ${{ secrets.PROFILER_STORAGE_DEPLOY_KEY }} # zizmor: ignore[secrets-outside-env]
+          fetch-depth: 0
+          # This separate storage checkout pushes trace commits using its scoped deploy key.
+          persist-credentials: true # zizmor: ignore[artipacked]
+
+      - name: Wipe storage repo after failed checkout
+        if: ${{ steps.checkout-storage.outcome == 'failure' }}
+        run: rm -rf "$GITHUB_WORKSPACE/storage"
+
+      - name: Retry storage repo checkout
+        if: ${{ steps.checkout-storage.outcome == 'failure' }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with: *storage-inputs # zizmor: ignore[artipacked]
+
+      - name: Push profile to storage repo
+        if: ${{ steps.run.outputs.trace != '' }}
+        id: push
+        env:
+          MATRIX_CONFIG_RUNNER: ${{ matrix.config.runner }}
+          MATRIX_CONFIG_EXP_NAME: ${{ matrix.config['exp-name'] }}
+          MATRIX_EP: ${{ matrix.config.ep || 1 }}
+          TRACE_LOCAL: ${{ steps.run.outputs.trace }}
+        shell: bash
+        run: |
+          set -eo pipefail
+
+          dest_dir="storage/profiles/${GITHUB_SHA}/${MATRIX_CONFIG_RUNNER}/${FRAMEWORK}/${MATRIX_CONFIG_EXP_NAME}_${PRECISION}_tp${TP}_pp${PP_SIZE}_dcp${DCP_SIZE}_pcp${PCP_SIZE}_ep${MATRIX_EP}_conc${CONC}"
+          mkdir -p "$dest_dir"
+          cp "$TRACE_LOCAL" "$dest_dir/trace.json.gz"
+
+          pushd storage >/dev/null
+          git config user.name "github-actions"
+          git config user.email "github-actions@github.com"
+          git add -A
+          git commit -m "Add profile: ${GITHUB_SHA} ${MATRIX_CONFIG_EXP_NAME} tp${TP} pp${PP_SIZE} dcp${DCP_SIZE} pcp${PCP_SIZE} ep${MATRIX_EP} conc${CONC}" || echo "Nothing to commit"
+          git push
+          STORAGE_SHA="$(git rev-parse HEAD)"
+          popd >/dev/null
+
+          export RAW_URL="https://raw.githubusercontent.com/SemiAnalysisAI/InferenceX-trace-storage/${STORAGE_SHA}/profiles/${GITHUB_SHA}/${MATRIX_CONFIG_RUNNER}/${FRAMEWORK}/${MATRIX_CONFIG_EXP_NAME}_${PRECISION}_tp${TP}_pp${PP_SIZE}_dcp${DCP_SIZE}_pcp${PCP_SIZE}_ep${MATRIX_EP}_conc${CONC}/trace.json.gz"
+          export TITLE="${MATRIX_CONFIG_EXP_NAME}_${PRECISION}_tp${TP}_pp${PP_SIZE}_dcp${DCP_SIZE}_pcp${PCP_SIZE}_ep${MATRIX_EP}_conc${CONC}"
+
+          enc_src="$(python3 -c 'import os,urllib.parse; print(urllib.parse.quote(os.environ["RAW_URL"], safe=""))')"
+          enc_title="$(python3 -c 'import os,urllib.parse; print(urllib.parse.quote(os.environ["TITLE"], safe=""))')"
+
+          relay="${PERFETTO_RELAY_URL%/}"
+          RELAY_URL="${relay}/?src=${enc_src}&title=${enc_title}"
+
+          echo "raw_url=$RAW_URL" >> "$GITHUB_OUTPUT"
+          echo "relay_url=$RELAY_URL" >> "$GITHUB_OUTPUT"
+
+      - name: Print Perfetto link (relay)
+        if: ${{ steps.push.outputs.relay_url != '' }}
+        env:
+          RELAY_URL: ${{ steps.push.outputs.relay_url }}
+          RAW_URL: ${{ steps.push.outputs.raw_url }}
+        shell: bash
+        run: |
+          set -eo pipefail
+          echo "RAW trace URL: $RAW_URL"
+          echo "Perfetto Relay URL: $RELAY_URL"
+          printf "\n**Perfetto (Relay):** %s\n" "$RELAY_URL" >> "$GITHUB_STEP_SUMMARY"
@@ -27,6 +27,7 @@
 result = os.environ["RESULT_FILENAME"]
 Path(result + ".json").write_text("{}\\n")
 Path(result + "_conc1.json").write_text('{"num_requests_successful": 1}\\n')
+Path("profile_" + result + ".trace.json.gz").write_bytes(b"fixture trace")
 Path("speedbench-reference-al.yaml").write_text("fixture: 1\\n")
 """
 
@@ -71,7 +72,7 @@ def workflow_step(workflow: str, step_name: str) -> dict:
     [
         ("benchmark-tmpl.yml", "Launch job script"),
         ("benchmark-multinode-tmpl.yml", "Launch multi-node job script"),
-        ("profile.yml", "Launch job script"),
+        ("profile.yml", "Launch + Profile (single-node sglang/vllm)"),
         ("speedbench-al.yml", "Collect AL matrix"),
     ],
 )
@@ -132,4 +133,6 @@ def test_launch_runs_the_python_entrypoint_from_the_project_root(tmp_path, workf
         assert "fixture: 1" in summary.read_text()
     else:
         assert (project / (exported["RESULT_FILENAME"] + ".json")).read_text() == "{}\n"
-
+    if workflow == "profile.yml":
+        output = dict(line.split("=", 1) for line in github_output.read_text().splitlines())
+        assert Path(output["trace"]).read_bytes() == b"fixture trace"