Skip to content

Commit 4d3a732

Browse files
committed
feat: return resolved symbol tables from create_job
Signed-off-by: David Leong <leongdl@amazon.com>
1 parent 69d996d commit 4d3a732

3 files changed

Lines changed: 344 additions & 7 deletions

File tree

‎src/openjd/model/__init__.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
22

33
from ._capabilities import validate_attribute_capability_name, validate_amount_capability_name
4-
from ._create_job import create_job, preprocess_job_parameters
4+
from ._create_job import (
5+
JobWithSymbolTables,
6+
create_job,
7+
create_job_with_symbol_tables,
8+
preprocess_job_parameters,
9+
)
510
from ._errors import (
611
CompatibilityError,
712
DecodeValidationError,
@@ -51,6 +56,8 @@
5156

5257
__all__ = (
5358
"create_job",
59+
"create_job_with_symbol_tables",
60+
"JobWithSymbolTables",
5461
"decode_template",
5562
"decode_environment_template",
5663
"decode_job_template",

‎src/openjd/model/_create_job.py‎

Lines changed: 143 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,22 @@
11
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
22

33
import json
4+
from dataclasses import dataclass
45
from os.path import normpath
56
from pathlib import Path
6-
from typing import Any, Optional, cast
7+
from typing import TYPE_CHECKING, Any, Optional, cast
78

89
from pydantic import ValidationError
910

1011
from ._errors import CompatibilityError, DecodeValidationError
1112
from ._format_strings import FormatStringError
1213
from ._symbol_table import SymbolTable
14+
15+
if TYPE_CHECKING:
16+
# Type-only: importing openjd.expr loads the Rust bindings, and importing this
17+
# module must not. The runtime imports live in _serialize_symbol_table, which
18+
# only the opt-in create_job_with_symbol_tables path reaches.
19+
from openjd.expr import SerializedSymbolTable
1320
from ._internal import instantiate_model
1421
from ._merge_job_parameter import merge_job_parameter_definitions
1522
from ._types import (
@@ -26,7 +33,28 @@
2633
)
2734
from ._convert_pydantic_error import pydantic_validationerrors_to_str
2835

29-
__all__ = ("preprocess_job_parameters",)
36+
__all__ = ("preprocess_job_parameters", "create_job_with_symbol_tables", "JobWithSymbolTables")
37+
38+
39+
@dataclass(frozen=True)
40+
class JobWithSymbolTables:
41+
"""A created job together with the symbol tables it was instantiated with.
42+
43+
Returned by :func:`create_job_with_symbol_tables`. The tables are in the
44+
``SerializedSymbolTable`` transport form, ready to persist or send to the host
45+
that will run the job's sessions.
46+
"""
47+
48+
job: Job
49+
"""The created job — identical to what ``create_job`` returns."""
50+
51+
job_symbol_table: "SerializedSymbolTable"
52+
"""Job scope: ``Param.*``, ``RawParam.*``, and ``Job.Name`` under EXPR."""
53+
54+
step_symbol_tables: dict[str, "SerializedSymbolTable"]
55+
"""Step scope keyed by step name: job scope plus ``Step.Name`` and the step's
56+
evaluated template-scope ``let`` bindings."""
57+
3058

3159
# The original scalar job-parameter type names whose values are carried as
3260
# strings through preprocessing. EXPR-extension types (BOOL, RANGE_EXPR, and
@@ -386,12 +414,12 @@ def preprocess_job_parameters(
386414
# =======================================================================
387415

388416

389-
def create_job(
417+
def _create_job_and_symbol_table(
390418
*,
391419
job_template: JobTemplate,
392420
job_parameter_values: JobParameterValues,
393421
environment_templates: Optional[list[EnvironmentTemplate]] = None,
394-
) -> Job:
422+
) -> tuple[Job, SymbolTable]:
395423
"""This function will create a job from a given Job Template and set of values for
396424
Job Parameters. Minimally, values must be provided for Job Parameters that do not have
397425
default values defined in the template.
@@ -409,7 +437,8 @@ def create_job(
409437
DecodeValidationError
410438
411439
Returns:
412-
Job: The job generated.
440+
tuple[Job, SymbolTable]: The job generated, and the job-scope symbol table
441+
it was instantiated with.
413442
"""
414443

415444
# Raises: ValueError
@@ -489,4 +518,112 @@ def create_job(
489518
pydantic_validationerrors_to_str(job_template.__class__, exc.errors())
490519
)
491520

492-
return cast(Job, job)
521+
return cast(Job, job), symtab
522+
523+
524+
def create_job(
525+
*,
526+
job_template: JobTemplate,
527+
job_parameter_values: JobParameterValues,
528+
environment_templates: Optional[list[EnvironmentTemplate]] = None,
529+
) -> Job:
530+
"""Create a job from a Job Template and a set of Job Parameter values.
531+
532+
See :func:`create_job_with_symbol_tables` when you also need the resolved
533+
symbol tables — for instance to transport them to a host that will run the
534+
job's sessions.
535+
536+
Raises:
537+
DecodeValidationError
538+
539+
Returns:
540+
Job: The job generated.
541+
"""
542+
job, _symtab = _create_job_and_symbol_table(
543+
job_template=job_template,
544+
job_parameter_values=job_parameter_values,
545+
environment_templates=environment_templates,
546+
)
547+
return job
548+
549+
550+
def create_job_with_symbol_tables(
551+
*,
552+
job_template: JobTemplate,
553+
job_parameter_values: JobParameterValues,
554+
environment_templates: Optional[list[EnvironmentTemplate]] = None,
555+
) -> JobWithSymbolTables:
556+
"""Create a job, and return the resolved symbol tables alongside it.
557+
558+
Same job as :func:`create_job`. The difference is that the symbol tables
559+
built during instantiation are returned instead of discarded, in the
560+
``SerializedSymbolTable`` transport form, so a caller can persist them and
561+
hand them to a session on another host.
562+
563+
This mirrors ``openjd-rs``, whose ``create_job`` attaches the step-scope
564+
table to each ``Step`` as ``resolved_symtab``. The tables are returned
565+
separately here rather than added to the ``Job`` model so that the model's
566+
serialized form does not change.
567+
568+
Scopes, matching the sessions API:
569+
570+
* ``job_symbol_table`` — job scope: ``Param.*``, ``RawParam.*`` and, with the
571+
EXPR extension, ``Job.Name``. Use it for job and queue environments.
572+
* ``step_symbol_tables`` — job scope plus ``Step.Name`` and the step's
573+
evaluated template-scope ``let`` bindings, keyed by step name. Use each for
574+
that step and its step environments.
575+
576+
Script-scope ``let`` bindings are deliberately absent: they resolve at
577+
session time, so the table carries the symbols they reference rather than
578+
their results.
579+
580+
Unlike ``openjd-rs``, the tables are not filtered down to the symbols a step
581+
actually references. They are a superset, which is valid input wherever a
582+
filtered table is — a session layers its own scopes on top either way.
583+
584+
Raises:
585+
DecodeValidationError
586+
587+
Returns:
588+
JobWithSymbolTables: The job and its resolved symbol tables.
589+
"""
590+
job, symtab = _create_job_and_symbol_table(
591+
job_template=job_template,
592+
job_parameter_values=job_parameter_values,
593+
environment_templates=environment_templates,
594+
)
595+
596+
step_tables: dict[str, SerializedSymbolTable] = {}
597+
for step_template in getattr(job_template, "steps", None) or []:
598+
# Reuse the model's own per-step hook — the one instantiate_model calls —
599+
# so the returned table is the step scope the job was instantiated with
600+
# rather than a reimplementation of it. It depends only on the step's
601+
# name, its `let` bindings and the job-scope table, so re-invoking it
602+
# here is deterministic.
603+
extends_symtab = step_template._job_creation_metadata.extends_symtab
604+
step_symtab = extends_symtab(step_template, symtab) if extends_symtab else symtab
605+
step_tables[str(step_template.name)] = _serialize_symbol_table(step_symtab)
606+
607+
return JobWithSymbolTables(
608+
job=job,
609+
job_symbol_table=_serialize_symbol_table(symtab),
610+
step_symbol_tables=step_tables,
611+
)
612+
613+
614+
def _serialize_symbol_table(symtab: SymbolTable) -> "SerializedSymbolTable":
615+
"""Convert a job-model symbol table into the EXPR transport form.
616+
617+
Goes through ``symtab_to_expr_values`` so the typed coercion is the engine's
618+
own, then through ``SerializedSymbolTable.from_symtab`` — the same serializer
619+
openjd-rs uses — so the bytes match what the Rust implementation produces for
620+
an equivalent table.
621+
"""
622+
# Imported here rather than at module scope: both of these load the Rust
623+
# bindings, and importing openjd.model must not.
624+
from openjd.expr import SerializedSymbolTable
625+
626+
from ._format_strings._expr_support import symtab_to_expr_values
627+
628+
engine_symtab = symtab_to_expr_values(symtab, types=symtab.expr_types)
629+
return SerializedSymbolTable.from_symtab(engine_symtab)

0 commit comments

Comments
 (0)