diff --git a/Cargo.lock b/Cargo.lock index 6bdf6dbc..90191e29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,6 +173,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + [[package]] name = "char_str" version = "0.0.4" @@ -223,6 +234,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crunchy" version = "0.2.4" @@ -235,17 +255,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -[[package]] -name = "displaydoc" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.4", -] - [[package]] name = "drop_bomb" version = "0.1.5" @@ -303,6 +312,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.11" @@ -409,9 +424,9 @@ dependencies = [ [[package]] name = "granit-parser" -version = "1.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ec0d45986cd51c847c75c5b69a00852c4fc84d0e5e79f041173f73437d0cdf" +checksum = "05f05586d4d4df3d47123207826c40b252728b438a1a07bad3c01ee6bea401df" dependencies = [ "arraydeque", "smallvec", @@ -453,69 +468,6 @@ dependencies = [ "cc", ] -[[package]] -name = "icu_collections" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_properties" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" - -[[package]] -name = "icu_provider" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - [[package]] name = "indexmap" version = "2.14.1" @@ -611,12 +563,6 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "litemap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" - [[package]] name = "log" version = "0.4.34" @@ -791,26 +737,28 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openjd-expr" -version = "0.12.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "febc1f6e714779847dfbb7c0754c274febc286a668076267612a8afa3db2fcee" +checksum = "379859f4b71a91967f6600d2204691c62f04e27b6cdd9f3ea2e4f6c22ff691b3" dependencies = [ "regex", "regex-syntax", - "rustpython-ruff_python_ast", - "rustpython-ruff_python_parser", - "rustpython-ruff_text_size", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", + "ryu", "serde", "serde_json", "shlex", "thiserror", + "unicode-normalization", ] [[package]] name = "openjd-model" -version = "0.13.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e8a7a5c765e901c5ba5e38b01775bdb9064fd9a3161eae1f241dbd66200bae9" +checksum = "4a24e459b033edce4ca749dcb9600dae160e0d6cd50f920b02ac867367c5631a" dependencies = [ "indexmap", "openjd-expr", @@ -840,9 +788,9 @@ dependencies = [ [[package]] name = "openjd-sessions" -version = "0.8.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f86b899ad49220c5c57a223d57f8adf421eb427ad6013f8047d07806931eb8ff" +checksum = "78edbce036740c750ec32f149d17a985c42ff32ccfc4da66e0530c1ff8cdcd5a" dependencies = [ "bitflags", "futures-util", @@ -887,7 +835,16 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_shared", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" +dependencies = [ + "phf_shared 0.14.0", ] [[package]] @@ -896,8 +853,18 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.3", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_codegen" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b585a510fb76fdebead6897982ef2a03a21d8e6cbcca904999742a4afc6ffe" +dependencies = [ + "phf_generator 0.14.0", + "phf_shared 0.14.0", ] [[package]] @@ -906,8 +873,18 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared", - "rand", + "phf_shared 0.11.3", + "rand 0.8.8", +] + +[[package]] +name = "phf_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" +dependencies = [ + "fastrand", + "phf_shared 0.14.0", ] [[package]] @@ -919,6 +896,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -940,15 +926,6 @@ dependencies = [ "portable-atomic", ] -[[package]] -name = "potential_utf" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" -dependencies = [ - "zerovec", -] - [[package]] name = "powerfmt" version = "0.2.0" @@ -1147,7 +1124,17 @@ checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "rand_core 0.10.1", ] [[package]] @@ -1157,7 +1144,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -1169,6 +1156,12 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rawpointer" version = "0.2.1" @@ -1204,6 +1197,106 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "ruff_python_ast" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8f77693e5f11e7d574c84a8dd63190142ae683749f5ef019bafeffb952fd19" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags", + "char_str", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 2.1.3", + "thin-vec", + "thiserror", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c567873f6a0e094d3f72b37862a8420a6eb4b3fc2950b4975e5a61b46aab3acc" +dependencies = [ + "bitflags", + "bstr", + "drop_bomb", + "get-size2", + "hashbrown", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "ruff_unicode_names2", + "rustc-hash 2.1.3", + "stacker", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78209bfee4543e650edffee040849c5ca828a05d60c7d3b8b53bb63d2dd49a70" +dependencies = [ + "itertools 0.15.0", + "memchr", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 2.1.3", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2159c6129bf08ecde949e8aca0edc5411fbcbaaecdbd401fd75385f5a74e95" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb9ffcbeaefe5f3a7bef2c80dbf591a0fa1f9ec22c5ce7bf4f4c2397104e6496" +dependencies = [ + "get-size2", +] + +[[package]] +name = "ruff_unicode_names2" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc534053fd582ce4b3431d94f6e9f398bb214eb9baf497b06b5c8d6446f379d" +dependencies = [ + "phf 0.14.0", + "ruff_unicode_names2_generator", +] + +[[package]] +name = "ruff_unicode_names2_generator" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3b540b2b312bde02dbfd5261013e06dda9d3ecd33cb6e62f962b019903ae56c" +dependencies = [ + "phf_codegen 0.14.0", + "rand 0.10.3", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -1241,8 +1334,8 @@ dependencies = [ "log", "num-bigint", "num-traits", - "phf", - "phf_codegen", + "phf 0.11.3", + "phf_codegen 0.11.3", "rustc-hash 1.1.0", "rustpython-ast", "rustpython-parser-core", @@ -1273,87 +1366,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "rustpython-ruff_python_ast" -version = "0.16.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7ffeb85f9a68c461d538839c79548584768df08f2d68cb1ee693be697e0355" -dependencies = [ - "aho-corasick", - "arrayvec", - "bitflags", - "char_str", - "compact_str", - "get-size2", - "is-macro", - "memchr", - "rustc-hash 2.1.3", - "rustpython-ruff_python_trivia", - "rustpython-ruff_source_file", - "rustpython-ruff_text_size", - "thin-vec", - "thiserror", -] - -[[package]] -name = "rustpython-ruff_python_parser" -version = "0.16.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c9729e35cac02839c7a67aaccfea3eba32032f7aeacca2e82ad79e8d7b7fd95" -dependencies = [ - "bitflags", - "bstr", - "drop_bomb", - "get-size2", - "hashbrown", - "icu_properties", - "memchr", - "rustc-hash 2.1.3", - "rustpython-ruff_python_ast", - "rustpython-ruff_python_trivia", - "rustpython-ruff_text_size", - "stacker", - "static_assertions", - "thin-vec", - "unicode-ident", - "unicode-normalization", - "unicode_names2", -] - -[[package]] -name = "rustpython-ruff_python_trivia" -version = "0.16.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764f43ea060f888164fbd360b0e1a36f2a35f5c03def85407bab7039fc62a645" -dependencies = [ - "itertools 0.15.0", - "memchr", - "rustc-hash 2.1.3", - "rustpython-ruff_source_file", - "rustpython-ruff_text_size", - "unicode-ident", - "unicode-normalization", -] - -[[package]] -name = "rustpython-ruff_source_file" -version = "0.16.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe26430c3f2eadb40524b6df7ac9a3970dd2b58f921b01ed75df0121e332fad" -dependencies = [ - "memchr", - "rustpython-ruff_text_size", -] - -[[package]] -name = "rustpython-ruff_text_size" -version = "0.16.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5377a04f2855030600b93a05754242950ea9b070507ef9aef67965a52eeb09" -dependencies = [ - "get-size2", -] - [[package]] name = "rustversion" version = "1.0.23" @@ -1378,9 +1390,9 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "1.2.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3afb591f9cdb6223c88ba39269aff895620c7f0716dc42b705b5733d5c7c0823" +checksum = "d43fb82d550b07e6f929a58d7d0935182602f48ddead875f5ee313c41c8f07fe" dependencies = [ "annotate-snippets", "base64", @@ -1478,12 +1490,6 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - [[package]] name = "stacker" version = "0.1.25" @@ -1604,17 +1610,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "synstructure" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.4", -] - [[package]] name = "target-lexicon" version = "0.13.5" @@ -1675,16 +1670,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "tinystr" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" -dependencies = [ - "displaydoc", - "zerovec", -] - [[package]] name = "tinyvec" version = "1.12.0" @@ -1863,7 +1848,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" dependencies = [ - "phf", + "phf 0.11.3", "unicode_names2_generator", ] @@ -1875,16 +1860,10 @@ checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" dependencies = [ "getopts", "log", - "phf_codegen", - "rand", + "phf_codegen 0.11.3", + "rand 0.8.8", ] -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - [[package]] name = "uuid" version = "1.26.0" @@ -2108,35 +2087,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" -[[package]] -name = "writeable" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" - -[[package]] -name = "yoke" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.4", - "synstructure", -] - [[package]] name = "zerocopy" version = "0.8.56" @@ -2157,60 +2107,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.4", - "synstructure", -] - -[[package]] -name = "zerotrie" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.4", -] - [[package]] name = "zmij" version = "1.0.23" diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index 273fb340..8040920f 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -611,7 +611,6 @@ PERFORMANCE OF THIS SOFTWARE. ** nohash-hasher; version 0.2.0 -- https://crates.io/crates/nohash-hasher ** static_assertions; version 1.1.0 -- https://crates.io/crates/static_assertions ** tinyvec; version 1.12.0 -- https://crates.io/crates/tinyvec -** utf8_iter; version 1.0.4 -- https://crates.io/crates/utf8_iter Apache License Version 2.0, January 2004 @@ -2094,7 +2093,6 @@ limitations under the License. ** bitflags; version 2.13.1 -- https://crates.io/crates/bitflags ** bstr; version 1.13.1 -- https://crates.io/crates/bstr ** cfg-if; version 1.0.4 -- https://crates.io/crates/cfg-if -** displaydoc; version 0.2.7 -- https://crates.io/crates/displaydoc ** drop_bomb; version 0.1.5 -- https://crates.io/crates/drop_bomb ** either; version 1.18.0 -- https://crates.io/crates/either ** encoding_rs_io; version 0.1.8 -- https://crates.io/crates/encoding_rs_io @@ -2112,13 +2110,12 @@ limitations under the License. ** regex-automata; version 0.4.18 -- https://crates.io/crates/regex-automata ** regex-syntax; version 0.8.11 -- https://crates.io/crates/regex-syntax ** regex; version 1.13.1 -- https://crates.io/crates/regex +** ruff_unicode_names2; version 0.0.21 -- https://crates.io/crates/ruff_unicode_names2 ** signal-hook-registry; version 1.4.8 -- https://crates.io/crates/signal-hook-registry ** smallvec; version 1.15.2 -- https://crates.io/crates/smallvec -** stable_deref_trait; version 1.2.1 -- https://crates.io/crates/stable_deref_trait ** stacker; version 0.1.25 -- https://crates.io/crates/stacker ** unicode-normalization; version 0.1.25 -- https://crates.io/crates/unicode-normalization ** unicode-width; version 0.2.2 -- https://crates.io/crates/unicode-width -** unicode_names2; version 1.3.0 -- https://crates.io/crates/unicode_names2 ** uuid; version 1.26.0 -- https://crates.io/crates/uuid ** value-bag; version 1.13.2 -- https://crates.io/crates/value-bag ** wasi; version 0.11.1+wasi-snapshot-preview1 -- https://crates.io/crates/wasi @@ -2739,13 +2736,13 @@ limitations under the License. ** arraydeque; version 0.5.1 -- https://crates.io/crates/arraydeque ** get-size-derive2; version 0.11.1 -- https://crates.io/crates/get-size-derive2 ** get-size2; version 0.11.1 -- https://crates.io/crates/get-size2 -** granit-parser; version 1.2.0 -- https://crates.io/crates/granit-parser +** granit-parser; version 2.1.0 -- https://crates.io/crates/granit-parser ** itoa; version 1.0.18 -- https://crates.io/crates/itoa ** libc; version 0.2.189 -- https://crates.io/crates/libc ** manyhow-macros; version 0.14.1 -- https://crates.io/crates/manyhow-macros -** openjd-expr; version 0.12.0 -- https://crates.io/crates/openjd-expr -** openjd-model; version 0.13.0 -- https://crates.io/crates/openjd-model -** openjd-sessions; version 0.8.1 -- https://crates.io/crates/openjd-sessions +** openjd-expr; version 0.15.0 -- https://crates.io/crates/openjd-expr +** openjd-model; version 0.16.0 -- https://crates.io/crates/openjd-model +** openjd-sessions; version 0.10.0 -- https://crates.io/crates/openjd-sessions ** pin-project-lite; version 0.2.17 -- https://crates.io/crates/pin-project-lite ** portable-atomic; version 1.15.0 -- https://crates.io/crates/portable-atomic ** proc-macro2; version 1.0.107 -- https://crates.io/crates/proc-macro2 @@ -2758,7 +2755,7 @@ limitations under the License. ** rustc-hash; version 2.1.3 -- https://crates.io/crates/rustc-hash ** rustversion; version 1.0.23 -- https://crates.io/crates/rustversion ** ryu; version 1.0.23 -- https://crates.io/crates/ryu -** serde-saphyr; version 1.2.0 -- https://crates.io/crates/serde-saphyr +** serde-saphyr; version 2.1.1 -- https://crates.io/crates/serde-saphyr ** serde; version 1.0.229 -- https://crates.io/crates/serde ** serde_core; version 1.0.229 -- https://crates.io/crates/serde_core ** serde_derive; version 1.0.229 -- https://crates.io/crates/serde_derive @@ -3074,17 +3071,6 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------- -** synstructure; version 0.14.0 -- https://crates.io/crates/synstructure -Copyright 2016 Nika Layzell - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - ------ ** tokio-macros; version 2.7.2 -- https://crates.io/crates/tokio-macros MIT License @@ -3212,11 +3198,11 @@ SOFTWARE. ------ -** rustpython-ruff_python_ast; version 0.16.10 -- https://crates.io/crates/rustpython-ruff_python_ast -** rustpython-ruff_python_parser; version 0.16.10 -- https://crates.io/crates/rustpython-ruff_python_parser -** rustpython-ruff_python_trivia; version 0.16.10 -- https://crates.io/crates/rustpython-ruff_python_trivia -** rustpython-ruff_source_file; version 0.16.10 -- https://crates.io/crates/rustpython-ruff_source_file -** rustpython-ruff_text_size; version 0.16.10 -- https://crates.io/crates/rustpython-ruff_text_size +** ruff_python_ast; version 0.0.21 -- https://crates.io/crates/ruff_python_ast +** ruff_python_parser; version 0.0.21 -- https://crates.io/crates/ruff_python_parser +** ruff_python_trivia; version 0.0.21 -- https://crates.io/crates/ruff_python_trivia +** ruff_source_file; version 0.0.21 -- https://crates.io/crates/ruff_source_file +** ruff_text_size; version 0.0.21 -- https://crates.io/crates/ruff_text_size MIT License Copyright (c) @@ -3291,8 +3277,8 @@ DEALINGS IN THE SOFTWARE. ------ -** phf; version 0.11.3 -- https://crates.io/crates/phf -** phf_shared; version 0.11.3 -- https://crates.io/crates/phf_shared +** phf; version 0.14.0 -- https://crates.io/crates/phf +** phf_shared; version 0.14.0 -- https://crates.io/crates/phf_shared The MIT License (MIT) Copyright (c) 2014-2022 Steven Fackler, Yuki Okushi @@ -3410,72 +3396,7 @@ authorization of the copyright holder. ------ -** icu_collections; version 2.3.0 -- https://crates.io/crates/icu_collections -** icu_locale_core; version 2.3.0 -- https://crates.io/crates/icu_locale_core -** icu_properties; version 2.3.0 -- https://crates.io/crates/icu_properties -** icu_properties_data; version 2.3.0 -- https://crates.io/crates/icu_properties_data -** icu_provider; version 2.3.1 -- https://crates.io/crates/icu_provider -** litemap; version 0.8.3 -- https://crates.io/crates/litemap -** potential_utf; version 0.1.6 -- https://crates.io/crates/potential_utf -** tinystr; version 0.8.4 -- https://crates.io/crates/tinystr -** writeable; version 0.6.4 -- https://crates.io/crates/writeable -** yoke-derive; version 0.8.4 -- https://crates.io/crates/yoke-derive -** yoke; version 0.8.3 -- https://crates.io/crates/yoke -** zerofrom-derive; version 0.1.8 -- https://crates.io/crates/zerofrom-derive -** zerofrom; version 0.1.8 -- https://crates.io/crates/zerofrom -** zerotrie; version 0.2.5 -- https://crates.io/crates/zerotrie -** zerovec-derive; version 0.11.6 -- https://crates.io/crates/zerovec-derive -** zerovec; version 0.11.8 -- https://crates.io/crates/zerovec -UNICODE LICENSE V3 - -COPYRIGHT AND PERMISSION NOTICE - -Copyright © 2020-2024 Unicode, Inc. - -NOTICE TO USER: Carefully read the following legal agreement. BY -DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR -SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE -TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT -DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. - -Permission is hereby granted, free of charge, to any person obtaining a -copy of data files and any associated documentation (the "Data Files") or -software and any associated documentation (the "Software") to deal in the -Data Files or Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, and/or sell -copies of the Data Files or Software, and to permit persons to whom the -Data Files or Software are furnished to do so, provided that either (a) -this copyright and permission notice appear with all copies of the Data -Files or Software, or (b) this copyright and permission notice appear in -associated Documentation. - -THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF -THIRD PARTY RIGHTS. - -IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE -BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, -OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, -WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, -ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA -FILES OR SOFTWARE. - -Except as contained in this notice, the name of a copyright holder shall -not be used in advertising or otherwise to promote the sale, use or other -dealings in these Data Files or Software without prior written -authorization of the copyright holder. - -SPDX-License-Identifier: Unicode-3.0 - -— - -Portions of ICU4X may have been adapted from ICU4C and/or ICU4J. -ICU 1.8.1 to ICU 57.1 © 1995-2016 International Business Machines Corporation and others. - - ------- -** unicode_names2; version 1.3.0 -- https://crates.io/crates/unicode_names2 +** ruff_unicode_names2; version 0.0.21 -- https://crates.io/crates/ruff_unicode_names2 UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE Unicode Data Files include all data files under the directories http://www.unicode.org/Public/, http://www.unicode.org/reports/, http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and http://www.unicode.org/utility/trac/browser/. diff --git a/reports/sessions-bindings-quality-evaluation-report.md b/reports/sessions-bindings-quality-evaluation-report.md index 08b3fe47..5bfb38e4 100644 --- a/reports/sessions-bindings-quality-evaluation-report.md +++ b/reports/sessions-bindings-quality-evaluation-report.md @@ -240,7 +240,7 @@ a Python class in `_v1/_session.py` that holds a `_RustSession` and adds: |---|---|---|---| | `Session.__init__(revision_extensions=...)` | accepted **and used** | accepted, never forwarded | `get_enabled_extensions()` always returns `[]` | | `exit_environment(keep_session_running=...)` | `False` | `True` | Default flip — silent semantic regression for callers who omit the kwarg | -| `run_subprocess(timeout=...)` | int>0; ValueError on 0/None handling | `float(timeout) if timeout else None` | `timeout=0` silently disables the timeout | +| `run_subprocess(timeout=...)` | int>0; ValueError on 0/None handling | `None if timeout is None else float(timeout)` | ✓ resolved: `timeout=0` reaches the crate and raises | | `run_task(task_parameter_values: TaskParameterSet)` | required positional / kw | required kw — but type-coerces dicts and ParameterValue instances inconsistently | Possible bad-value paths | ### 3.4 Missing wrapper API @@ -346,12 +346,12 @@ counted as test-suite failures. | `ActionResult` | not in reference | new pyclass, Python-constructible via `#[new]`, pickleable | ✓ resolved (Rec #11) | | `Session.__init__(callback=...)` | accepted | accepted only by wrapper, NOT by binding | ⚠ binding gap | | `Session.__init__(revision_extensions=...)` | accepted, used | accepted by wrapper, **not forwarded** | ❌ silent regression | -| `Session.cancel_action()` | cancels a running action | raises `RuntimeError("session is busy")` while running | ❌ correctness regression | +| `Session.cancel_action()` | cancels a running action | cancels it, also straight after the call that started it | ✓ resolved | | `Session.__enter__/__exit__` | implemented | not implemented on wrapper | ❌ missing | | `Session.session_id` (property) | not exposed in ref either; spec says yes | not on wrapper; on binding only | ⚠ matches ref | -| `Session.run_subprocess(timeout=0)` | raises `ValueError` | silently runs without timeout (wrapper: `if timeout else None`) | ❌ correctness regression | -| `Session.run_subprocess(timeout=-5)` | raises `ValueError` | panics in background thread; action wedges | ❌ correctness regression | -| `Session.run_subprocess(command="")` | raises `ValueError` | accepts; action wedges with `action_status=None` | ❌ correctness regression | +| `Session.run_subprocess(timeout=0)` | raises `ValueError` | raises `SessionError("timeout must be positive")`, the crate's refusal | ⚠ class change | +| `Session.run_subprocess(timeout=-5)` | raises `ValueError` | binding raises `ValueError` | ✓ resolved | +| `Session.run_subprocess(command="")` | raises `ValueError` | binding raises `SessionError("command must be a non-empty string")`, the crate's refusal | ⚠ class change | | `Session.run_task(task_parameter_values)` | required (`TaskParameterSet`) | wrapper makes it required, but accepts dicts of dicts | ⚠ silent type drift | | `Session.exit_environment(keep_session_running=False)` | default `False` | wrapper default `True` | ❌ silent default flip | | `Session.environments_entered` (property) | `tuple[str, ...]` | binding returns `list[str]`; wrapper returns `tuple` | ✓ wrapper matches | @@ -369,12 +369,16 @@ counted as test-suite failures. | Failure | Reference | Binding | Status | |---|---|---|---| -| `run_task` while session not READY | `RuntimeError("Session must be in the READY state to run a task.")` | `RuntimeError("An action is already running")` | ⚠ message change | -| `cancel_action` when nothing is running | `RuntimeError("No actions are running")` | `SessionError("Session must be in RUNNING state, current: READY")` | ⚠ class+message change | -| `cancel_action` while running | succeeds | `RuntimeError("Cannot cancel: session is busy with an action")` | ❌ contract inversion | +| `run_task` while session not READY | `RuntimeError("Session must be in the READY state to run a task.")` | `SessionError("Session must be in READY state, current: RUNNING")` (the crate's message; a `RuntimeError`) | ⚠ message change | +| `run_task` / `enter_environment` on a `READY_ENDING` session | `RuntimeError("Session must be in the READY state ...")` | `SessionError("Session must be in READY state, current: READY_ENDING")`, a `RuntimeError`; nothing changes | ⚠ message change (the crate's) | +| `cancel_action` while a cancel is in progress | accepted; shortens the remaining grace period to at most the new `time_limit` | `SessionError("Session must be in RUNNING state, current: CANCELING")`, the crate's refusal; the crate can't shorten a cancel in progress | ⚠ behavior change | +| `Session.clear_ending_only()` | not in reference | takes a `READY_ENDING` session back to `READY`, as the crate's `clear_ending_only` does | ⚠ binding-only API | +| `WindowsSessionUser(...)` when the process user can't be looked up | `get_process_user()` raises | raises `RuntimeError("Could not determine the process user: ...")` (before, treated as "not the process user") | ⚠ class change | +| `cancel_action` when nothing is running | `RuntimeError("No actions are running")` | `SessionError("Session must be in RUNNING state, current: READY")` | ⚠ class+message change (the crate's) | +| `cancel_action` while running | succeeds | succeeds, including straight after the call that started the action | ✓ resolved | | `Session(session_root_directory=…)` non-existent dir | `RuntimeError("Ensure that the root directory ...")` | likely Rust I/O error → `SessionError(...)` (untested) | ⚠ untested | -| `enter_environment` with same identifier twice | `RuntimeError("Environment {id} has already been entered ...")` | not verified | ⚠ untested | -| `exit_environment` with unknown id | `RuntimeError("Cannot exit unknown Environment ...")` | not verified | ⚠ untested | +| `enter_environment` with same identifier twice | `RuntimeError("Environment {id} has already been entered ...")` | `SessionError` with the same message | ✓ tested | +| `exit_environment` with unknown id | `RuntimeError("Cannot exit unknown Environment ...")` | `SessionError("Unknown environment identifier: {id}")` | ⚠ message change (the crate's; tested) | ### 5.3 Underlying Rust crate — `openjd-rs` @@ -471,6 +475,9 @@ All 17 are reproducible failures of the binding/wrapper. `test_known_gaps::test_cancel_action_actually_cancels_running_action`. Reference fix path: thread the Rust crate's `CancellationToken` to a `cancel()` method that does not need `&mut Session`. **High priority.** + **Resolved** — the binding cancels through the crate's + `SessionCancelHandle`, and retries for the moment between the crate + reporting `Running` and installing the action's cancel token. 2. **`run_subprocess` skips all input validation.** * `timeout=0` silently disables (wrapper bug at `_session.py` line in @@ -481,6 +488,10 @@ All 17 are reproducible failures of the binding/wrapper. * `command=""` — same wedge: the action sits at RUNNING with `action_status=None`. Reference's reference behavior is `ValueError` for all three. **High priority.** + **Partly resolved** — the binding raises `SessionError` with the crate's + refusal for a blank `command` and a zero `timeout` (given to the binding), + and `ValueError` for a negative, NaN or infinite `timeout`. The `_v1` + wrapper's `if timeout else None` is fixed in openjd-sessions-for-python. 3. **State enums lost `.value`.** `SessionState.READY.value` → `AttributeError`. The new `bindings-rs` test_session_scenarios.py uses `state.value` and is @@ -534,12 +545,14 @@ Numbered for the report-driven workflow in `~/openjd-rs/AGENTS.md`. Strike throu ### Priority 1 — correctness regressions (fix before any caller migration) -1. Make `Session.cancel_action()` work while an action is running. Wire the +1. ~~Make `Session.cancel_action()` work while an action is running. Wire the Rust crate's `CancellationToken` through to a binding-owned handle so cancel does not require `&mut Session`. Reference test: `test/openjd/sessions-v1/test_known_gaps.py::test_cancel_action_actually_cancels_running_action`. Files: `rust-bindings/src/sessions/session.rs` lines 446–459; openjd-rs - `openjd-sessions/src/session.rs` cancel surface. + `openjd-sessions/src/session.rs` cancel surface.~~ **Resolved** — through + `SessionCancelHandle`, including straight after the call that started the + action (`test/openjd/sessions/test_refused_calls.py::TestCancelAction`). 2. Validate `run_subprocess(timeout=...)` and `run_subprocess(command=...)` arguments at the **wrapper** boundary so timeout=0/negative and empty command raise `ValueError` matching reference. Reference tests: @@ -547,12 +560,16 @@ Numbered for the report-driven workflow in `~/openjd-rs/AGENTS.md`. Strike throu `::test_run_subprocess_rejects_negative_timeout`, `::test_run_subprocess_rejects_empty_command`. Files: `openjd-sessions-for-python/src/openjd/sessions/_v1/_session.py` `Session.run_subprocess` (replace `if timeout else None` with explicit - `is None` check; add empty-command check at top). + `is None` check; add empty-command check at top). **Partly resolved** — + the wrapper passes `timeout=0` through and the binding raises the crate's + `SessionError` for a zero timeout and a blank command (not `ValueError`). -3. In Rust, make negative `timeout` reject cleanly instead of panicking: +3. ~~In Rust, make negative `timeout` reject cleanly instead of panicking: replace `Duration::from_secs_f64(t).unwrap()` with a check that returns a `SessionError`. File: `rust-bindings/src/sessions/session.rs` line 425 - (`run_subprocess`'s `let duration = timeout.map(std::time::Duration::from_secs_f64);`). + (`run_subprocess`'s `let duration = timeout.map(std::time::Duration::from_secs_f64);`).~~ + **Resolved** — a negative, NaN or infinite `timeout` or `time_limit` raises + `ValueError`. 4. Fix `exit_environment` default `keep_session_running=False` to match reference. Reference test: diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml index 3306161c..7136b91d 100644 --- a/rust-bindings/Cargo.toml +++ b/rust-bindings/Cargo.toml @@ -12,9 +12,9 @@ name = "_openjd_rs" crate-type = ["cdylib", "rlib"] [dependencies] -openjd-expr = "0.12.0" -openjd-model = "0.13.0" -openjd-sessions = "0.8.1" +openjd-expr = "0.15.0" +openjd-model = "0.16.0" +openjd-sessions = "0.10.0" tokio = { version = "1", features = ["rt-multi-thread"] } uuid = { version = "1", features = ["v4"] } serde_json = "1" diff --git a/rust-bindings/src/expr/expr_value.rs b/rust-bindings/src/expr/expr_value.rs index 9aa1eb58..0d093135 100644 --- a/rust-bindings/src/expr/expr_value.rs +++ b/rust-bindings/src/expr/expr_value.rs @@ -331,7 +331,7 @@ impl PyExprValue { fn __len__(&self) -> PyResult { match &self.inner { - ExprValue::RangeExpr(r) => Ok(r.len()), + ExprValue::RangeExpr(r) => crate::len_to_usize(r.len()), _ => self.inner.list_len().ok_or_else(|| { pyo3::exceptions::PyTypeError::new_err("ExprValue is not a list or range_expr") }), diff --git a/rust-bindings/src/expr/path_mapping.rs b/rust-bindings/src/expr/path_mapping.rs index 77a8c6b0..98560bd2 100644 --- a/rust-bindings/src/expr/path_mapping.rs +++ b/rust-bindings/src/expr/path_mapping.rs @@ -6,7 +6,7 @@ use pyo3::types::PyDict; #[cfg(feature = "stub-gen")] use pyo3_stub_gen::derive::*; -use openjd_expr::path_mapping::{PathFormat, PathMappingRule}; +use openjd_expr::path_mapping::{PathFormat, PathMapper, PathMappingRule}; use crate::expr::path_format::PyPathFormat; @@ -167,16 +167,28 @@ impl PyPathMappingRule { h.finish() } + /// Apply this rule to `path`. Returns `(True, mapped)` when the rule + /// matches and `(False, path)` when it does not. Raises `ValueError` + /// when the rule itself is invalid, for example a POSIX or WINDOWS + /// `source_path` that is not absolute. #[pyo3(signature = (*, path, output_format=None))] - fn apply(&self, path: &str, output_format: Option) -> (bool, String) { + fn apply(&self, path: &str, output_format: Option) -> PyResult<(bool, String)> { + // openjd-expr applies rules only through a `PathMapper`, so a + // one-rule mapper gives this rule's mapping. `validate` first so the + // error does not name an index in a list the caller never made. + let invalid = |e: openjd_expr::path_mapping::InvalidPathMappingRule| { + pyo3::exceptions::PyValueError::new_err(e.to_string()) + }; + self.inner.validate().map_err(invalid)?; + let mapper = PathMapper::new(vec![self.inner.clone()]).map_err(invalid)?; let result = match output_format { - Some(fmt) => self.inner.apply_with_format(path, fmt.into()), - None => self.inner.apply(path), + Some(fmt) => mapper.apply_with_format(path, fmt.into()), + None => mapper.apply(path), }; - match result { + Ok(match result { Some(mapped) => (true, mapped), None => (false, path.to_string()), - } + }) } fn to_dict(&self) -> std::collections::HashMap { diff --git a/rust-bindings/src/expr/profile.rs b/rust-bindings/src/expr/profile.rs index 55f3897f..96791d9a 100644 --- a/rust-bindings/src/expr/profile.rs +++ b/rust-bindings/src/expr/profile.rs @@ -22,6 +22,7 @@ use pyo3::types::{PyDict, PyType}; #[cfg(feature = "stub-gen")] use pyo3_stub_gen::derive::*; +use openjd_expr::path_mapping::PathMapper; use openjd_expr::profile::{ExprExtension, ExprProfile, ExprRevision, HostContext}; use crate::expr::path_mapping::PyPathMappingRule; @@ -237,13 +238,19 @@ impl PyHostContext { /// host context (`apply_path_mapping` is registered, returns /// the path unchanged). Passing zero rules is *not* the same as /// passing no host context at all. + /// + /// Raises `ValueError` when a rule is invalid, for example a POSIX or + /// WINDOWS `source_path` that is not absolute. The message names the + /// rule's index in `rules`. #[classmethod] - fn with_rules(_cls: &Bound<'_, PyType>, rules: Vec) -> Self { + fn with_rules(_cls: &Bound<'_, PyType>, rules: Vec) -> PyResult { let rust_rules: Vec = rules.into_iter().map(|r| r.inner).collect(); - Self { - inner: HostContext::WithRules(Arc::new(rust_rules)), - } + let mapper = PathMapper::new(rust_rules) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; + Ok(Self { + inner: HostContext::WithRules(Arc::new(mapper)), + }) } /// Whether this host context registers any host-context @@ -263,7 +270,7 @@ impl PyHostContext { HostContext::None => "HostContext.none()".to_string(), HostContext::Unresolved => "HostContext.unresolved()".to_string(), HostContext::WithRules(rules) => { - format!("HostContext.with_rules(<{} rule(s)>)", rules.len()) + format!("HostContext.with_rules(<{} rule(s)>)", rules.rules().len()) } } } @@ -306,6 +313,7 @@ impl PyHostContext { HostContext::WithRules(rules) => { let f = cls.getattr("with_rules")?; let py_rules: Vec = rules + .rules() .iter() .cloned() .map(|inner| PyPathMappingRule { inner }) @@ -328,8 +336,9 @@ fn host_context_eq(a: &HostContext, b: &HostContext) -> bool { match (a, b) { (HostContext::None, HostContext::None) => true, (HostContext::Unresolved, HostContext::Unresolved) => true, - (HostContext::WithRules(la), HostContext::WithRules(lb)) => { + (HostContext::WithRules(ma), HostContext::WithRules(mb)) => { // Compare rule-by-rule; `Arc` identity is irrelevant. + let (la, lb) = (ma.rules(), mb.rules()); la.len() == lb.len() && la.iter().zip(lb.iter()).all(|(ra, rb)| { ra.source_path_format == rb.source_path_format @@ -347,7 +356,8 @@ fn host_context_hash(hc: &HostContext) -> u64 { match hc { HostContext::None => 0u8.hash(&mut h), HostContext::Unresolved => 1u8.hash(&mut h), - HostContext::WithRules(rules) => { + HostContext::WithRules(mapper) => { + let rules = mapper.rules(); 2u8.hash(&mut h); (rules.len() as u64).hash(&mut h); for r in rules.iter() { diff --git a/rust-bindings/src/expr/range_expr.rs b/rust-bindings/src/expr/range_expr.rs index da2eca3f..b3900f26 100644 --- a/rust-bindings/src/expr/range_expr.rs +++ b/rust-bindings/src/expr/range_expr.rs @@ -54,8 +54,8 @@ impl PyIntRange { self.inner.step() } - fn __len__(&self) -> usize { - self.inner.len() + fn __len__(&self) -> PyResult { + crate::len_to_usize(self.inner.len()) } fn __contains__(&self, value: i64) -> bool { @@ -205,8 +205,8 @@ impl PyRangeExpr { .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Range expression is empty")) } - fn __len__(&self) -> usize { - self.inner.len() + fn __len__(&self) -> PyResult { + crate::len_to_usize(self.inner.len()) } fn __eq__(&self, other: &PyRangeExpr) -> bool { @@ -218,13 +218,13 @@ impl PyRangeExpr { } fn __getitem__(&self, index: isize) -> PyResult { - let len = self.inner.len(); + let len = i128::from(self.inner.len()); let idx = if index < 0 { - len as isize + index + len + index as i128 } else { - index + index as i128 }; - if idx < 0 || idx as usize >= len { + if idx < 0 || idx >= len { return Err(pyo3::exceptions::PyIndexError::new_err( "index out of range", )); diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index b90f334f..a7f87ca5 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -48,6 +48,18 @@ fn register_renamed_exception( Ok(()) } +/// Convert a `u64` count from the openjd crates to the `usize` that +/// `__len__` must return. The crates keep counts 64-bit on every target, +/// so on a 32-bit target a count above `usize::MAX` raises `OverflowError`, +/// which is what Python's `len()` raises for a length it cannot represent. +pub(crate) fn len_to_usize(len: u64) -> PyResult { + usize::try_from(len).map_err(|_| { + pyo3::exceptions::PyOverflowError::new_err( + "length does not fit in this platform's Py_ssize_t", + ) + }) +} + // ── Module ── #[pymodule] diff --git a/rust-bindings/src/model/create_job_fns.rs b/rust-bindings/src/model/create_job_fns.rs index 1cbea67a..c354a685 100644 --- a/rust-bindings/src/model/create_job_fns.rs +++ b/rust-bindings/src/model/create_job_fns.rs @@ -293,8 +293,23 @@ fn default_to_native( .ok() .map(|v| v.into_py_any(py)) .unwrap_or_else(|| raw.into_py_any(py)), - JobParameterType::ListFloat => serde_json::from_str::>(raw) + // Since openjd-model 0.16.0 (openjd-rs#538) an item written in the + // template as a string, such as `"2.50"`, is a JSON string here so + // that job creation keeps its spelling. A Python `float` can't hold a + // spelling, so read either form as a number, as + // `JobListFloatParameterDefinition.default` does. + JobParameterType::ListFloat => serde_json::from_str::>(raw) .ok() + .and_then(|items| { + items + .iter() + .map(|item| match item { + serde_json::Value::Number(n) => n.as_f64(), + serde_json::Value::String(s) => s.trim().parse::().ok(), + _ => None, + }) + .collect::>>() + }) .map(|v| v.into_py_any(py)) .unwrap_or_else(|| raw.into_py_any(py)), JobParameterType::ListBool => serde_json::from_str::>(raw) diff --git a/rust-bindings/src/model/step_param_space.rs b/rust-bindings/src/model/step_param_space.rs index 5f0df8e7..f5618441 100644 --- a/rust-bindings/src/model/step_param_space.rs +++ b/rust-bindings/src/model/step_param_space.rs @@ -109,13 +109,13 @@ pub(crate) struct PyStepParameterSpaceIterator { space: StepParameterSpace, /// Cached length captured at construction time. Used by `__len__` /// for non-adaptive spaces (adaptive spaces raise instead). - len: usize, + len: u64, /// Cached parameter names captured at construction time. names: HashSet, /// The persistent iterator that backs `__next__`, `reset_iter`, /// `__contains__`, and `chunks_default_task_count` (getter and /// setter). Holding it across calls is what lets the setter - /// actually mutate state — the underlying `Arc` for + /// actually mutate state — the underlying `Arc` for /// adaptive chunking lives inside this iterator. /// /// `Mutex` is required because pyclass types must be `Sync`. The @@ -127,7 +127,7 @@ pub(crate) struct PyStepParameterSpaceIterator { /// so `__getitem__` can rebuild an equivalent iterator. Without it, /// random access would silently fall back to the template's own /// `defaultTaskCount` and disagree with iteration. - chunk_override: Option, + chunk_override: Option, } #[cfg_attr(feature = "stub-gen", gen_stub_pymethods)] @@ -153,20 +153,20 @@ impl PyStepParameterSpaceIterator { space: Option<&PyStepParameterSpace>, chunks_task_count_override: Option, ) -> PyResult { - // Taken as i64, not usize, so a negative value reaches this check instead of + // Taken as i64, not u64, so a negative value reaches this check instead of // failing pyo3's unsigned extraction with OverflowError. Every non-positive // value should report the same ValueError the message and the spec promise. // // Rejected rather than clamped: openjd-model applies `.max(1)` to the override, // so 0 would silently mean 1. The `chunks_default_task_count` setter already // refuses 0, and the two should not disagree. - let chunks_task_count_override: Option = match chunks_task_count_override { + let chunks_task_count_override: Option = match chunks_task_count_override { Some(n) if n <= 0 => { return Err(pyo3::exceptions::PyValueError::new_err( "chunks_task_count_override must be a positive integer.", )); } - Some(n) => Some(n as usize), + Some(n) => Some(n as u64), None => None, }; let ps = if let Some(s) = space { @@ -210,7 +210,7 @@ impl PyStepParameterSpaceIterator { "Length is not available because the parameter space uses adaptive chunking.", )); } - Ok(self.len) + crate::len_to_usize(self.len) } fn __getitem__(&self, py: Python<'_>, index: isize) -> PyResult> { @@ -235,15 +235,15 @@ impl PyStepParameterSpaceIterator { )); } let idx = if index < 0 { - let adjusted = self.len as isize + index; + let adjusted = i128::from(self.len) + index as i128; if adjusted < 0 { return Err(pyo3::exceptions::PyIndexError::new_err( "index out of range", )); } - adjusted as usize + adjusted as u64 } else { - index as usize + index as u64 }; match iter.get(idx) { Some(params) => task_param_set_to_py(py, ¶ms), @@ -312,13 +312,13 @@ impl PyStepParameterSpaceIterator { } #[getter] - fn chunks_default_task_count(&self) -> Option { + fn chunks_default_task_count(&self) -> Option { let iter = lock_recover(&self.iter); iter.chunks_default_task_count() } #[setter] - fn set_chunks_default_task_count(&self, value: usize) -> PyResult<()> { + fn set_chunks_default_task_count(&self, value: u64) -> PyResult<()> { if value == 0 { return Err(pyo3::exceptions::PyValueError::new_err( "chunks_default_task_count must be a positive integer.", diff --git a/rust-bindings/src/model/step_param_space_def.rs b/rust-bindings/src/model/step_param_space_def.rs index 48aa9f7f..bace5b16 100644 --- a/rust-bindings/src/model/step_param_space_def.rs +++ b/rust-bindings/src/model/step_param_space_def.rs @@ -10,7 +10,7 @@ //! //! Five typed pyclasses mirror the `TaskParameterDefinition` variants: //! -//! * `IntTaskParameterDefinition` — `range: list[int] | FormatString` +//! * `IntTaskParameterDefinition` — `range: list[int | FormatString] | FormatString` //! * `FloatTaskParameterDefinition` — `range: list[float | FormatString] | FormatString` //! * `StringTaskParameterDefinition` — `range: list[FormatString] | FormatString` //! * `PathTaskParameterDefinition` — `range: list[FormatString] | FormatString` @@ -76,14 +76,14 @@ fn float_range_item_to_py<'py>(py: Python<'py>, v: &FloatRangeItem) -> PyResult< } } -/// Convert an `IntRange` to a Python value (`list[int]` or -/// `FormatString`). +/// Convert an `IntRange` to a Python value (`list[int | FormatString]` +/// or `FormatString`). fn int_range_to_py<'py>(py: Python<'py>, r: &IntRange) -> PyResult> { match r { IntRange::List(items) => { let py_list = PyList::empty(py); for item in items { - py_list.append(item.0)?; + py_list.append(int_or_fs_to_py(py, item)?)?; } Ok(py_list.into_any()) } diff --git a/rust-bindings/src/model/task_parameter.rs b/rust-bindings/src/model/task_parameter.rs index 57cc1868..54327c36 100644 --- a/rust-bindings/src/model/task_parameter.rs +++ b/rust-bindings/src/model/task_parameter.rs @@ -73,8 +73,8 @@ impl PyTaskChunksDefinition { #[new] #[pyo3(signature = (*, default_task_count, target_runtime_seconds=None, range_constraint))] fn new( - default_task_count: usize, - target_runtime_seconds: Option, + default_task_count: u64, + target_runtime_seconds: Option, range_constraint: &str, ) -> PyResult { let constraint = match range_constraint { @@ -96,12 +96,12 @@ impl PyTaskChunksDefinition { } #[getter] - fn default_task_count(&self) -> usize { + fn default_task_count(&self) -> u64 { self.inner.default_task_count } #[getter] - fn target_runtime_seconds(&self) -> Option { + fn target_runtime_seconds(&self) -> Option { self.inner.target_runtime_seconds } diff --git a/rust-bindings/src/sessions/session.rs b/rust-bindings/src/sessions/session.rs index 61e3c492..9002cfc7 100644 --- a/rust-bindings/src/sessions/session.rs +++ b/rust-bindings/src/sessions/session.rs @@ -12,7 +12,9 @@ use pyo3_stub_gen::derive::*; use openjd_model::types::{JobParameterType, JobParameterValue, JobParameterValues}; use openjd_model::types::{TaskParameterSet, TaskParameterType, TaskParameterValue}; +use openjd_sessions::action::ActionState; use openjd_sessions::action_status::ActionStatus; +use openjd_sessions::error::SessionError; use openjd_sessions::limits::SessionLimits; use openjd_sessions::session::{Session, SessionConfig, SessionState}; @@ -143,9 +145,9 @@ fn lock_recover(m: &Mutex) -> std::sync::MutexGuard<'_, T> { /// Build a terminal FAILED `ActionStatus` carrying `message`. /// -/// Used when an action thread can't even start (e.g. the Tokio runtime -/// fails to build) so the Python poller observes a terminal state and -/// stops waiting on RUNNING rather than spinning forever. +/// Used when an accepted action panics, or returns an error the crate +/// didn't record, so the Python poller observes a terminal state and stops +/// waiting on RUNNING rather than spinning forever. fn failed_action_status(message: &str) -> ActionStatus { ActionStatus { state: openjd_sessions::action::ActionState::Failed, @@ -158,6 +160,87 @@ fn failed_action_status(message: &str) -> ActionStatus { } } +/// Convert a timeout or time limit given in seconds. Zero passes through to +/// the crate, which refuses a zero `run_subprocess` timeout and reads a zero +/// cancel time limit as "terminate now". A negative, NaN or infinite value +/// has no `Duration`, so it raises `ValueError` here rather than panicking. +fn seconds_to_duration(name: &str, seconds: f64) -> PyResult { + std::time::Duration::try_from_secs_f64(seconds).map_err(|_| { + pyo3::exceptions::PyValueError::new_err(format!( + "{name} must be a finite, non-negative number of seconds, got {seconds}" + )) + }) +} + +/// What an action thread tells the Python thread that handed it a call. +/// +/// A non-blocking method must not return until it knows whether the crate +/// accepted the call. The crate refuses a call (wrong state, unknown or +/// duplicate environment, out-of-order exit, bad `run_subprocess` +/// arguments) before it touches the session, and such a refusal has to +/// raise. It accepts a call by reporting the call's action through the +/// session callback, before its first `.await`; from then on every outcome, +/// errors included, is recorded on the action status. +enum CallOutcome { + /// The crate accepted the call and its action is `Running`. + Started, + /// The crate returned before the action reported `Running`. + /// `accepted == false` means it refused the call and changed nothing, so + /// `error` is the refusal. Otherwise the action ended without running a + /// subprocess (an environment with no script, or a failure before the + /// action started), and its status says how. + Returned { + accepted: bool, + error: Option, + }, +} + +/// The call an action thread is running, until the Python thread that made +/// it has been told the outcome. +struct PendingCall { + /// Tells this call apart from the next one in the same slot. + id: u64, + tx: std::sync::mpsc::SyncSender, + /// Set by the session callback: the crate reported an action for this + /// call, so it accepted it. + accepted: bool, +} + +type PendingSlot = Arc>>; + +static NEXT_CALL_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// The thread running a session's current action, so `cancel_action` can +/// tell whether it is called from that thread. +type ActionThread = Arc>>; + +/// Held by an action thread for its whole life. If the thread ends without +/// having told its caller the outcome (it can't today, but a panic outside +/// `catch_unwind` would), dropping the call's sender wakes the caller instead +/// of leaving it waiting for ever. Also clears the thread's entry in +/// `ActionThread`. Each slot is cleared only while it still names this call +/// or thread, not a later one. +struct ActionThreadGuard { + pending: PendingSlot, + call_id: u64, + action_thread: ActionThread, +} + +impl Drop for ActionThreadGuard { + fn drop(&mut self) { + { + let mut slot = lock_recover(&self.pending); + if slot.as_ref().is_some_and(|c| c.id == self.call_id) { + slot.take(); + } + } + let mut thread = lock_recover(&self.action_thread); + if *thread == Some(std::thread::current().id()) { + *thread = None; + } + } +} + /// Body of a spawned action thread. /// /// Builds a lightweight current-thread Tokio runtime, runs the action @@ -166,23 +249,28 @@ fn failed_action_status(message: &str) -> ActionStatus { /// the runtime failed to build or the future panicked. This guarantees: /// /// * the session is never left permanently "taken" (which would wedge -/// every later call into `"An action is already running"`), and +/// every later call), and /// * a panic in async session code can't escape the detached thread to /// poison shared state silently and tear down the interpreter's view /// of the session. /// -/// A runtime-build failure is surfaced as a terminal FAILED -/// `ActionStatus` so the Python-side poller terminates cleanly. A panic -/// inside the action is likewise converted into a terminal FAILED -/// `ActionStatus` (rather than copying the possibly-still-Running real -/// session state), so the poller never hangs on a panicked action. +/// A runtime-build failure is a refusal: the session goes back unchanged +/// and the caller raises. A panic inside the action is converted into a +/// terminal FAILED `ActionStatus` (rather than copying the +/// possibly-still-Running real session state), so the poller never hangs +/// on a panicked action. +/// +/// If the calling Python thread is still waiting (the action never +/// reported `Running`), it is told the outcome after the session is back in +/// its slot, so it can make its next call straight away. fn run_action( mut session: Session, session_arc: Arc>>, snapshot: Arc>, + pending: PendingSlot, action: F, ) where - F: FnOnce(&tokio::runtime::Runtime, &mut Session), + F: FnOnce(&tokio::runtime::Runtime, &mut Session) -> Result<(), SessionError>, { let rt = match tokio::runtime::Builder::new_current_thread() .enable_all() @@ -190,33 +278,37 @@ fn run_action( { Ok(rt) => rt, Err(e) => { - // Couldn't build a runtime (e.g. fd/thread exhaustion). Mark - // the action FAILED and drop out of RUNNING so the poller - // delivers a terminal callback instead of hanging. - { - let mut snap = lock_recover(&snapshot); - snap.state = SessionState::Ready; - snap.action_status = Some(failed_action_status(&format!( - "failed to start action runtime: {e}" - ))); - } + // Couldn't build a runtime (e.g. fd/thread exhaustion). The + // crate never saw the call, so treat it as refused: put the + // session back unchanged and raise in the caller. + let call = lock_recover(&pending).take(); *lock_recover(&session_arc) = Some(session); + tell( + call, + false, + Some(SessionError::Runtime(format!( + "failed to start action runtime: {e}" + ))), + ); return; } }; // Catch panics so a panic inside async session code still restores // the session below rather than leaking out of this detached thread. - let action_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - action(&rt, &mut session); - })); + let action_result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| action(&rt, &mut session))); + + // Take the waiting call, if any, before the session goes back: once it + // is back another thread can start a call and install its own. + let call = lock_recover(&pending).take(); + let accepted = call.as_ref().is_none_or(|c| c.accepted); // Restore the session BEFORE refreshing the snapshot, holding the guard // across both so there's no window where a caller finds the slot empty. // Python callers watch `state` via the snapshot and dispatch the next // action the moment it transitions out of Running; if the snapshot - // updated first they could find the mutex still empty ("An action is - // already running"). + // updated first they could find the mutex still empty. let guard = { let mut slot = lock_recover(&session_arc); *slot = Some(session); @@ -224,28 +316,67 @@ fn run_action( }; match action_result { - Ok(()) => { + Ok(result) => { if let Some(s) = guard.as_ref() { PySession::update_snapshot(s, &snapshot); } + // Still under the session guard, so no next action can have + // started and been overwritten. + if accepted { + if let Err(e) = &result { + record_unreported_failure(&snapshot, e); + } + } + drop(guard); + tell(call, accepted, result.err()); } Err(payload) => { // The action panicked mid-run. `catch_unwind` kept the session // usable for future calls, but the real `session.state()` may // still read Running with no terminal `ActionStatus` — which // would leave the Python poller waiting forever. Force a - // terminal FAILED status and drop out of RUNNING, mirroring the - // runtime-build-failure branch above. - let mut snap = lock_recover(&snapshot); - snap.state = SessionState::Ready; - snap.action_status = Some(failed_action_status(&format!( - "action panicked during execution: {}", - panic_detail(payload.as_ref()) - ))); + // terminal FAILED status and drop out of RUNNING. + { + let mut snap = lock_recover(&snapshot); + snap.state = SessionState::Ready; + snap.action_status = Some(failed_action_status(&format!( + "action panicked during execution: {}", + panic_detail(payload.as_ref()) + ))); + } + drop(guard); + tell(call, true, None); } } } +/// Tell a waiting Python thread how its call came out. Nothing to do when +/// it was already told the action started. +fn tell(call: Option, accepted: bool, error: Option) { + if let Some(call) = call { + // `send` fails only when the receiver is gone, and the caller drops + // it only after receiving: there is nobody left to tell. + let _ = call.tx.send(CallOutcome::Returned { accepted, error }); + } +} + +/// An accepted call returned an error. The crate records every such error +/// on the action status (`fail_action_setup`, `drive_action`, or a failed +/// script's own status), so this normally changes nothing. If the status +/// doesn't show a failure, record the error there rather than lose it. +fn record_unreported_failure(snapshot: &Arc>, e: &SessionError) { + let mut snap = lock_recover(snapshot); + let reported = snap.action_status.as_ref().is_some_and(|st| { + matches!( + st.state, + ActionState::Failed | ActionState::Canceled | ActionState::Timeout + ) + }); + if !reported { + snap.action_status = Some(failed_action_status(&e.to_string())); + } +} + /// Best-effort extraction of a human-readable message from a caught panic /// payload (`std::panic::catch_unwind`'s `Err` value), which is usually the /// `&str` or `String` passed to `panic!`. @@ -263,6 +394,11 @@ fn panic_detail(payload: &(dyn std::any::Any + Send)) -> String { #[pyclass(module = "openjd.sessions._v1", name = "Session")] pub(crate) struct PySession { /// The Rust session. None while a background thread has taken ownership. + /// + /// Lock it only with the GIL released (`Python::detach`). `cleanup` and + /// `cancel_action` call into the crate while holding it, the crate logs, + /// and `pyo3_log` takes the GIL for each record, so a thread that held + /// the GIL while waiting for this lock would deadlock with them. session: Arc>>, /// Snapshot updated after each state change, readable without blocking on the session. snapshot: Arc>, @@ -270,7 +406,12 @@ pub(crate) struct PySession { /// to whichever action is running even while the `Session` value is owned /// by a background action thread — the case where `Session::cancel_action` /// is unreachable (it needs `&mut Session`). - cancel_handle: openjd_sessions::session::SessionCancelHandle, + cancel_handle: Arc, + /// The call whose caller is waiting to learn whether the crate accepted + /// it. Shared with the session callback, which marks it accepted. + pending: PendingSlot, + /// The thread running the current action, if any. + action_thread: ActionThread, } impl PySession { @@ -280,6 +421,100 @@ impl PySession { snap.action_status = session.action_status(); snap.environments_entered = session.environments_entered().to_vec(); } + + /// The error for a call made while an action thread owns the session. + /// The crate itself would refuse it with `InvalidState`, because the + /// session is `Running` or `Canceling`; give the same message. Until the + /// action reports `Running` the snapshot may still show the state + /// before it, so anything else reads as `Running`. + fn busy_error(&self, expected: &[SessionState]) -> PyErr { + let current = match lock_recover(&self.snapshot).state { + s @ (SessionState::Running | SessionState::Canceling) => s, + _ => SessionState::Running, + }; + session_err_to_py(SessionError::InvalidState { + expected: expected.to_vec(), + current, + }) + } + + /// Run `action` on a background thread and return once the crate has + /// accepted it and its action is `Running`, or once it has returned. + /// A call the crate refuses raises `SessionError` with the crate's + /// message and leaves the session, its state and its action status as + /// they were. Every later outcome of an accepted call is reported on the + /// action status. The GIL is released while waiting, because the action + /// thread needs it to log. + fn dispatch(&self, py: Python<'_>, expected: &[SessionState], action: F) -> PyResult<()> + where + F: FnOnce(&tokio::runtime::Runtime, &mut Session) -> Result<(), SessionError> + + Send + + 'static, + { + // The session mutex is only ever locked with the GIL released; see + // `PySession::session`. + let session_arc = self.session.clone(); + let Some(session) = py.detach(move || lock_recover(&session_arc).take()) else { + return Err(self.busy_error(expected)); + }; + let (tx, rx) = std::sync::mpsc::sync_channel(1); + let call_id = NEXT_CALL_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + *lock_recover(&self.pending) = Some(PendingCall { + id: call_id, + tx, + accepted: false, + }); + + // `run_action` owns the session from here. If the thread can't be + // spawned, the closure is dropped unrun, so keep the session where + // it can be recovered, put it back, and raise, as for a refusal. + let session_slot = Arc::new(Mutex::new(Some(session))); + let for_thread = session_slot.clone(); + let session_arc = self.session.clone(); + let snapshot = self.snapshot.clone(); + let pending = self.pending.clone(); + let action_thread = self.action_thread.clone(); + let spawned = std::thread::Builder::new() + .name("openjd-session-action".into()) + .spawn(move || { + *lock_recover(&action_thread) = Some(std::thread::current().id()); + let _guard = ActionThreadGuard { + pending: pending.clone(), + call_id, + action_thread, + }; + let Some(session) = lock_recover(&for_thread).take() else { + return; + }; + run_action(session, session_arc, snapshot, pending, action) + }); + if let Err(e) = spawned { + *lock_recover(&self.pending) = None; + if let Some(session) = lock_recover(&session_slot).take() { + let session_arc = self.session.clone(); + py.detach(move || *lock_recover(&session_arc) = Some(session)); + } + return Err(session_err_to_py(SessionError::Runtime(format!( + "failed to start the action thread: {e}" + )))); + } + + match py.detach(move || rx.recv()) { + Ok(CallOutcome::Returned { + accepted: false, + error: Some(e), + }) => Err(session_err_to_py(e)), + // `Err`: the action thread ended without saying, which + // `ActionThreadGuard` turns into a wake-up. Nothing does that + // today; if it did, the session may not be back in its slot. + Err(_) => Err(session_err_to_py(SessionError::Runtime( + "the action thread ended unexpectedly".into(), + ))), + // Started, or an accepted call that returned (its status says + // how). + Ok(_) => Ok(()), + } + } } #[cfg_attr(feature = "stub-gen", gen_stub_pymethods)] @@ -345,9 +580,27 @@ impl PySession { environments_entered: vec![], })); let snapshot_for_cb = snapshot.clone(); + let pending: PendingSlot = Arc::new(Mutex::new(None)); + let pending_for_cb = pending.clone(); + // The crate calls this for every action it accepts before the + // action's first `.await`, so the first call for a pending call is + // its acceptance. On `Running`, the caller is told the action + // started; `state` is set here, under the snapshot lock and before + // the action thread's final `update_snapshot`, so it can't + // overwrite a later state. Lock order: snapshot, then pending. let callback = Box::new(move |_session_id: &str, status: &ActionStatus| { - if let Ok(mut snap) = snapshot_for_cb.lock() { - snap.action_status = Some(status.clone()); + let mut snap = lock_recover(&snapshot_for_cb); + snap.action_status = Some(status.clone()); + let mut slot = lock_recover(&pending_for_cb); + if let Some(call) = slot.as_mut() { + call.accepted = true; + if status.state == ActionState::Running { + snap.state = SessionState::Running; + if let Some(call) = slot.take() { + // Fails only if the caller is gone; see `tell`. + let _ = call.tx.send(CallOutcome::Started); + } + } } }); @@ -377,11 +630,13 @@ impl PySession { snap.files_directory = session.files_directory().to_string_lossy().to_string(); snap.action_status = session.action_status(); } - let cancel_handle = session.cancel_handle(); + let cancel_handle = Arc::new(session.cancel_handle()); Ok(PySession { session: Arc::new(Mutex::new(Some(session))), snapshot, cancel_handle, + pending, + action_thread: Arc::new(Mutex::new(None)), }) } @@ -422,8 +677,9 @@ impl PySession { /// Extend the session's path mapping rules with additional rules. /// - /// Rules are stored sorted by source-path length (longest first) so that - /// the most specific rule matches first during FormatString resolution. + /// When several rules match a path, the one whose source path has the + /// most names wins, and between rules for the same location the + /// one given last wins, wherever it sits in the list. /// /// Consumers (notably the Deadline Cloud worker agent's Job Attachments /// download action) call this after an action assigns additional path @@ -431,30 +687,56 @@ impl PySession { /// /// Returns an error if an action is currently in-flight (the underlying /// Session has been taken by a background thread). Call this between - /// actions. + /// actions. Raises `SessionError`, leaving the rules unchanged, when an + /// added rule is invalid, for example a POSIX or WINDOWS `source_path` + /// that is not absolute. fn extend_path_mapping_rules( &self, + py: Python<'_>, additional: Vec, ) -> PyResult<()> { - let mut guard = lock_recover(&self.session); - let session = guard.as_mut().ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err( - "cannot extend path mapping rules while an action is running", - ) - })?; let rules: Vec = additional.into_iter().map(|p| p.inner).collect(); - session.extend_path_mapping_rules(rules); - Ok(()) + let session_arc = self.session.clone(); + let result = py.detach(move || { + lock_recover(&session_arc) + .as_mut() + .map(|session| session.extend_path_mapping_rules(rules)) + }); + match result { + Some(r) => r.map_err(session_err_to_py), + None => Err(pyo3::exceptions::PyRuntimeError::new_err( + "cannot extend path mapping rules while an action is running", + )), + } } // ── Non-blocking action methods ── + // + // Each returns once its action is `Running` (or, for an action that + // runs no subprocess, once it has ended) and leaves the rest to a + // background thread. A call the crate refuses raises `SessionError` + // with the crate's message, as the Rust method returns `Err`, and + // changes nothing; that includes a call while another action is + // running or the session is ending-only. Once the crate has accepted a + // call, its outcome, including an error before the action's subprocess + // starts, is on `action_status`. - /// Enter an environment. Non-blocking — spawns the onEnter action on a - /// background thread and returns the environment identifier immediately. + /// Enter an environment. Non-blocking: returns the environment + /// identifier once the onEnter action is running. + /// + /// Raises `SessionError` if the session isn't `READY`, if + /// `identifier` is already entered, or if a second environment with a + /// wrap hook is entered. A failure after that, such as an environment + /// variable that doesn't resolve, fails the action; the environment + /// still counts as entered and must be exited. + /// + /// Waits for the action thread, which logs through Python's `logging`, + /// so don't call it while holding a lock that a logging handler takes. #[pyo3(signature = (*, environment, identifier=None, resolved_symtab=None, os_env_vars=None))] fn enter_environment( &self, + py: Python<'_>, environment: &PyEnvironment, identifier: Option, resolved_symtab: Option<&crate::expr::PySerializedSymbolTable>, @@ -478,86 +760,62 @@ impl PySession { // evaluation or expression interpolation. let resolved = resolved_symtab.map(|st| st.inner.clone()); - // Take the session out of the mutex so the background thread owns it - let mut guard = lock_recover(&self.session); - let session = guard.take().ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("An action is already running") + self.dispatch(py, &[SessionState::Ready], move |rt, session| { + rt.block_on(session.enter_environment( + &env, + resolved.as_ref(), + Some(&env_id), + os_env_vars.as_ref(), + )) + .map(|_| ()) })?; - drop(guard); - - // Update snapshot to RUNNING and CLEAR action_status. - // The worker's poll loop treats a non-None action_status as "the current - // action has reported state". If we leave the previous action's status - // here, the poll loop will immediately fire a stale terminal callback - // for the new action — causing the worker to think envEnter / run_task - // finished in ~1ms with the previous action's SUCCEEDED result. - { - let mut snap = lock_recover(&self.snapshot); - snap.state = SessionState::Running; - snap.action_status = None; - } - - let session_arc = self.session.clone(); - let snapshot = self.snapshot.clone(); - - std::thread::spawn(move || { - run_action(session, session_arc, snapshot, move |rt, session| { - let env_ref = os_env_vars.as_ref(); - let _ = rt.block_on(session.enter_environment( - &env, - resolved.as_ref(), - Some(&env_id), - env_ref, - )); - }); - }); - Ok(return_id) } - /// Exit an environment. Non-blocking — spawns the onExit action on a - /// background thread. + /// Exit an environment. Non-blocking: returns once the onExit action is + /// running. + /// + /// Raises `SessionError` if the session isn't `READY` or + /// `READY_ENDING`, if `identifier` isn't entered, or if it isn't the + /// most recently entered environment. A failure after that fails the + /// action, and the environment still counts as exited. + /// + /// Waits for the action thread, which logs through Python's `logging`, + /// so don't call it while holding a lock that a logging handler takes. #[pyo3(signature = (*, identifier, resolved_symtab=None, keep_session_running=true, os_env_vars=None))] fn exit_environment( &self, + py: Python<'_>, identifier: String, resolved_symtab: Option<&crate::expr::PySerializedSymbolTable>, keep_session_running: bool, os_env_vars: Option>, ) -> PyResult<()> { let resolved = resolved_symtab.map(|st| st.inner.clone()); - - let mut guard = lock_recover(&self.session); - let session = guard.take().ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("An action is already running") - })?; - drop(guard); - - { - let mut snap = lock_recover(&self.snapshot); - snap.state = SessionState::Running; - snap.action_status = None; - } - - let session_arc = self.session.clone(); - let snapshot = self.snapshot.clone(); - - std::thread::spawn(move || { - run_action(session, session_arc, snapshot, move |rt, session| { - let env_ref = os_env_vars.as_ref(); - let _ = rt.block_on(session.exit_environment( + self.dispatch( + py, + &[SessionState::Ready, SessionState::ReadyEnding], + move |rt, session| { + rt.block_on(session.exit_environment( &identifier, resolved.as_ref(), keep_session_running, - env_ref, - )); - }); - }); - - Ok(()) + os_env_vars.as_ref(), + )) + .map(|_| ()) + }, + ) } - /// Run a task. Non-blocking — spawns the onRun action on a background thread. + /// Run a task. Non-blocking: returns once the onRun action is running. + /// + /// Raises `SessionError` if the session isn't `READY`, for example + /// because an earlier action failed and the session is ending-only. A + /// failure after that, such as a task parameter value that can't be + /// coerced, fails the action. + /// + /// Waits for the action thread, which logs through Python's `logging`, + /// so don't call it while holding a lock that a logging handler takes. /// /// `step_name` is surfaced as `WrappedStep.Name` to a wrapping /// environment's `onWrapTaskRun` hook (RFC 0008). It defaults to an empty @@ -565,6 +823,7 @@ impl PySession { #[pyo3(signature = (*, step_script, step_name=None, task_parameter_values=None, resolved_symtab=None, os_env_vars=None))] fn run_task( &self, + py: Python<'_>, step_script: &PyStepScript, step_name: Option, task_parameter_values: Option<&Bound<'_, PyDict>>, @@ -579,42 +838,31 @@ impl PySession { }; let resolved = resolved_symtab.map(|st| st.inner.clone()); - let mut guard = lock_recover(&self.session); - let session = guard.take().ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("An action is already running") - })?; - drop(guard); - - { - let mut snap = lock_recover(&self.snapshot); - snap.state = SessionState::Running; - snap.action_status = None; - } - - let session_arc = self.session.clone(); - let snapshot = self.snapshot.clone(); - - std::thread::spawn(move || { - run_action(session, session_arc, snapshot, move |rt, session| { - let env_ref = os_env_vars.as_ref(); - let task_ref = task_params.as_ref(); - let _ = rt.block_on(session.run_task( - &step_name, - &script, - task_ref, - resolved.as_ref(), - env_ref, - )); - }); - }); - - Ok(()) + self.dispatch(py, &[SessionState::Ready], move |rt, session| { + rt.block_on(session.run_task( + &step_name, + &script, + task_params.as_ref(), + resolved.as_ref(), + os_env_vars.as_ref(), + )) + .map(|_| ()) + }) } - /// Run a subprocess. Non-blocking — spawns on a background thread. + /// Run a subprocess. Non-blocking: returns once it is running. + /// + /// Raises `SessionError` if the session isn't `READY`, if `command` is + /// empty or blank, or if `timeout` is zero, and `ValueError` if + /// `timeout` is negative, NaN or infinite. + /// + /// Waits for the action thread, which logs through Python's `logging`, + /// so don't call it while holding a lock that a logging handler takes. #[pyo3(signature = (*, command, args=None, timeout=None, os_env_vars=None, use_session_env_vars=true, log_banner_message=None))] + #[allow(clippy::too_many_arguments)] // PyO3 method mirrors a kwarg-rich public signature fn run_subprocess( &self, + py: Python<'_>, command: String, args: Option>, timeout: Option, @@ -622,85 +870,155 @@ impl PySession { use_session_env_vars: bool, log_banner_message: Option, ) -> PyResult<()> { - let mut guard = lock_recover(&self.session); - let session = guard.take().ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("An action is already running") - })?; - drop(guard); - - { - let mut snap = lock_recover(&self.snapshot); - snap.state = SessionState::Running; - snap.action_status = None; - } + let duration = timeout + .map(|t| seconds_to_duration("timeout", t)) + .transpose()?; + self.dispatch(py, &[SessionState::Ready], move |rt, session| { + rt.block_on(session.run_subprocess( + &command, + args.as_deref(), + duration, + os_env_vars.as_ref(), + use_session_env_vars, + log_banner_message.as_deref(), + )) + .map(|_| ()) + }) + } + /// Take an ending-only session (`READY_ENDING`) back to `READY`, so it + /// accepts `run_task`, `enter_environment` and `run_subprocess` again. + /// The environments entered, the last action's status and the + /// environment variables stay as they were. + /// + /// Raises `SessionError` if the session isn't `READY_ENDING`. The + /// crate also refuses after its session-wide cancel token is canceled; + /// this binding doesn't expose one, so that can't happen here. + fn clear_ending_only(&self, py: Python<'_>) -> PyResult<()> { let session_arc = self.session.clone(); let snapshot = self.snapshot.clone(); - - std::thread::spawn(move || { - run_action(session, session_arc, snapshot, move |rt, session| { - let duration = timeout.map(std::time::Duration::from_secs_f64); - let env_ref = os_env_vars.as_ref(); - let args_ref = args.as_deref(); - let banner_ref = log_banner_message.as_deref(); - let _ = rt.block_on(session.run_subprocess( - &command, - args_ref, - duration, - env_ref, - use_session_env_vars, - banner_ref, - )); - }); + let result = py.detach(move || { + let mut guard = lock_recover(&session_arc); + let session = guard.as_mut()?; + let r = session.clear_ending_only(); + Self::update_snapshot(session, &snapshot); + Some(r) }); - - Ok(()) + match result { + Some(r) => r.map_err(session_err_to_py), + None => Err(self.busy_error(&[SessionState::ReadyEnding])), + } } + /// Cancel the running action. + /// + /// Raises `SessionError` with the crate's message if no action is + /// running or one is already being canceled, and `ValueError` if + /// `time_limit` is negative, NaN or infinite. Raises + /// `SessionError("Cannot cancel: the action has not started yet")` if + /// the action hasn't installed its cancel token: at once when called + /// from the action's own thread (a logging handler), and after 20 s + /// otherwise, typically because the action thread is blocked on a lock + /// the caller holds, such as a logging handler's. + /// + /// Unlike the pure-Python reference, a second cancel while one is in + /// progress raises; the crate doesn't shorten the first cancel's grace + /// period. + #[pyo3(signature = (time_limit=None, mark_action_failed=None))] fn cancel_action( &self, + py: Python<'_>, time_limit: Option, mark_action_failed: Option, ) -> PyResult<()> { - let duration = time_limit.map(std::time::Duration::from_secs_f64); + let duration = time_limit + .map(|t| seconds_to_duration("time_limit", t)) + .transpose()?; let mark_failed = mark_action_failed.unwrap_or(false); - // Direct path when the session is not taken (e.g. between actions): - // Session::cancel_action also performs the Running -> Canceling state - // transition, which the handle cannot (it doesn't own the Session). - let mut guard = lock_recover(&self.session); - if let Some(session) = guard.as_mut() { - let result = session - .cancel_action(duration, mark_failed) - .map_err(session_err_to_py); - Self::update_snapshot(session, &self.snapshot); - return result; - } - drop(guard); - - // Session is owned by a background action thread: deliver through the - // thread-safe cancel handle. Cancellation follows the action's own - // cancelation method (including cross-user helper delivery), exactly - // like Session::cancel_action. - if self.cancel_handle.cancel(duration, mark_failed) { - // The handle cannot set the session's transient Canceling state; - // reflect it in the snapshot so Python-side pollers observe the - // cancel-in-progress phase until the terminal state lands. - let mut snap = lock_recover(&self.snapshot); - snap.state = SessionState::Canceling; - Ok(()) - } else { - Err(pyo3::exceptions::PyRuntimeError::new_err( - "Cannot cancel: no action is running", - )) - } + let session_arc = self.session.clone(); + let snapshot = self.snapshot.clone(); + let handle = self.cancel_handle.clone(); + // The crate reports an action `Running` just before it logs the + // action's banner and installs its cancel token, so a cancel straight + // after the call that started it can find no token yet. Logging needs + // the GIL and runs every Python handler, so the gap is as long as the + // action thread waits for those, which under load can be seconds. + // Keep trying, with the GIL released, until the token is there or the + // action thread has given the session back. Give up after + // `CANCEL_WAIT`, which is reached only when the action thread is + // stuck on something this thread holds, such as a logging handler's + // lock. On the action thread itself (a logging handler that cancels) + // the token can't appear while this call runs, so don't wait at all. + const CANCEL_WAIT: std::time::Duration = std::time::Duration::from_secs(20); + let on_action_thread = + *lock_recover(&self.action_thread) == Some(std::thread::current().id()); + let deadline = std::time::Instant::now() + + if on_action_thread { + std::time::Duration::ZERO + } else { + CANCEL_WAIT + }; + py.detach(move || loop { + { + let mut guard = lock_recover(&session_arc); + if let Some(session) = guard.as_mut() { + // Between actions, or the action has ended: the crate + // decides, and refuses with `InvalidState` unless the + // session is `Running` (which it isn't here). + let result = session + .cancel_action(duration, mark_failed) + .map_err(session_err_to_py); + Self::update_snapshot(session, &snapshot); + return result; + } + // Session is owned by a background action thread: deliver + // through the thread-safe cancel handle, which follows the + // action's own cancelation method (including cross-user + // helper delivery), exactly like Session::cancel_action. + // A second cancel while one is in progress: the crate's + // `Session::cancel_action` refuses it, so do the same rather + // than deliver it through the handle. + { + let snap = lock_recover(&snapshot); + if snap.state == SessionState::Canceling { + return Err(session_err_to_py(SessionError::InvalidState { + expected: vec![SessionState::Running], + current: SessionState::Canceling, + })); + } + } + // Holding the session lock means the action thread can't + // write its final snapshot before the `Canceling` below, so + // the snapshot can't be left reading `Canceling` after the + // action has ended. + if handle.cancel(duration, mark_failed) { + lock_recover(&snapshot).state = SessionState::Canceling; + return Ok(()); + } + } + if std::time::Instant::now() >= deadline { + return Err(session_err_to_py(SessionError::Runtime( + "Cannot cancel: the action has not started yet".into(), + ))); + } + std::thread::sleep(std::time::Duration::from_micros(100)); + }) } - fn cleanup(&self) { - let mut guard = lock_recover(&self.session); - if let Some(session) = guard.as_mut() { - session.cleanup(); - Self::update_snapshot(session, &self.snapshot); - } + /// Clean up the session, deleting its working directory unless it is + /// retained. Does nothing while an action is running. The GIL is + /// released meanwhile: on Windows the crate retries a removal that + /// fails for up to about 31 s. + fn cleanup(&self, py: Python<'_>) { + let session_arc = self.session.clone(); + let snapshot = self.snapshot.clone(); + py.detach(move || { + let mut guard = lock_recover(&session_arc); + if let Some(session) = guard.as_mut() { + session.cleanup(); + Self::update_snapshot(session, &snapshot); + } + }); } fn __repr__(&self, py: Python<'_>) -> PyResult { diff --git a/rust-bindings/src/sessions/session_user.rs b/rust-bindings/src/sessions/session_user.rs index e1441f39..8aab7b4d 100644 --- a/rust-bindings/src/sessions/session_user.rs +++ b/rust-bindings/src/sessions/session_user.rs @@ -200,10 +200,16 @@ impl PyWindowsSessionUser { )); } - // Determine whether `user` matches the process user. - let is_process_user = openjd_sessions::win32::get_process_user() - .map(|proc_user| user.eq_ignore_ascii_case(&proc_user)) - .unwrap_or(false); + // Determine whether `user` matches the process user. A failure to + // look up the process user raises, as the pure-Python reference's + // `get_process_user()` does, rather than reading as "not the + // process user". + let process_user = openjd_sessions::win32::get_process_user().map_err(|e| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "Could not determine the process user: {e}" + )) + })?; + let is_process_user = user.eq_ignore_ascii_case(&process_user); if is_process_user { if password.is_some() { diff --git a/specs/python-expr-interface.md b/specs/python-expr-interface.md index 63f0a72e..ff634a14 100644 --- a/specs/python-expr-interface.md +++ b/specs/python-expr-interface.md @@ -65,6 +65,8 @@ metered.operation_count # operations performed (>= 0) ### `escape_format_string` Escape `{{` and `}}` in a string for use as a literal in a format string. +A lone `{` just before `}}` is escaped too, so the result always parses +and resolves back to the input. ```python from openjd.expr import escape_format_string @@ -363,7 +365,31 @@ HostContext.unresolved().is_enabled() # True HostContext.unresolved().is_unresolved() # True — uses stub implementations HostContext.with_rules([]).is_enabled() # True — empty rules ≠ no host context HostContext.with_rules([]).is_unresolved() # False — uses real implementations +``` + +`HostContext.with_rules` raises `ValueError` when a rule is invalid, +such as a POSIX or WINDOWS rule whose `source_path` is not absolute; +the message names the rule's index, for example `path mapping rule 1: +source_path 'rel' must be an absolute POSIX path, starting with '/'`. +When several rules match a path, the one whose source path has the +most names applies, wherever it is in the list. Between a POSIX rule +and a WINDOWS rule with the same number of names (both can match a +path such as `//srv/share/x`), the WINDOWS rule applies. Between rules +for the same location, the last one given applies (How Jobs Are Run, +"Path Mapping"). The order of `rules` matters only for that last tie. +Paths are compared component by component, as How Jobs Are Run, +"Applying Path Mapping Rules within a Job Template", item 2 says: a +drive or UNC share must match as a whole, +WINDOWS components compare the way Windows compares file names +(non-ASCII letters included), and `.` and empty components are ignored +and left out of the mapped path. The pure-Python reference in +openjd-sessions folds only ASCII letters. +A `source_path` or `destination_path` longer than 32,768 bytes makes a +rule invalid, and the rules together are limited to about 30 MiB of +lookup structure. `apply_path_mapping` raises `ExpressionError` for a +path longer than 32,768 bytes. +```python # Inspecting a profile profile.revision # ExprRevision.V2026_02 profile.extensions # [] today @@ -635,6 +661,13 @@ matched, result = rule.apply(path="/mnt/shared/project/file.exr") matched, result = rule.apply(path="/other/path") # matched=False, result="/other/path" +# A rule whose POSIX or WINDOWS source_path is not absolute, or whose +# source_path or destination_path is longer than 32,768 bytes, can be +# constructed, but apply() raises ValueError for it: +# "source_path 'rel' must be an absolute POSIX path, starting with '/'" +# A path longer than 32,768 bytes matches no rule here (apply_path_mapping +# raises for it instead). + # Serialization d = rule.to_dict() # {"source_path_format": "POSIX", "source_path": "/mnt/shared", "destination_path": "/local/cache"} @@ -727,6 +760,18 @@ RangeExpr.from_list(["1", "2", "3"]) # 1-3 input string. Two `RangeExpr` values that compare equal also hash equal (suitable as `set` / `dict` keys). +A `RangeExpr` is kept in the canonical form of Template Schemas +§3.4.1.1.1, whatever elements the input used. `str`, `repr` and +`ranges()` give the canonical runs, so `RangeExpr("1,3,5,7,9")` is +`1-9:2` and `RangeExpr("1-2").ranges()` is two single-value ranges. +Two `RangeExpr`s are equal exactly when they have the same values. +Every constructor raises `RangeExprError` when the canonical form would +have more than 50,000 runs, and parsing accepts at most 50,000 +comma-separated elements. Values can be any 64-bit integer except the +most negative one. The pure-Python +`openjd.model._range_expr.IntRangeExpr` follows the input's elements +instead. + ### `IntRange` A single contiguous integer range — the element type returned by @@ -912,7 +957,7 @@ from openjd.expr import ExpressionError, RangeExprError, evaluate_expression try: evaluate_expression("1 +") except ExpressionError as e: - str(e) # "Syntax error: ...\n 1 +\n ^~~" + str(e) # "Syntax error: Expected an expression\n 1 +\n ^~~" # Undefined variable try: @@ -927,6 +972,24 @@ except RangeExprError as e: str(e) # "Expected integer in 'not-a-range'" ``` +Syntax errors use the wording of Ruff's parser (`Expected an +expression`, `unexpected EOF while parsing`), which openjd-expr uses +since 0.15.0 (openjd-rs#531). Parsing an invalid expression takes time +roughly linear in its length, whatever its bracket nesting, except for +unclosed nested f-strings, which are quadratic. Brackets are no longer +limited to 200 levels; the 64 KB length cap and the AST depth limit +still apply. A multi-line expression is parsed as written, so a caret +points into the text the author wrote, and it may end in a comment. +Coercing a string to a number, for example with a `target_type` of +`int` or `list[int]`, ignores the white space around the number +(openjd-rs#530). + +A computed float is written as Python's `repr(float)` writes it, +including a shortest-digit tie, which rounds to even: +`string(1000000000000000.25 * 1.0)` is `"1000000000000000.2"` +(openjd-expr 0.15.0, openjd-rs#539; before, ties rounded up). + + | Exception | Base | |---|---| | `ExpressionError` | `ValueError` | diff --git a/specs/python-model-interface.md b/specs/python-model-interface.md index e297e53f..55b7b15f 100644 --- a/specs/python-model-interface.md +++ b/specs/python-model-interface.md @@ -287,6 +287,23 @@ value is joined to `current_working_dir`. Joined values are lexically normalized; an absolute submitted value is returned as written. A default's constraints are checked against the joined value. URI values are refused. Violations raise `DecodeValidationError` (openjd-rs#421). +A FLOAT given as a number, as a default, a `LIST[FLOAT]` item or a +value, is written the way Python's `repr(float)` writes it, with zero +always `0.0`: `default: 2` gives `"2.0"`, `1e20` gives `"1e+20"`, and +the tie 1000000000000000.25 gives `"1000000000000000.2"`. A FLOAT given +as a string keeps its spelling (Template Schemas §7.5), and a +`LIST[FLOAT]` value given as JSON text reads each number as the nearest +double. Before openjd-model 0.16.0 (openjd-rs#539), a scalar FLOAT +`default` given as a number kept its own digits (`2`, +`100000000000000000000`), a tie rounded up, and JSON text could read a +double one ULP off. White space around a number in a string value is +ignored, so `" 7 "` is the INT `7` (openjd-rs#530). A `LIST[INT]`, +`LIST[FLOAT]` or `LIST[LIST[INT]]` value given as a Python list of +numeric strings is coerced item by item, the same as the JSON text of +that list, and an item error names its position (`item[1]: ...`); a +`LIST[FLOAT]` default item written as a string keeps its spelling +(openjd-rs#538). + ```python from openjd.model._v1 import decode_job_template, preprocess_job_parameters from pathlib import Path @@ -1028,11 +1045,11 @@ underlying `template::TaskParameterDefinition` enum 1:1: | Variant | Pyclass | `range` element type | |---|---|---| -| `INT` | `IntTaskParameterDefinition` | `list[int]` or `FormatString` | +| `INT` | `IntTaskParameterDefinition` | `list[int \| FormatString]` or `FormatString` | | `FLOAT` | `FloatTaskParameterDefinition` | `list[float \| FormatString]` or `FormatString` | | `STRING` | `StringTaskParameterDefinition` | `list[FormatString]` or `FormatString` | | `PATH` | `PathTaskParameterDefinition` | `list[FormatString]` or `FormatString` | -| `CHUNK[INT]` | `ChunkIntTaskParameterDefinition` | `list[int]` or `FormatString` | +| `CHUNK[INT]` | `ChunkIntTaskParameterDefinition` | `list[int \| FormatString]` or `FormatString` | Common attributes on every variant: @@ -1048,10 +1065,13 @@ For lists where the element type is `int`/`float`/`FormatString`, the list-form vs format-string-form is dispatched by the binding: the `.range` getter returns either a Python list (literal range) or a `FormatString` (e.g. `"1-10:2"`, possibly carrying a -`{{Param.X}}` interpolation under the EXPR extension). Only `INT` and -`CHUNK[INT]` accept the list-form `[1, 2, 3]`; the others always carry -`FormatString` elements (which may themselves be literal or -interpolating). +`{{Param.X}}` interpolation under the EXPR extension). An `INT` or +`CHUNK[INT]` list element is a `FormatString` when it holds an +interpolation, such as `"{{Param.Frame}}"`, and an `int` otherwise +(`"5"` reads as `5`). A `FLOAT` list element is a `float` when the +template wrote a number and a `FormatString` when it wrote a string. +`STRING` and `PATH` elements are always `FormatString` (which may +itself be literal or interpolating). The `CHUNK[INT]` variant additionally exposes: @@ -1233,7 +1253,10 @@ Iteration observes it, and `len()` counts the overridden granularity. The rendered form of a chunk depends on `rangeConstraint`, which matters because these are strings a consumer parses and may feed back through `__contains__`. A single-task chunk is `"1-1"` under `CONTIGUOUS` and a -bare `"1"` under `NONCONTIGUOUS`. +bare `"1"` under `NONCONTIGUOUS`. A `NONCONTIGUOUS` chunk is written in +the canonical form of Template Schemas §3.4.1.1.1, so the values +`[1, 2, 4, 6]` render `"1,2-6:2"`. The pure-Python reference renders +`"1,2,4,6"` there. It is currently the only way to change the chunk size of a *static* chunked space: the `chunks_default_task_count` setter accepts adaptive diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index 243c6182..494fd810 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -100,13 +100,11 @@ class Session: session_id: str, job_parameter_values: dict[str, ParameterValue], path_mapping_rules: Optional[list[PathMappingRule]] = None, - callback: Optional[Callable] = None, - os_env_vars: Optional[dict[str, str]] = None, retain_working_dir: bool = False, + os_env_vars: Optional[dict[str, str]] = None, session_root_directory: Optional[Path] = None, user: Optional[SessionUser] = None, - job_template: Optional[JobTemplate] = None, - environment_templates: Optional[list[EnvironmentTemplate]] = None, + profile: Optional[ModelProfile] = None, caller_limits: Optional[CallerLimits] = None, ): ... @@ -116,13 +114,17 @@ class Session: working_directory: Path files_directory: Path action_status: Optional[ActionStatus] + environments_entered: list[str] - # Environment lifecycle (non-blocking — starts action, returns immediately) + def extend_path_mapping_rules(self, additional: list[PathMappingRule]) -> None: ... + + # Environment lifecycle (non-blocking — returns once the action is running) def enter_environment( self, *, environment: Environment, identifier: Optional[str] = None, + resolved_symtab: Optional[SerializedSymbolTable] = None, os_env_vars: Optional[dict[str, str]] = None, ) -> str: ... @@ -130,19 +132,23 @@ class Session: self, *, identifier: str, + resolved_symtab: Optional[SerializedSymbolTable] = None, + keep_session_running: bool = True, os_env_vars: Optional[dict[str, str]] = None, ) -> None: ... - # Task execution (non-blocking — starts action, returns immediately) + # Task execution (non-blocking — returns once the action is running) def run_task( self, *, step_script: StepScript, + step_name: Optional[str] = None, task_parameter_values: Optional[dict] = None, + resolved_symtab: Optional[SerializedSymbolTable] = None, os_env_vars: Optional[dict[str, str]] = None, ) -> None: ... - # Subprocess execution (non-blocking) + # Subprocess execution (non-blocking — returns once it is running) def run_subprocess( self, *, @@ -157,11 +163,12 @@ class Session: # Control def cancel_action( self, - *, time_limit: Optional[float] = None, - mark_action_failed: bool = False, + mark_action_failed: Optional[bool] = None, ) -> None: ... + def clear_ending_only(self) -> None: ... + def cleanup(self) -> None: ... ``` @@ -175,6 +182,91 @@ boundary for those caps: a worker can run a job that never passed through the validating process, so a session that is given no limits enforces nothing beyond the spec. +`path_mapping_rules`, and the rules given to +`extend_path_mapping_rules`, must each have an absolute POSIX or +WINDOWS `source_path`, and no `source_path` or `destination_path` +longer than 32,768 bytes (How Jobs Are Run, "Path Mapping"). A set +with an invalid rule raises `SessionError`, and +`extend_path_mapping_rules` leaves the existing rules unchanged. When +several rules match a path, the one whose source path has the most +names applies, wherever it is in the list, and a WINDOWS rule applies +over a POSIX rule with the same number of names. Between rules for the +same location the last one applies, so a rule added by +`extend_path_mapping_rules` replaces an existing rule for its location. +Rules match paths component by component, as described for +`HostContext.with_rules` in +[python-expr-interface.md](./python-expr-interface.md). A PATH or +LIST[PATH] value longer than 32,768 bytes is an error when the session +resolves it. + +`working_directory` is a directory under the session root whose name +is 8 random characters from `[a-z0-9_]`, with no part of the session +ID in it, as the pure-Python reference names it. + +A call the openjd-sessions crate refuses raises `SessionError` with the +crate's message, as the Rust method returns `Err`, and changes nothing: +the state, `action_status` (still the previous action's) and +`environments_entered` stay as they were. The crate refuses `run_task`, +`enter_environment` and `run_subprocess` unless the session is `READY`, +`exit_environment` unless it is `READY` or `READY_ENDING`, +`cancel_action` unless an action is running, an unknown, out-of-order or +duplicate environment identifier, a second environment with wrap hooks, +and a blank `command` or zero `timeout`. A call made while another +action runs gets the crate's `InvalidState` message, such as `Session +must be in READY state, current: RUNNING`; before, it was +`RuntimeError("An action is already running")`, and `SessionError` is a +`RuntimeError`. A `timeout` or `time_limit` that is negative, NaN or +infinite raises `ValueError`. + +Once the crate accepts a call, the method returns when its action is +`RUNNING` (or, for an environment with no script, has ended), and +`action_status` describes that action. A `cancel_action` made as soon as +it returns reaches the action, unless the action has already ended, as +one that fails before its subprocess starts can: then it raises the +crate's `InvalidState`. A zero `time_limit` terminates at once. A second +`cancel_action` while the session is `CANCELING` raises the crate's +`InvalidState`; the pure-Python reference accepts it and shortens the +remaining grace period to at most its `time_limit`, which the crate +can't do. If the action hasn't installed its cancel token yet, +`cancel_action` raises `SessionError("Cannot cancel: the action has not +started yet")`, the binding's own message: at once when called from the +action's own thread (a logging handler), and after 20 s otherwise, +typically because the action thread is blocked on a lock the caller +holds. Every error after acceptance, such as a parameter value that +can't be coerced, is reported on `action_status` and doesn't raise, even +when it arrives before the call returns. The crate documents those as +failing the action rather than refusing the call, and the pure-Python +reference reports them the same way. + +The crate runs each action on a background thread, and that thread logs +through Python's `logging` (each record needs the GIL and runs the +handlers). An action method waits for the crate to accept or refuse the +call, and `cancel_action` waits for the action's cancel token, so don't +call either while holding a lock that a logging handler takes on another +thread: the action thread would wait for the lock, and the call for the +action thread. The pure-Python reference logs on the caller's thread, so +it has no such restriction. A fix belongs in the crate: report that it +accepted a call, and install the cancel token, before logging. + +An action that ends `FAILED`, `CANCELED` or `TIMEOUT` makes the session +ending-only (`READY_ENDING`): `exit_environment` and `cleanup` still +work, and a successful `exit_environment(keep_session_running=True)` +leaves it `READY_ENDING` (openjd-rs#524). `clear_ending_only()` takes it +back to `READY`, keeping the environments entered and the last action's +status; it raises `SessionError` unless the session is `READY_ENDING`. +When `run_task`, `enter_environment` or `exit_environment` fails before +its subprocess starts, for example because a parameter value can't be +coerced, the action is `FAILED` with the error's message, as in the +pure-Python reference (openjd-rs#532). An environment whose enter fails +this way still counts as entered, and one whose exit fails this way +counts as exited. When `exit_environment` can't build its symbol table +from `resolved_symtab`, it runs `onExit` with the table the environment +was entered with (openjd-rs#536). White space around a number in an INT +or FLOAT job or task parameter value is ignored, so `" 7 "` is `7` and +`" 1.50 "` is `1.50` (openjd-rs#530). On Windows, `cleanup` retries +removing a session directory that is still in use, for up to about 31 s, +with the GIL released (openjd-rs#485). + ### SessionUser (stays in Python, passed to Rust) The `SessionUser` types involve OS-specific credential validation (sudo @@ -192,6 +284,10 @@ class WindowsSessionUser: password: Optional[str] ``` +`WindowsSessionUser` raises `RuntimeError("Could not determine the +process user: ...")` if it can't look up the process user, as the +pure-Python reference's `get_process_user()` raises. + ### Callback interface The Python callback `Callable[[str, ActionStatus], None]` is invoked from diff --git a/src/openjd/_openjd_rs.pyi b/src/openjd/_openjd_rs.pyi index 5bb5740a..3f26b57b 100644 --- a/src/openjd/_openjd_rs.pyi +++ b/src/openjd/_openjd_rs.pyi @@ -1057,6 +1057,10 @@ class HostContext: host context (`apply_path_mapping` is registered, returns the path unchanged). Passing zero rules is *not* the same as passing no host context at all. + + Raises `ValueError` when a rule is invalid, for example a POSIX or + WINDOWS `source_path` that is not absolute. The message names the + rule's index in `rules`. """ def is_enabled(self) -> builtins.bool: @@ -1913,7 +1917,14 @@ class PathMappingRule: def apply( self, *, path: builtins.str, output_format: typing.Optional[PathFormat] = None - ) -> tuple[builtins.bool, builtins.str]: ... + ) -> tuple[builtins.bool, builtins.str]: + r""" + Apply this rule to `path`. Returns `(True, mapped)` when the rule + matches and `(False, path)` when it does not. Raises `ValueError` + when the rule itself is invalid, for example a POSIX or WINDOWS + `source_path` that is not absolute. + """ + def to_dict(self) -> builtins.dict[builtins.str, builtins.str]: ... @staticmethod def from_dict(d: dict) -> PathMappingRule: ... @@ -2178,8 +2189,9 @@ class Session: r""" Extend the session's path mapping rules with additional rules. - Rules are stored sorted by source-path length (longest first) so that - the most specific rule matches first during FormatString resolution. + When several rules match a path, the one whose source path has the + most names wins, and between rules for the same location the + one given last wins, wherever it sits in the list. Consumers (notably the Deadline Cloud worker agent's Job Attachments download action) call this after an action assigns additional path @@ -2187,7 +2199,9 @@ class Session: Returns an error if an action is currently in-flight (the underlying Session has been taken by a background thread). Call this between - actions. + actions. Raises `SessionError`, leaving the rules unchanged, when an + added rule is invalid, for example a POSIX or WINDOWS `source_path` + that is not absolute. """ def enter_environment( @@ -2199,8 +2213,17 @@ class Session: os_env_vars: typing.Optional[typing.Mapping[builtins.str, builtins.str]] = None, ) -> builtins.str: r""" - Enter an environment. Non-blocking — spawns the onEnter action on a - background thread and returns the environment identifier immediately. + Enter an environment. Non-blocking: returns the environment + identifier once the onEnter action is running. + + Raises `SessionError` if the session isn't `READY`, if + `identifier` is already entered, or if a second environment with a + wrap hook is entered. A failure after that, such as an environment + variable that doesn't resolve, fails the action; the environment + still counts as entered and must be exited. + + Waits for the action thread, which logs through Python's `logging`, + so don't call it while holding a lock that a logging handler takes. """ def exit_environment( @@ -2212,20 +2235,41 @@ class Session: os_env_vars: typing.Optional[typing.Mapping[builtins.str, builtins.str]] = None, ) -> None: r""" - Exit an environment. Non-blocking — spawns the onExit action on a - background thread. + Exit an environment. Non-blocking: returns once the onExit action is + running. + + Raises `SessionError` if the session isn't `READY` or + `READY_ENDING`, if `identifier` isn't entered, or if it isn't the + most recently entered environment. A failure after that fails the + action, and the environment still counts as exited. + + Waits for the action thread, which logs through Python's `logging`, + so don't call it while holding a lock that a logging handler takes. """ def run_task( self, *, step_script: StepScript, + step_name: typing.Optional[builtins.str] = None, task_parameter_values: typing.Optional[dict] = None, resolved_symtab: typing.Optional[SerializedSymbolTable] = None, os_env_vars: typing.Optional[typing.Mapping[builtins.str, builtins.str]] = None, ) -> None: r""" - Run a task. Non-blocking — spawns the onRun action on a background thread. + Run a task. Non-blocking: returns once the onRun action is running. + + Raises `SessionError` if the session isn't `READY`, for example + because an earlier action failed and the session is ending-only. A + failure after that, such as a task parameter value that can't be + coerced, fails the action. + + Waits for the action thread, which logs through Python's `logging`, + so don't call it while holding a lock that a logging handler takes. + + `step_name` is surfaced as `WrappedStep.Name` to a wrapping + environment's `onWrapTaskRun` hook (RFC 0008). It defaults to an empty + string when the caller does not supply one. """ def run_subprocess( @@ -2239,15 +2283,58 @@ class Session: log_banner_message: typing.Optional[builtins.str] = None, ) -> None: r""" - Run a subprocess. Non-blocking — spawns on a background thread. + Run a subprocess. Non-blocking: returns once it is running. + + Raises `SessionError` if the session isn't `READY`, if `command` is + empty or blank, or if `timeout` is zero, and `ValueError` if + `timeout` is negative, NaN or infinite. + + Waits for the action thread, which logs through Python's `logging`, + so don't call it while holding a lock that a logging handler takes. + """ + + def clear_ending_only(self) -> None: + r""" + Take an ending-only session (`READY_ENDING`) back to `READY`, so it + accepts `run_task`, `enter_environment` and `run_subprocess` again. + The environments entered, the last action's status and the + environment variables stay as they were. + + Raises `SessionError` if the session isn't `READY_ENDING`. The + crate also refuses after its session-wide cancel token is canceled; + this binding doesn't expose one, so that can't happen here. """ def cancel_action( self, - time_limit: typing.Optional[builtins.float], - mark_action_failed: typing.Optional[builtins.bool], - ) -> None: ... - def cleanup(self) -> None: ... + time_limit: typing.Optional[builtins.float] = None, + mark_action_failed: typing.Optional[builtins.bool] = None, + ) -> None: + r""" + Cancel the running action. + + Raises `SessionError` with the crate's message if no action is + running or one is already being canceled, and `ValueError` if + `time_limit` is negative, NaN or infinite. Raises + `SessionError("Cannot cancel: the action has not started yet")` if + the action hasn't installed its cancel token: at once when called + from the action's own thread (a logging handler), and after 20 s + otherwise, typically because the action thread is blocked on a lock + the caller holds, such as a logging handler's. + + Unlike the pure-Python reference, a second cancel while one is in + progress raises; the crate doesn't shorten the first cancel's grace + period. + """ + + def cleanup(self) -> None: + r""" + Clean up the session, deleting its working directory unless it is + retained. Does nothing while an action is running. The GIL is + released meanwhile: on Windows the crate retries a removal that + fails for up to about 31 s. + """ + def __repr__(self) -> builtins.str: ... @typing.final diff --git a/src/openjd/model/_format_strings/_nodes.py b/src/openjd/model/_format_strings/_nodes.py index 8e74fdf6..750283b6 100644 --- a/src/openjd/model/_format_strings/_nodes.py +++ b/src/openjd/model/_format_strings/_nodes.py @@ -240,8 +240,12 @@ def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: values = symtab_to_expr_values( symtab, types=symtab.expr_types or None, path_format=path_format ) - profile = profile_for_symtab(symtab) try: + # Inside the try: ``HostContext.with_rules`` raises ``ValueError`` + # for an invalid path mapping rule, such as one whose source path + # is not absolute, and that reaches the caller as an expression + # error like any other. + profile = profile_for_symtab(symtab) return self._parsed.evaluate(values=values, profile=profile, path_format=path_format) except Exception as exc: # noqa: BLE001 - boundary; re-raise as model error raise map_eval_error(exc) diff --git a/test/openjd/expr/test_error_formatting.py b/test/openjd/expr/test_error_formatting.py index 8268fc56..b4fb5170 100644 --- a/test/openjd/expr/test_error_formatting.py +++ b/test/openjd/expr/test_error_formatting.py @@ -302,15 +302,21 @@ def test_leading_whitespace_error_in_middle(self): class TestSyntaxErrorCarets: - """Test that syntax errors include caret pointers.""" + """Test that syntax errors include caret pointers. + + Since openjd-expr 0.15.0 (openjd-rs#531) expressions are parsed with + Astral's ruff_python_parser, so syntax errors use Ruff's wording + ("unexpected EOF while parsing"). From openjd-expr 0.11.0 (openjd-rs#429) + until then, RustPython's fork of the parser gave CPython's wording + ("'(' was never closed"). The caret is unchanged. + """ def test_unclosed_paren(self): """Unclosed parenthesis shows caret spanning the expression.""" with pytest.raises(ExpressionError) as exc_info: evaluate_expression("(1 + 2", values=SymbolTable()) - # CPython's wording since ruff 0.16.10 (openjd-rs#429). expected = [ - "Syntax error: '(' was never closed\n", + "Syntax error: unexpected EOF while parsing\n", " (1 + 2\n", " ^~~~~~", ] @@ -321,7 +327,7 @@ def test_unclosed_bracket(self): with pytest.raises(ExpressionError) as exc_info: evaluate_expression("[1, 2, 3", values=SymbolTable()) expected = [ - "Syntax error: '[' was never closed\n", + "Syntax error: unexpected EOF while parsing\n", " [1, 2, 3\n", " ^~~~~~~~", ] @@ -332,15 +338,48 @@ def test_unclosed_string(self): with pytest.raises(ExpressionError) as exc_info: evaluate_expression("'unclosed", values=SymbolTable()) expected = [ - "Syntax error: unterminated string literal (detected at line 1)\n", + "Syntax error: missing closing quote in string literal\n", " 'unclosed\n", " ^~~~~~~~~", ] assert str(exc_info.value) == "".join(expected) + @pytest.mark.parametrize( + "expr, expected", + [ + pytest.param( + "1 +", "Syntax error: Expected an expression\n 1 +\n ^~~", id="no operand" + ), + pytest.param( + "f(a b)", + "Syntax error: Expected `,`, found identifier\n f(a b)\n ^", + id="missing comma", + ), + pytest.param( + "Param.A\nParam.B", + "Syntax error: Unexpected token at the end of an expression\n Param.B\n ^~~~~", + id="two lines", + ), + ], + ) + def test_ruff_wording(self, expr: str, expected: str) -> None: + """More of Ruff's wording (openjd-rs#531). Before, these were + "invalid syntax", "invalid syntax. Perhaps you forgot a comma?" with a + caret on ``a b``, and, for two lines, the same comma hint on the first + line, because the parser saw them wrapped in parentheses.""" + with pytest.raises(ExpressionError) as exc_info: + evaluate_expression(expr, values=SymbolTable()) + assert str(exc_info.value) == expected + class TestMultiLineExpressions: - """Tests for error formatting in multi-line expressions.""" + """Tests for error formatting in multi-line expressions. + + The caret sits on the operator, as it does for a single-line expression. + Since openjd-expr 0.15.0 (openjd-rs#531) a multi-line expression is parsed + as written rather than wrapped in parentheses. Before, the wrap shifted + the caret one character right of the operator (``~~~^~~~``). + """ def test_error_in_parentheses_multiline(self): """Error on second line of parenthesized expression.""" @@ -349,7 +388,7 @@ def test_error_in_parentheses_multiline(self): expected = [ "Cannot use '+' operator with int and string\n", " 1 + 'x'\n", - " ~~~^~~~", + " ~~^~~~~", ] assert str(exc_info.value) == "".join(expected) @@ -360,7 +399,7 @@ def test_error_in_list_multiline(self): expected = [ "Cannot use '+' operator with int and string\n", " 2 + 'x',\n", - " ~~~^~~~", + " ~~^~~~~", ] assert str(exc_info.value) == "".join(expected) @@ -371,7 +410,7 @@ def test_error_on_first_line_multiline(self): expected = [ "Cannot use '+' operator with int and string\n", " 1 + 'x' + (\n", - " ~~~^~~~", + " ~~^~~~~", ] assert str(exc_info.value) == "".join(expected) @@ -382,7 +421,7 @@ def test_deeply_nested_multiline(self): expected = [ "Cannot use '+' operator with int and string\n", " 1 + 'x'\n", - " ~~~^~~~", + " ~~^~~~~", ] assert str(exc_info.value) == "".join(expected) diff --git a/test/openjd/expr/test_lists.py b/test/openjd/expr/test_lists.py index 15c8b632..6391da15 100644 --- a/test/openjd/expr/test_lists.py +++ b/test/openjd/expr/test_lists.py @@ -124,6 +124,14 @@ def test_three_level_nesting_in_comprehension_fails(self) -> None: with pytest.raises(ExpressionError, match="nested at most 2 levels"): evaluate_expression("[[[x]] for x in [1, 2]]") + def test_a_range_expr_in_two_levels_is_not_a_third(self) -> None: + """A ``range_expr`` element is not a list level, so ``[[r]]`` is two + levels deep (Expression Language §1.2.1). openjd-expr 0.13.0 + (openjd-rs#462); before, it raised "Lists may be nested at most 2 + levels deep".""" + result = evaluate_expression("[[range_expr('1-3')]]") + assert result.type == ExprType("list[list[range_expr]]") + class TestListElementCoercion: """Tests for list element coercion when target type has single list type (RFC 0005).""" diff --git a/test/openjd/expr/test_parse_expression.py b/test/openjd/expr/test_parse_expression.py index 208c3182..b30c3a91 100644 --- a/test/openjd/expr/test_parse_expression.py +++ b/test/openjd/expr/test_parse_expression.py @@ -263,3 +263,13 @@ def test_escaped_braces(self): fs = FormatString(escaped) st = SymbolTable() assert fs.resolve_string(st) == "{{literal}}" + + @pytest.mark.parametrize("raw", ["{}}", "a{}}b", "{{}}}"]) + def test_escaped_brace_before_a_closing_pair(self, raw: str) -> None: + """A lone ``{`` just before ``}}`` is escaped too, so the result + parses and resolves to ``raw``. openjd-expr 0.14.0 (openjd-rs#515); + before, ``"{}}"`` gave ``{{{ "}" + "}" }}``, which failed to parse.""" + from openjd.expr import escape_format_string + + escaped = escape_format_string(raw) + assert FormatString(escaped).resolve_string(SymbolTable()) == raw diff --git a/test/openjd/expr/test_parser_work_bound.py b/test/openjd/expr/test_parser_work_bound.py new file mode 100644 index 00000000..4e5dc1d9 --- /dev/null +++ b/test/openjd/expr/test_parser_work_bound.py @@ -0,0 +1,129 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Parsing an invalid expression takes bounded time (openjd-rs#489). + +openjd-expr 0.11.0 to 0.14.0 parsed with RustPython's fork of Ruff's parser +(openjd-rs#429), whose CPython-style syntax-error hints parse ahead and rewind. +Hints nest inside brackets, so parse time doubled per unclosed bracket level. +Through these bindings, ``decode_job_template`` took 2.6 s in a release build +(39 s in a debug build) on the 101-byte ``"[i f(" * 20 + "x"``, and 5.0 s on +the 363-byte fuzz reproducer below. Any template field that takes an +expression could stall template decoding. openjd-expr 0.15.0 (openjd-rs#531) +parses with Astral's ``ruff_python_parser``, which has no such hints; each +input here now fails in a few milliseconds. + +Each case runs through the expression API and through both template decoders, +since the pure-Python ``openjd.model`` evaluates EXPR expressions with the same +Rust parser. The 1 s bound is generous so the tests don't flake on a slow CI +runner; one decode runs first so that one-time setup isn't timed. +""" + +from __future__ import annotations + +import time +from typing import Any, Callable + +import pytest + +from openjd.expr import ExpressionError, FormatString, parse_expression +from openjd.model import DecodeValidationError as V0DecodeValidationError +from openjd.model import decode_job_template as v0_decode_job_template +from openjd.model._v1 import decode_job_template as v1_decode_job_template +from openjd.model._v1.errors import DecodeValidationError as V1DecodeValidationError + +_TIME_BOUND_SECONDS = 1.0 + +# The fuzz harness's reproducer for openjd-rs#489, 363 bytes. +_SLOW_DOTS = ( + "ginaen([manifestVersion.......................(............(......................" + "........join.................(.....................n.................(..........." + "................destination_55555path.(..............data......................" + "......@....so, 1],.so[vted().d(......destination_55555path.(..............data.." + "..............................so, dx in e" +) + +_CASES = [ + pytest.param("[i f(" * 20 + "x", "Expected `,`, found identifier", id="unclosed [i f( x20"), + pytest.param("f(a " * 20 + "x", "Expected `,`, found identifier", id="unclosed f(a x20"), + pytest.param(_SLOW_DOTS, "Expected `,`, found `...`", id="fuzz slow-dots"), +] + + +def _template(expr: str) -> dict[str, Any]: + return { + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "steps": [ + { + "name": "S", + "script": { + "actions": {"onRun": {"command": "echo", "args": ["{{ " + expr + " }}"]}} + }, + } + ], + } + + +def _timed(f: Callable[[], object], exc_type: type[Exception]) -> tuple[float, str]: + start = time.perf_counter() + with pytest.raises(exc_type) as exc_info: + f() + return time.perf_counter() - start, str(exc_info.value) + + +@pytest.fixture(scope="module", autouse=True) +def _warm_up() -> None: + v0_decode_job_template(template=_template("1"), supported_extensions=["EXPR"]) + v1_decode_job_template(template=_template("1"), supported_extensions=["EXPR"]) + + +def test_the_101_byte_reproducer_is_101_bytes() -> None: + """The input size the module docstring quotes.""" + assert len("[i f(" * 20 + "x") == 101 + assert len(_SLOW_DOTS) == 363 + + +@pytest.mark.parametrize("expr, headline", _CASES) +def test_parse_expression(expr: str, headline: str) -> None: + elapsed, message = _timed(lambda: parse_expression(expr), ExpressionError) + assert message.startswith(f"Syntax error: {headline}\n {expr}\n") + assert elapsed < _TIME_BOUND_SECONDS + + +@pytest.mark.parametrize("expr, headline", _CASES) +def test_format_string(expr: str, headline: str) -> None: + elapsed, message = _timed(lambda: FormatString("{{ " + expr + " }}"), ExpressionError) + assert f"Syntax error: {headline}\n" in message + assert elapsed < _TIME_BOUND_SECONDS + + +@pytest.mark.parametrize("expr, headline", _CASES) +def test_v0_decode_job_template(expr: str, headline: str) -> None: + """The pure-Python model, which the Python CLI uses.""" + elapsed, message = _timed( + lambda: v0_decode_job_template(template=_template(expr), supported_extensions=["EXPR"]), + V0DecodeValidationError, + ) + assert message.startswith( + "1 validation errors for JobTemplate\n" + "steps[0] -> script -> actions -> onRun -> args[0]:\n" + f"\tFailed to parse interpolation expression at [0, {len(expr) + 6}]. " + f"Reason: Syntax error: {headline}\n" + ) + assert elapsed < _TIME_BOUND_SECONDS + + +@pytest.mark.parametrize("expr, headline", _CASES) +def test_v1_decode_job_template(expr: str, headline: str) -> None: + elapsed, message = _timed( + lambda: v1_decode_job_template(template=_template(expr), supported_extensions=["EXPR"]), + V1DecodeValidationError, + ) + assert message.startswith( + "'jobtemplate-2023-09' failed checks: " + f"Failed to parse interpolation expression at [0, {len(expr) + 6}]. " + f"Reason: Syntax error: {headline}\n" + ) + assert elapsed < _TIME_BOUND_SECONDS diff --git a/test/openjd/expr/test_parsing.py b/test/openjd/expr/test_parsing.py index 05021420..362ce597 100644 --- a/test/openjd/expr/test_parsing.py +++ b/test/openjd/expr/test_parsing.py @@ -21,6 +21,46 @@ def test_keyword_in_expression_context(self) -> None: symtab = SymbolTable({"Param.if": 10, "Param.else": 20}) assert evaluate_expression("Param.if + Param.else", values=symtab).item() == 30 + def test_placeholder_never_equals_an_nfkc_normalized_name(self) -> None: + """A keyword attribute is renamed to a placeholder before parsing, and + ``Param.is`` gets ``bs``. The parser NFKC-normalizes identifiers, so + fullwidth ``Param.bs`` is the name ``bs`` too. Since openjd-expr 0.15.0 + (openjd-rs#534) the placeholder is checked against normalized names; + before, this gave ``"isis"``.""" + symtab = SymbolTable({"Param.bs": "fullwidth-bs", "Param.is": "is"}) + assert evaluate_expression("Param.bs + Param.is", values=symtab).item() == ( + "fullwidth-bsis" + ) + + def test_placeholder_when_every_first_letter_is_taken(self) -> None: + """The string literal holds every first-letter candidate for ``if`` + (``af`` to ``zf``). Before openjd-expr 0.15.0 (openjd-rs#534, + openjd-rs#333) that made the placeholder fall back to ``xx`` without a + check, so ``Param.xx`` read ``Param.if``'s value and this gave + ``[1, 1, 52]``. Now text in a string literal doesn't block a candidate, + and a candidate is checked against every name the parser sees.""" + symtab = SymbolTable({"Param.if": 1, "Param.xx": 2}) + names = "".join(c + "f" for c in "abcdefghijklmnopqrstuvwxyz") + result = evaluate_expression(f"[Param.if, Param.xx, len('{names}')]", values=symtab) + assert result.item() == [1, 2, 52] + + def test_two_keyword_attributes_never_share_a_placeholder(self) -> None: + """Before openjd-expr 0.15.0 (openjd-rs#531) ``is`` and ``assert`` could + be renamed so that one placeholder was a prefix of the other, and this + gave the wrong value about half the time. Repeat it to catch that.""" + symtab = SymbolTable({"Param.is": "a", "Param.assert": "b"}) + for _ in range(50): + assert evaluate_expression("Param.is + Param.assert", values=symtab).item() == "ab" + + def test_a_syntax_error_shows_the_text_as_written(self) -> None: + """Before openjd-expr 0.15.0 (openjd-rs#531) the source line showed the + placeholder (``Param.af +``) instead of the keyword.""" + with pytest.raises(ExpressionError) as exc_info: + evaluate_expression("Param.if +", values=SymbolTable({"Param.if": 1})) + assert str(exc_info.value) == ( + "Syntax error: Expected an expression\n Param.if +\n ^~~~~~~~~~" + ) + def test_keyword_with_conditional(self) -> None: symtab = SymbolTable({"Param.if": 100}) assert evaluate_expression("Param.if if True else 0", values=symtab).item() == 100 diff --git a/test/openjd/expr/test_path_mapping.py b/test/openjd/expr/test_path_mapping.py index be8b97e0..e1239ca2 100644 --- a/test/openjd/expr/test_path_mapping.py +++ b/test/openjd/expr/test_path_mapping.py @@ -7,7 +7,7 @@ import pytest -from openjd.expr import PathMappingRule, PathFormat, ExprProfile, HostContext +from openjd.expr import ExpressionError, PathMappingRule, PathFormat, ExprProfile, HostContext class TestPathMappingRuleFromPosix: @@ -215,6 +215,59 @@ def test_custom_scheme(self, tmp_path) -> None: class TestPathMappingRuleValidation: """Tests for from_dict validation shared across formats.""" + @pytest.mark.parametrize( + "source_path_format, source_path, expected", + [ + pytest.param( + PathFormat.POSIX, + "rel/a", + "source_path 'rel/a' must be an absolute POSIX path, starting with '/'", + id="relative POSIX", + ), + pytest.param( + PathFormat.WINDOWS, + "a\\b", + "source_path 'a\\b' must be an absolute WINDOWS path, starting with a drive " + "such as 'C:\\' or a UNC share such as '\\\\server\\share'", + id="relative WINDOWS", + ), + ], + ) + def test_apply_rejects_a_relative_source_path( + self, source_path_format: PathFormat, source_path: str, expected: str + ) -> None: + """Since openjd-expr 0.14.0 (openjd-rs#472) a rule must have an + absolute source path (How Jobs Are Run, "Applying Path Mapping Rules + within a Job Template", item 1), and applying one that doesn't raises. + Before, such a rule matched relative paths. The pure-Python reference + in openjd-sessions still accepts it.""" + rule = PathMappingRule( + source_path_format=source_path_format, + source_path=source_path, + destination_path="/b", + ) + with pytest.raises(ValueError) as exc_info: + rule.apply(path="rel/a/x") + assert str(exc_info.value) == expected + + def test_with_rules_rejects_a_relative_source_path(self) -> None: + """``HostContext.with_rules`` checks every rule, and the message + names the rule's index.""" + rules = [ + PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/a", destination_path="/b" + ), + PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="rel", destination_path="/b" + ), + ] + with pytest.raises(ValueError) as exc_info: + HostContext.with_rules(rules) + assert str(exc_info.value) == ( + "path mapping rule 1: source_path 'rel' must be an absolute POSIX path, " + "starting with '/'" + ) + class TestPathMappingRuleSerialization: """Tests for from_dict/to_dict serialization.""" @@ -362,6 +415,146 @@ def test_windows_trailing_forward_slash(self, tmp_path) -> None: assert result.endswith("/") or result.endswith(os.sep) or result.endswith("\\") +class TestComponentMatching: + """Since openjd-expr 0.14.0 (openjd-rs#472) rules match whole components + the way How Jobs Are Run, "Applying Path Mapping Rules within a Job + Template", item 2 says: a UNC share is one component and must match as a + whole, WINDOWS components compare the way Windows compares file names + (non-ASCII letters too), and ``.`` and empty components are ignored and + left out of the mapped path. Before, a rooted path such as ``\\srv\\share`` + matched a UNC rule, ``C:\\Ünï`` did not match ``C:\\üNÏ``, a ``.`` before + the end of the source path prevented a match, and ``.`` and empty + components after it were kept. The pure-Python reference agrees on every + row except the non-ASCII one, where it folds ASCII letters only.""" + + @pytest.mark.parametrize( + "source_path_format, source_path, path, expected", + [ + pytest.param( + PathFormat.WINDOWS, + "\\\\srv\\share", + "\\\\SRV\\SHARE\\x", + (True, "/d/x"), + id="UNC share, case-insensitive", + ), + pytest.param( + PathFormat.WINDOWS, + "\\\\srv\\share", + "\\srv\\share\\x", + (False, "\\srv\\share\\x"), + id="a rooted path is not a UNC share", + ), + pytest.param( + PathFormat.WINDOWS, + "C:\\Ünï", + "C:\\üNÏ\\f", + (True, "/d/f"), + id="non-ASCII letters compare case-insensitively", + ), + pytest.param( + PathFormat.WINDOWS, + "C:\\proj", + "C:\\proj\\.\\f\\", + (True, "/d/f/"), + id="WINDOWS dot component dropped", + ), + pytest.param( + PathFormat.POSIX, + "/mnt/a", + "/mnt/./a/b", + (True, "/d/b"), + id="dot component inside the source path", + ), + pytest.param( + PathFormat.POSIX, + "/mnt/a", + "/mnt/a/./b//c", + (True, "/d/b/c"), + id="dot and empty components dropped", + ), + ], + ) + def test_apply( + self, + source_path_format: PathFormat, + source_path: str, + path: str, + expected: tuple[bool, str], + ) -> None: + rule = PathMappingRule( + source_path_format=source_path_format, source_path=source_path, destination_path="/d" + ) + assert rule.apply(path=path, output_format=PathFormat.POSIX) == expected + + +class TestDestinationEndingInASeparator: + """Since openjd-expr 0.14.0 (openjd-rs#483) no separator is added after a + destination that already ends with one. Before, ``/dst/`` gave + ``/dst//a/b``. The pure-Python reference gives ``/dst/a/b`` too.""" + + def test_apply(self) -> None: + rule = PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/mnt", destination_path="/dst/" + ) + assert rule.apply(path="/mnt/a/b", output_format=PathFormat.POSIX) == (True, "/dst/a/b") + + def test_apply_path_mapping(self) -> None: + from openjd.expr import parse_expression + + rule = PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/mnt", destination_path="/dst/" + ) + profile = ExprProfile().with_host_context(HostContext.with_rules([rule])) + parsed = parse_expression("apply_path_mapping('/mnt/a/b')") + result = parsed.evaluate(profile=profile, path_format=PathFormat.POSIX) + assert result.item() == "/dst/a/b" + + +class TestPathLengthLimit: + """Since openjd-expr 0.14.0 (openjd-rs#472) a path or rule path longer than + 32 KiB (``MAX_PATH_LEN``, 32,768 bytes) is refused. A longer path matches + no rule, and a rule with a longer source or destination path is invalid. + Before, both mapped. The pure-Python reference still maps them.""" + + _RULE = PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/mnt", destination_path="/dst" + ) + + def test_a_path_at_the_limit_maps(self) -> None: + path = "/mnt/" + "a" * (32 * 1024 - len("/mnt/")) + matched, _ = self._RULE.apply(path=path, output_format=PathFormat.POSIX) + assert matched is True + + def test_a_longer_path_matches_no_rule(self) -> None: + path = "/mnt/" + "a" * (32 * 1024 - len("/mnt/") + 1) + assert self._RULE.apply(path=path) == (False, path) + + def test_apply_path_mapping_rejects_a_longer_path(self) -> None: + """The ``apply_path_mapping`` expression function raises rather than + returning the path unmapped.""" + from openjd.expr import parse_expression + + profile = ExprProfile().with_host_context(HostContext.with_rules([self._RULE])) + path = "/mnt/" + "a" * (32 * 1024 - len("/mnt/") + 1) + with pytest.raises(ExpressionError) as exc_info: + parse_expression(f"apply_path_mapping('{path}')").evaluate(profile=profile) + assert str(exc_info.value).split("\n")[0] == ( + "apply_path_mapping: the path is 32769 bytes long, more than the limit of 32768 bytes" + ) + + def test_a_longer_destination_path_is_invalid(self) -> None: + rule = PathMappingRule( + source_path_format=PathFormat.POSIX, + source_path="/mnt", + destination_path="/" + "d" * (32 * 1024), + ) + with pytest.raises(ValueError) as exc_info: + rule.apply(path="/mnt/x") + assert str(exc_info.value) == ( + "destination_path is 32769 bytes long, more than the limit of 32768 bytes" + ) + + class TestFormatMismatch: """Tests that constructor rejects wrong path types for the format.""" @@ -572,6 +765,84 @@ def test_format_string_resolve_string_applies_rules(self, tmp_path) -> None: result = fs.resolve_string(SymbolTable({}), profile=self._profile_with_rule(dest)) assert result == dest + os.sep + "file.exr" + @pytest.mark.parametrize("reverse", [False, True], ids=["shorter first", "longer first"]) + def test_the_rule_with_the_most_names_wins_in_any_order(self, tmp_path, reverse) -> None: + """Since openjd-expr 0.14.0 (openjd-rs#472) the rule whose source + path has the most names applies, wherever it is in the list (How + Jobs Are Run, "Applying Path Mapping Rules within a Job Template", + item 3). Before, ``with_rules`` used the first rule that matched.""" + from openjd.expr import parse_expression + + short_dest = str(tmp_path / "short") + long_dest = str(tmp_path / "long") + rules = [ + PathMappingRule( + source_path_format=PathFormat.POSIX, + source_path="/mnt", + destination_path=short_dest, + ), + PathMappingRule( + source_path_format=PathFormat.POSIX, + source_path="/mnt/shared", + destination_path=long_dest, + ), + ] + if reverse: + rules.reverse() + profile = ExprProfile().with_host_context(HostContext.with_rules(rules)) + parsed = parse_expression("apply_path_mapping('/mnt/shared/file.exr')") + assert parsed.evaluate(profile=profile).item() == long_dest + os.sep + "file.exr" + + @pytest.mark.parametrize("reverse", [False, True], ids=["WINDOWS first", "POSIX first"]) + def test_windows_wins_a_tie_with_posix_in_any_order(self, reverse) -> None: + """A POSIX rule and a WINDOWS rule with the same number of names can + both match a path such as ``//srv/share/x``, and the WINDOWS rule + applies wherever it is in the list (How Jobs Are Run, "Applying Path + Mapping Rules within a Job Template", item 3). openjd-expr 0.14.0 + (openjd-rs#472); before, the first one applied.""" + from openjd.expr import parse_expression + + rules = [ + PathMappingRule( + source_path_format=PathFormat.WINDOWS, + source_path="\\\\srv\\share", + destination_path="/win", + ), + PathMappingRule( + source_path_format=PathFormat.POSIX, + source_path="//srv/share", + destination_path="/posix", + ), + ] + if reverse: + rules.reverse() + profile = ExprProfile().with_host_context(HostContext.with_rules(rules)) + parsed = parse_expression("apply_path_mapping('//srv/share/x')") + result = parsed.evaluate(profile=profile, path_format=PathFormat.POSIX) + assert result.item() == "/win/x" + + def test_the_last_rule_for_the_same_location_wins(self, tmp_path) -> None: + """Rules whose source paths have the same components name the same + location, and the last one given applies (How Jobs Are Run, "Applying + Path Mapping Rules within a Job Template", item 3). openjd-expr 0.14.0 + (openjd-rs#472); before, the first one applied. ``/mnt`` and ``/mnt/`` + name the same location.""" + from openjd.expr import parse_expression + + first = str(tmp_path / "first") + last = str(tmp_path / "last") + rules = [ + PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/mnt", destination_path=first + ), + PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/mnt/", destination_path=last + ), + ] + profile = ExprProfile().with_host_context(HostContext.with_rules(rules)) + parsed = parse_expression("apply_path_mapping('/mnt/file.exr')") + assert parsed.evaluate(profile=profile).item() == last + os.sep + "file.exr" + def test_format_string_resolve_applies_rules(self, tmp_path) -> None: from openjd.expr import FormatString, SymbolTable diff --git a/test/openjd/expr/test_paths.py b/test/openjd/expr/test_paths.py index 2ae13366..0787d5e7 100644 --- a/test/openjd/expr/test_paths.py +++ b/test/openjd/expr/test_paths.py @@ -4,7 +4,7 @@ import sys import pytest -from pathlib import PurePath +from pathlib import PurePath, PurePosixPath from openjd.expr import evaluate_expression, ExprValue, ExpressionError, SymbolTable from openjd.expr import PathFormat @@ -71,6 +71,24 @@ def test_path_join(self, tmp_path) -> None: result = evaluate_expression("Dir / 'subdir' / 'file.txt'", values=symtab) assert str(result) == str(d / "subdir" / "file.txt") + @pytest.mark.parametrize( + "left, right, expected", + [ + pytest.param("", "a", "a", id="empty left"), + pytest.param(".", "a", "a", id="dot left"), + pytest.param("a", "", "a", id="empty right"), + pytest.param("/x", "", "/x", id="empty right after a root"), + pytest.param("", "", ".", id="both empty"), + ], + ) + def test_path_join_with_an_empty_operand(self, left: str, right: str, expected: str) -> None: + """An empty operand gives the other operand, and two give ``.``, as + ``pathlib`` does (Expression Language §2.1.5). openjd-expr 0.14.0 + (openjd-rs#516); before, ``path('') / 'a'`` gave ``./a`` and + ``path('a') / ''`` gave ``a/``.""" + result = evaluate_expression(f"path('{left}') / '{right}'", path_format=PathFormat.POSIX) + assert str(result) == expected == str(PurePosixPath(left) / right) + def test_path_concat(self, tmp_path) -> None: p = tmp_path / "output" / "file" symtab = SymbolTable({"P": _p(p)}) diff --git a/test/openjd/expr/test_range_expr.py b/test/openjd/expr/test_range_expr.py index 93e25821..f039abe1 100644 --- a/test/openjd/expr/test_range_expr.py +++ b/test/openjd/expr/test_range_expr.py @@ -4,7 +4,7 @@ import pytest from openjd.expr import evaluate_expression, SymbolTable, ExpressionError, TypeCode -from openjd.expr import RangeExpr, IntRange +from openjd.expr import RangeExpr, RangeExprError, IntRange class TestRangeExpr: @@ -184,6 +184,71 @@ def test_from_list_invalid_type_raises(self) -> None: RangeExpr.from_list([1.5]) +class TestRangeExprCanonicalForm: + """Since openjd-expr 0.13.0 (openjd-rs#469) a ``RangeExpr`` is kept and + written in the canonical form of Template Schemas §3.4.1.1.1, and two + compare equal exactly when they have the same values. Before, the string + and ``ranges()`` followed the input's elements, so ``"1,3,5,7,9"`` stayed + five runs and did not equal ``"1-9:2"``. The pure-Python + ``openjd.model._range_expr.IntRangeExpr`` still follows the input; see + ``test_v0_range_list_renders_in_canonical_form`` in + ``test/openjd/model_v1/test_known_gaps.py``.""" + + @pytest.mark.parametrize( + "text, expected", + [ + pytest.param("1,3,5,7,9", "1-9:2", id="singles join a run"), + pytest.param("1,2,4,6", "1,2-6:2", id="a pair leaves the run"), + pytest.param("1-10:2,11", "1-11:2", id="a run extends across elements"), + pytest.param("1-10:4", "1-9:4", id="the end is the last value"), + pytest.param("13-18,19", "13-19", id="adjacent elements merge"), + ], + ) + def test_str_is_canonical(self, text: str, expected: str) -> None: + assert str(RangeExpr(text)) == expected + assert repr(RangeExpr(text)) == f'RangeExpr("{expected}")' + + def test_ranges_are_the_canonical_runs(self) -> None: + """A run of two values is written as two single values, so ``"1-2"`` + has two ranges.""" + assert RangeExpr("1-2").ranges() == [IntRange(1, 1, 1), IntRange(2, 2, 1)] + assert RangeExpr("1,3,5,7,9").ranges() == [IntRange(1, 9, 2)] + + def test_equality_is_by_value(self) -> None: + a = RangeExpr("1,3,5,7,9") + b = RangeExpr("1-9:2") + assert a == b + assert hash(a) == hash(b) + + def test_values_span_the_64_bit_range(self) -> None: + """openjd-expr 0.13.0 (openjd-rs#469) widened the value domain from + magnitudes below 2**62 to every 64-bit integer except the most + negative one. Before, both of these raised an overflow error.""" + assert str(RangeExpr("-9223372036854775807,9223372036854775807")) == ( + "-9223372036854775807,9223372036854775807" + ) + assert str(RangeExpr("4611686018427387904")) == "4611686018427387904" + + def test_parsing_accepts_more_than_10000_elements(self) -> None: + """openjd-expr 0.14.0 (openjd-rs#470) raised the element cap from + 10,000 to 50,000. Before, 12,000 elements raised ``RangeExprError``.""" + r = RangeExpr(",".join(str(i * 3) for i in range(12_000))) + assert r.ranges() == [IntRange(0, 35_997, 3)] + + def test_from_list_caps_the_canonical_runs(self) -> None: + """openjd-expr 0.14.0 (openjd-rs#470) caps every constructor at 50,000 + runs of the canonical form. 25,000 pairs are 50,000 runs, and one more + value is one too many.""" + values = [v for i in range(25_000) for v in (i * 10, i * 10 + 1)] + assert len(RangeExpr.from_list(values).ranges()) == 50_000 + with pytest.raises(RangeExprError) as exc_info: + RangeExpr.from_list(values + [10**7]) + assert str(exc_info.value) == ( + "Range expression has too many sub-ranges in canonical form (50001); " + "maximum is 50000" + ) + + class TestExprValueRangeExprProtocols: """Test len, indexing, and iteration on ExprValue with range_expr type.""" diff --git a/test/openjd/expr/test_strings.py b/test/openjd/expr/test_strings.py index f50d69f5..9792f66b 100644 --- a/test/openjd/expr/test_strings.py +++ b/test/openjd/expr/test_strings.py @@ -1338,3 +1338,46 @@ def test_cmd_set_variable_roundtrip(self) -> None: # limitation — set requires ^& escaping even inside double quotes. cmd = evaluate_expression(r"'set ' + repr_cmd('MY_VAR=hello world')").item() assert self._cmd_eval(f"{cmd}\necho %MY_VAR%") == "hello world" + + +class TestComputedFloatText: + @pytest.mark.parametrize( + "expr, expected", + [ + pytest.param("string(1000000000000000.25 * 1.0)", "1000000000000000.2", id="scalar"), + pytest.param( + "string([1000000000000000.25 * 1.0])", "[1000000000000000.2]", id="list item" + ), + pytest.param( + "string(float('1000000000000000.25'))", "1000000000000000.2", id="from a string" + ), + ], + ) + def test_a_shortest_digit_tie_rounds_as_repr_does(self, expr: str, expected: str) -> None: + """A computed float is written as ``repr(float)`` writes it (Expression + Language §1.3.4). 1000000000000000.25 lies halfway between the two + shortest candidates, and ``repr`` rounds the tie to even. Since + openjd-expr 0.15.0 (openjd-rs#539) the digits come from ryu, which does + the same; before, Rust's shortest form rounded up, giving + ``1000000000000000.3``.""" + assert repr(1000000000000000.25) == "1000000000000000.2" + assert evaluate_expression(expr).item() == expected + + +class TestNumericStringCoercion: + @pytest.mark.parametrize( + "expr, target, expected", + [ + pytest.param("' 7 '", "int", 7, id="int"), + pytest.param("['1', ' 2']", "list[int]", [1, 2], id="list[int]"), + ], + ) + def test_white_space_around_the_number_is_ignored( + self, expr: str, target: str, expected: object + ) -> None: + """Expression Language §1.2.3: coercing a string to a number ignores the + white space around it. Since openjd-expr 0.15.0 (openjd-rs#530); before, + ``' 7 '`` raised ``Cannot convert ' 7 ' to int``.""" + from openjd.expr import ExprType + + assert evaluate_expression(expr, target_type=ExprType(target)).item() == expected diff --git a/test/openjd/expr/test_unresolved_eval.py b/test/openjd/expr/test_unresolved_eval.py index 1242ee55..03eca8ab 100644 --- a/test/openjd/expr/test_unresolved_eval.py +++ b/test/openjd/expr/test_unresolved_eval.py @@ -361,31 +361,37 @@ def test_empty_list_unchanged(self) -> None: class TestUnknownListComprehensions: - """Test list comprehension with unresolved iterables.""" + """Test list comprehension with unresolved iterables. + + Since openjd-expr 0.13.0 (openjd-rs#462) the element type is nullable, + ``list[T?]`` rather than ``list[T]``: a run may keep no element, and the + empty list has type ``list[nulltype]`` (Expression Language §1.3.7), so the + unresolved type has to cover it. + """ def test_unknown_list_iterable(self) -> None: - """[x for x in unresolved[list[int]]] -> unresolved[list[int]].""" + """[x for x in unresolved[list[int]]] -> unresolved[list[int?]].""" values = SymbolTable({"X": ExprValue.unresolved("list[int]")}) result = evaluate_expression("[x for x in X]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") def test_unknown_list_with_body_expr(self) -> None: - """[x + 1 for x in unresolved[list[int]]] -> unresolved[list[int]].""" + """[x + 1 for x in unresolved[list[int]]] -> unresolved[list[int?]].""" values = SymbolTable({"X": ExprValue.unresolved("list[int]")}) result = evaluate_expression("[x + 1 for x in X]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") def test_unknown_range_iterable(self) -> None: - """[x for x in unresolved[range_expr]] -> unresolved[list[int]].""" + """[x for x in unresolved[range_expr]] -> unresolved[list[int?]].""" values = SymbolTable({"X": ExprValue.unresolved("range_expr")}) result = evaluate_expression("[x for x in X]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") def test_unknown_iterable_with_transform(self) -> None: - """[string(x) for x in unresolved[list[int]]] -> unresolved[list[string]].""" + """[string(x) for x in unresolved[list[int]]] -> unresolved[list[string?]].""" values = SymbolTable({"X": ExprValue.unresolved("list[int]")}) result = evaluate_expression("[string(x) for x in X]", values=values) - assert result.type == ExprType("unresolved[list[string]]") + assert result.type == ExprType("unresolved[list[string?]]") class TestUnknownCoercion: @@ -740,18 +746,19 @@ class TestConcreteIterableUnresolvedFilter: the one stage where the iterable is concrete and the filter is not. Extends ``TestUnknownListComprehensions`` above, which covers the - unresolved-iterable path. + unresolved-iterable path. The element type is nullable for the reason given + there. """ def test_concrete_list_iterable(self) -> None: values = SymbolTable({"Skip": ExprValue.unresolved(ExprType("string"))}) result = evaluate_expression("[f for f in 'a,b,c'.split(',') if f != Skip]", values=values) - assert result.type == ExprType("unresolved[list[string]]") + assert result.type == ExprType("unresolved[list[string?]]") def test_concrete_range_iterable(self) -> None: values = SymbolTable({"N": ExprValue.unresolved(ExprType("int"))}) result = evaluate_expression("[x * 2 for x in range(3) if x > N]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") def test_the_body_type_is_derived_under_an_unresolved_loop_variable(self) -> None: """``10 // 0`` would raise if the body were evaluated on the concrete element @@ -761,7 +768,7 @@ def test_the_body_type_is_derived_under_an_unresolved_loop_variable(self) -> Non each element".""" values = SymbolTable({"N": ExprValue.unresolved(ExprType("int"))}) result = evaluate_expression("[10 // x for x in [0, 2] if x > N]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") def test_a_filter_short_circuiting_before_the_unresolved_term(self) -> None: """For ``x = 0`` the filter is decided concretely (``0 > 0`` is false); for @@ -769,12 +776,14 @@ def test_a_filter_short_circuiting_before_the_unresolved_term(self) -> None: rather than returned as a partial list.""" values = SymbolTable({"B": ExprValue.unresolved(ExprType("bool"))}) result = evaluate_expression("[x for x in [0, 1, 2] if x > 0 and B]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") def test_a_filter_that_can_never_be_a_boolean_is_still_rejected(self) -> None: """Negative control. The change is not "stop checking the filter type" — a - filter whose type can never be a boolean errors on both paths. A mutation that - widened the fix to swallow this would pass every case above.""" + filter whose type can never be a boolean errors over a concrete, non-empty + iterable. A mutation that widened the fix to swallow this would pass every + case above. Over an unresolved iterable see + ``test_a_never_boolean_filter_over_an_unresolved_iterable_keeps_nothing``.""" values = SymbolTable({"Skip": ExprValue.unresolved(ExprType("string"))}) with pytest.raises(ExpressionError) as exc_info: evaluate_expression("[x for x in [1, 2] if Skip]", values=values) @@ -787,7 +796,19 @@ def test_the_unresolved_iterable_path_still_concludes_unresolved(self) -> None: {"X": ExprValue.unresolved("list[int]"), "N": ExprValue.unresolved("int")} ) result = evaluate_expression("[x for x in X if x > N]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") + + def test_a_never_boolean_filter_over_an_unresolved_iterable_keeps_nothing(self) -> None: + """Since openjd-expr 0.13.0 (openjd-rs#462), a filter that can never be a + boolean fails every run that evaluates it, so only a run that keeps no + element succeeds, and the result is the empty list's type rather than an + error (Expression Language §1.3.7). Before, this raised + ``List comprehension filter must be a boolean, got int``.""" + values = SymbolTable( + {"L": ExprValue.unresolved("list[int]"), "P": ExprValue.unresolved("int")} + ) + result = evaluate_expression("[x for x in L if P]", values=values) + assert result.type == ExprType("unresolved[list[nulltype]]") class TestBoolOpBudgetErrorsPropagate: diff --git a/test/openjd/model_v0/format_strings/test_expr_node.py b/test/openjd/model_v0/format_strings/test_expr_node.py index e54fa7fe..6976d463 100644 --- a/test/openjd/model_v0/format_strings/test_expr_node.py +++ b/test/openjd/model_v0/format_strings/test_expr_node.py @@ -355,6 +355,33 @@ def test_evaluate_value_returns_typed_engine_value(self): assert str(value) == "1.50" +class TestHostRules: + """A session sets ``expr_host_rules`` on the symbol table, which gives the + engine a ``HostContext.with_rules`` profile.""" + + def test_an_invalid_rule_raises_an_expression_error(self): + """Since openjd-expr 0.14.0 (openjd-rs#472) ``HostContext.with_rules`` + rejects a rule whose source path is not absolute (How Jobs Are Run, + "Path Mapping"), so every EXPR expression evaluated under such a set of + rules fails, even one that maps no path. The failure is the model's + ``ExpressionError``, not a bare ``ValueError``.""" + from openjd.expr import PathMappingRule + + st = SymbolTable() + st.expr_host_rules = [ + PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="rel", destination_path="/b" + ) + ] + node = parse_format_string_expr("1 + 2", context=_ctx(["EXPR"])) + with pytest.raises(ExpressionError) as excinfo: + node.evaluate(symtab=st) + assert str(excinfo.value) == ( + "path mapping rule 0: source_path 'rel' must be an absolute POSIX path, " + "starting with '/'" + ) + + class TestFormatStringResolveValue: """`FormatString.resolve_value` (RFC 0005/0006 typed resolution): a whole-field `{{ ... }}` EXPR expression resolves to the engine's typed diff --git a/test/openjd/model_v1/test_create_job.py b/test/openjd/model_v1/test_create_job.py index dc3b32a2..4e825457 100644 --- a/test/openjd/model_v1/test_create_job.py +++ b/test/openjd/model_v1/test_create_job.py @@ -2973,3 +2973,178 @@ def test_two_missing_extensions_are_listed_sorted(self) -> None: "missing EXPR, FEATURE_BUNDLE_1. An application that does not support an extension " "should reject the template at decode via its supported-extensions list." ) + + +class TestIntRangeListFormatStringElements: + """Since openjd-model 0.15.0 (openjd-rs#486) an INT range list element may + be a format string. ``create_job`` resolves it and requires an integer + (Template Schemas §3.4.1, Expression Language §1.3.2).""" + + @staticmethod + def _create(value: str) -> Any: + decoded = decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "parameterDefinitions": [{"name": "X", "type": "STRING"}], + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "F", "type": "INT", "range": [1, "{{Param.X}}"]} + ] + }, + "script": {"actions": {"onRun": {"command": "echo"}}}, + } + ], + } + ) + return create_job(job_template=decoded, job_parameter_values={"X": value}) + + def test_an_integer_value_resolves(self) -> None: + job = self._create(" 7 ") + space = job.steps[0].parameterSpace + assert space is not None + assert space.taskParameterDefinitions["F"].range == [1, 7] + + @pytest.mark.parametrize("value", ["abc", "1.5"]) + def test_a_value_that_is_not_an_integer_fails(self, value: str) -> None: + from openjd.expr import ExpressionError + + with pytest.raises(ExpressionError) as excinfo: + self._create(value) + assert ( + str(excinfo.value) == f"Task parameter 'F' range[1]: value '{value}' is not an integer" + ) + + +def _preprocess(parameter_definitions: list[dict], values: dict, tmp_path: Path) -> dict: + """Decode a one-step template with ``parameter_definitions`` (with EXPR, for + the list types) and return ``preprocess_job_parameters``' values as text.""" + job_template = decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "parameterDefinitions": parameter_definitions, + "steps": minimal_steps_v2023_09, + }, + supported_extensions=["EXPR"], + ) + result = preprocess_job_parameters( + job_template=job_template, + job_parameter_values=values, + job_template_dir=tmp_path, + current_working_dir=tmp_path, + ) + return {name: value.value for name, value in result.items()} + + +class TestFloatGivenAsANumber: + """A FLOAT given as a YAML or JSON number is written the way Python's + ``repr(float)`` writes it, with zero always ``0.0`` (Expression Language + §1.3.4). A FLOAT given as a string keeps its spelling (Template Schemas §7.5). + + openjd-model 0.16.0 and openjd-expr 0.15.0 (openjd-rs#539) changed three + things here. A FLOAT parameter's ``default`` given as a number used to keep + its own digits, so ``2`` stayed ``2`` and ``1e20`` became + ``100000000000000000000``. A shortest-digit tie such as + 1000000000000000.25, in a value or a list item, rounded up + (``1000000000000000.3``) where ``repr`` rounds to even. And a LIST[FLOAT] + value given as JSON text could read a double one ULP off. Other values given + as a Python number, and other LIST[FLOAT] default items, were already + written as ``repr`` writes them; ``test_value`` and + ``test_list_default_items`` pin them. + """ + + @pytest.mark.parametrize( + "default, expected", + [ + pytest.param(2, "2.0", id="int"), + pytest.param(1e20, "1e+20", id="large"), + pytest.param(0.00001, "1e-05", id="small"), + pytest.param(0, "0.0", id="zero"), + pytest.param(2.5, "2.5", id="plain"), + pytest.param("2", "2", id="string keeps its spelling"), + ], + ) + def test_default(self, default: Any, expected: str, tmp_path: Path) -> None: + defs = [{"name": "F", "type": "FLOAT", "default": default}] + assert _preprocess(defs, {}, tmp_path) == {"F": expected} + + @pytest.mark.parametrize("value", [2, 2.0]) + def test_value(self, value: Any, tmp_path: Path) -> None: + assert _preprocess([{"name": "F", "type": "FLOAT"}], {"F": value}, tmp_path) == {"F": "2.0"} + + def test_list_default_items(self, tmp_path: Path) -> None: + defs = [{"name": "L", "type": "LIST[FLOAT]", "default": [2, 1e20]}] + assert _preprocess(defs, {}, tmp_path) == {"L": "[2.0, 1e+20]"} + + def test_a_shortest_digit_tie_rounds_as_repr_does(self, tmp_path: Path) -> None: + tie = 1000000000000000.2 + assert repr(tie) == "1000000000000000.2" + assert _preprocess([{"name": "F", "type": "FLOAT"}], {"F": tie}, tmp_path) == { + "F": "1000000000000000.2" + } + defs = [{"name": "L", "type": "LIST[FLOAT]", "default": [tie]}] + assert _preprocess(defs, {}, tmp_path) == {"L": "[1000000000000000.2]"} + + def test_list_value_given_as_json_text_reads_the_nearest_double(self, tmp_path: Path) -> None: + """Before, ``2.2250738585072011e-308`` read as ``2.2250738585072014e-308`` + and ``1000000000000000.25`` as ``1000000000000000.4``.""" + text = "[1000000000000000.25, 2.2250738585072011e-308]" + expected = ( + "[" + ", ".join(repr(v) for v in [1000000000000000.25, 2.2250738585072011e-308]) + "]" + ) + assert expected == "[1000000000000000.2, 2.225073858507201e-308]" + defs = [{"name": "L", "type": "LIST[FLOAT]"}] + assert _preprocess(defs, {"L": text}, tmp_path) == {"L": expected} + + +class TestNumericStringsInListParameters: + """openjd-model 0.16.0 (openjd-rs#538). A list value given as a native + Python list of numeric strings is coerced item by item, as the same value + given as JSON text is (Template Schemas §2.13, §2.14, §2.16). Before, a + LIST[INT] or LIST[FLOAT] list raised ``Cannot coerce list to LIST[INT]``, + and a LIST[LIST[INT]] one kept its items as strings. A LIST[FLOAT] default + item written as a string keeps its spelling; before, ``"2.50"`` became + ``2.5``.""" + + @pytest.mark.parametrize( + "param_type, value, expected", + [ + pytest.param("LIST[INT]", ["1", " 2"], "[1, 2]", id="LIST[INT]"), + pytest.param("LIST[FLOAT]", ["1.50", " 2"], "[1.50, 2]", id="LIST[FLOAT]"), + pytest.param("LIST[LIST[INT]]", [["1", " 2"]], "[[1, 2]]", id="LIST[LIST[INT]]"), + ], + ) + def test_native_list_value( + self, param_type: str, value: list, expected: str, tmp_path: Path + ) -> None: + defs = [{"name": "L", "type": param_type}] + assert _preprocess(defs, {"L": value}, tmp_path) == {"L": expected} + + def test_a_bad_item_names_its_position(self, tmp_path: Path) -> None: + with pytest.raises(DecodeValidationError) as exc_info: + _preprocess([{"name": "L", "type": "LIST[INT]"}], {"L": ["1", "x"]}, tmp_path) + assert str(exc_info.value) == ( + "Parameter 'L': item[1]: Value 'x' is not a valid integer or integer string." + ) + + def test_list_float_default_string_item_keeps_its_spelling(self, tmp_path: Path) -> None: + defs = [{"name": "L", "type": "LIST[FLOAT]", "default": [1.50, "2.50"]}] + assert _preprocess(defs, {}, tmp_path) == {"L": "[1.5, 2.50]"} + + +class TestWhiteSpaceAroundNumericStrings: + """Since openjd-model 0.16.0 (openjd-rs#530) white space around a number in + a string value for an INT or FLOAT parameter is ignored (Template Schemas + §7.5). Before, ``" 7 "`` was ``not a valid integer``.""" + + def test_int(self, tmp_path: Path) -> None: + assert _preprocess([{"name": "I", "type": "INT"}], {"I": " 7 "}, tmp_path) == {"I": "7"} + + def test_float_keeps_its_spelling(self, tmp_path: Path) -> None: + defs = [{"name": "F", "type": "FLOAT"}] + assert _preprocess(defs, {"F": " 1.50 "}, tmp_path) == {"F": "1.50"} diff --git a/test/openjd/model_v1/test_known_gaps.py b/test/openjd/model_v1/test_known_gaps.py index a6e18b17..9966817d 100644 --- a/test/openjd/model_v1/test_known_gaps.py +++ b/test/openjd/model_v1/test_known_gaps.py @@ -93,3 +93,47 @@ def test_v0_joins_list_path_defaults_to_the_template_directory() -> None: def test_v0_normalizes_a_relative_submitted_path() -> None: out = _v0_preprocess({"name": "P", "type": "PATH"}, {"P": "sub/../other"}) assert out["P"] == "/tmp/cwd/other" + + +# ── v0 writes a NONCONTIGUOUS chunk outside canonical form ── +# +# This runs v0, not the binding. Template Schemas §3.4.1.5 writes a NONCONTIGUOUS +# chunk in the canonical form of §3.4.1.1.1, which keeps a run only when it has +# three or more values, so ``[1, 2, 4, 6]`` is ``1,2-6:2``. v0's +# ``IntRangeExpr.from_list`` commits to the first gap as the step and writes +# ``1,2,4,6``, and ``IntRangeExpr.from_str`` keeps the input's elements. +# openjd-model 0.14.0 (openjd-rs#469) brought v1 to canonical form; +# ``TestNoncontiguousChunkRendering`` in ``test_step_param_space_iter.py`` pins +# the v1 side. + +_V0_CANONICAL_GAP = "v0 IntRangeExpr does not write canonical form (§3.4.1.1.1)" + + +@pytest.mark.xfail(strict=True, reason=_V0_CANONICAL_GAP) +@pytest.mark.parametrize( + "values, expected", + [ + pytest.param([1, 2, 4, 6], "1,2-6:2", id="pair then even progression"), + pytest.param([1, 3, 4, 5], "1,3-5", id="gap then consecutive run"), + ], +) +def test_v0_range_list_renders_in_canonical_form(values: list[int], expected: str) -> None: + from openjd.model._range_expr import IntRangeExpr + + assert str(IntRangeExpr.from_list(values)) == expected + + +@pytest.mark.xfail(strict=True, reason=_V0_CANONICAL_GAP) +@pytest.mark.parametrize( + "text, expected", + [ + pytest.param("1,3,5,7,9", "1-9:2", id="singles join a run"), + pytest.param("1,2,4,6", "1,2-6:2", id="a pair leaves the run"), + ], +) +def test_v0_range_string_renders_in_canonical_form(text: str, expected: str) -> None: + """``openjd.expr.RangeExpr`` has written canonical form since openjd-expr + 0.13.0; ``TestRangeExprCanonicalForm`` in ``test/openjd/expr`` pins it.""" + from openjd.model._range_expr import IntRangeExpr + + assert str(IntRangeExpr.from_str(text)) == expected diff --git a/test/openjd/model_v1/test_merge_job_parameters.py b/test/openjd/model_v1/test_merge_job_parameters.py index 1ac4befa..d733ae2b 100644 --- a/test/openjd/model_v1/test_merge_job_parameters.py +++ b/test/openjd/model_v1/test_merge_job_parameters.py @@ -227,6 +227,15 @@ def test_list_float(self) -> None: assert m["default"] == [1.5, 2.5] assert all(isinstance(v, float) for v in m["default"]) + def test_list_float_with_string_items(self) -> None: + """Since openjd-model 0.16.0 (openjd-rs#538) the crate writes a string + item of a LIST[FLOAT] default as a JSON string, to keep its spelling for + job creation. The binding reads it as a ``float``, as the template's + ``default`` does; a ``float`` can't hold the spelling.""" + m = self._merge_one(param_type="LIST[FLOAT]", default=[1.5, "2.50", " 3 "]) + assert m["default"] == [1.5, 2.5, 3.0] + assert all(isinstance(v, float) for v in m["default"]) + def test_list_string(self) -> None: m = self._merge_one(param_type="LIST[STRING]", default=["a", "b"]) assert m["default"] == ["a", "b"] diff --git a/test/openjd/model_v1/test_parse.py b/test/openjd/model_v1/test_parse.py index 263bf9e4..f0a1311e 100644 --- a/test/openjd/model_v1/test_parse.py +++ b/test/openjd/model_v1/test_parse.py @@ -369,6 +369,49 @@ def test_invalid_yaml_raises(self) -> None: with pytest.raises(DecodeValidationError): decode_job_template_str("specificationVersion: nope") + @pytest.mark.parametrize("scalar", ["tRUE", "fALSE", "nULL"]) + def test_mixed_case_bool_and_null_spellings_are_strings(self, scalar: str) -> None: + """YAML 1.2 Core reads only ``true``/``True``/``TRUE`` (and the same for + ``false`` and ``null``) as booleans and null. With serde-saphyr 2.1, + which Cargo.lock resolves since the move to openjd-model 0.15.0 + (openjd-rs adopted it in #487), other spellings are strings, as they + are in the pure-Python reference. Before, with serde-saphyr 1.2, + ``tRUE`` was a boolean and ``nULL`` was null.""" + t = decode_job_template_str( + "specificationVersion: jobtemplate-2023-09\n" + "name: T\n" + "parameterDefinitions:\n" + f"- {{name: P, type: STRING, default: {scalar}}}\n" + "steps:\n" + "- name: S\n" + " script: {actions: {onRun: {command: echo}}}\n" + ) + assert t.parameter_definitions is not None + assert t.parameter_definitions[0].default == scalar + + @pytest.mark.parametrize( + "ptype, scalar, expected", + [ + pytest.param("FLOAT", "!!float 1.5", 1.5, id="float"), + pytest.param("INT", "!!int 3", 3, id="int"), + ], + ) + def test_a_core_tagged_scalar_decodes(self, ptype: str, scalar: str, expected: Any) -> None: + """With serde-saphyr 2.1 a YAML 1.2 Core tag on a scalar is read as its + type. Before, with serde-saphyr 1.2, the decode failed with + ``cannot deserialize tagged scalar into string``.""" + t = decode_job_template_str( + "specificationVersion: jobtemplate-2023-09\n" + "name: T\n" + "parameterDefinitions:\n" + f"- {{name: P, type: {ptype}, default: {scalar}}}\n" + "steps:\n" + "- name: S\n" + " script: {actions: {onRun: {command: echo}}}\n" + ) + assert t.parameter_definitions is not None + assert t.parameter_definitions[0].default == expected + class TestDecodeEnvironmentTemplateStr: """``decode_environment_template_str`` wrapper: parses YAML or diff --git a/test/openjd/model_v1/test_step_param_space_def.py b/test/openjd/model_v1/test_step_param_space_def.py index a26c0837..b609d736 100644 --- a/test/openjd/model_v1/test_step_param_space_def.py +++ b/test/openjd/model_v1/test_step_param_space_def.py @@ -181,6 +181,26 @@ def test_range_expression_string(self) -> None: assert isinstance(d.range, FormatString) assert d.range.raw() == "1-10:2" + def test_list_with_format_string(self) -> None: + """Since openjd-model 0.15.0 (openjd-rs#486) an INT list element may + be a format string, and it is exposed as a ``FormatString`` beside the + ``int`` literals, as the pure-Python reference exposes its + ``list[int | TaskParameterStringValue]``.""" + step = _decode_step( + parameter_definitions=[{"name": "X", "type": "INT"}], + parameter_space={ + "taskParameterDefinitions": [ + {"name": "F", "type": "INT", "range": [1, "{{Param.X}}", 3]}, + ], + }, + ) + d = step.parameter_space.task_parameter_definitions[0] + items = d.range + assert items[0] == 1 + assert isinstance(items[1], FormatString) + assert items[1].raw() == "{{Param.X}}" + assert items[2] == 3 + class TestFloatTaskParameterDefinition: def test_list_of_literals(self) -> None: @@ -198,9 +218,15 @@ def test_list_of_literals(self) -> None: def test_list_with_format_string(self) -> None: """``FloatRangeItem`` is a union — list elements can be - either literals (Python ``float``) or ``FormatString``s.""" + either literals (Python ``float``) or ``FormatString``s. + + ``Param.X`` is declared because since openjd-model 0.15.0 + (openjd-rs#486) a FLOAT list element's format string is checked at + decode time, as the pure-Python reference already did; see + ``test_list_element_with_undefined_reference_is_rejected``.""" step = _decode_step( extensions=["EXPR"], + parameter_definitions=[{"name": "X", "type": "FLOAT"}], parameter_space={ "taskParameterDefinitions": [ { @@ -218,6 +244,29 @@ def test_list_with_format_string(self) -> None: assert items[1].raw() == "{{Param.X}}" assert items[2] == 3.5 + def test_list_element_with_undefined_reference_is_rejected(self) -> None: + """openjd-model 0.15.0 (openjd-rs#486) checks the format strings in + FLOAT list elements at decode time. Before, an undefined reference + here decoded and failed only at job creation; the pure-Python + reference rejects it at decode time too.""" + with pytest.raises(ModelValidationError) as exc_info: + _decode_step( + extensions=["EXPR"], + parameter_space={ + "taskParameterDefinitions": [ + {"name": "F", "type": "FLOAT", "range": [1.5, "{{Param.X}}"]}, + ], + }, + ) + assert str(exc_info.value) == ( + "1 validation error for JobTemplate\n" + "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range[1]:\n" + "\tFailed to parse interpolation expression at [0, 11]. " + "Undefined variable: 'Param.X'.\n" + " Param.X\n" + " ~~~~~~^" + ) + class TestStringTaskParameterDefinition: def test_list_of_format_strings(self) -> None: @@ -285,6 +334,33 @@ def test_minimal_chunks(self) -> None: assert chunks.target_runtime_seconds is None assert chunks.range_constraint == "CONTIGUOUS" + def test_list_with_format_string(self) -> None: + """``CHUNK[INT]`` shares the INT range list type, so since + openjd-model 0.15.0 (openjd-rs#486) its list elements are also + ``int`` or ``FormatString``.""" + step = _decode_step( + extensions=["TASK_CHUNKING"], + parameter_definitions=[{"name": "X", "type": "INT"}], + parameter_space={ + "taskParameterDefinitions": [ + { + "name": "F", + "type": "CHUNK[INT]", + "range": ["{{Param.X}}", 7], + "chunks": { + "defaultTaskCount": 4, + "rangeConstraint": "NONCONTIGUOUS", + }, + } + ] + }, + ) + d = step.parameter_space.task_parameter_definitions[0] + items = d.range + assert isinstance(items[0], FormatString) + assert items[0].raw() == "{{Param.X}}" + assert items[1] == 7 + def test_chunks_with_target_runtime(self) -> None: """``targetRuntimeSeconds`` and a ``NONCONTIGUOUS`` constraint round-trip through the typed pyclass surface.""" diff --git a/test/openjd/model_v1/test_step_param_space_iter.py b/test/openjd/model_v1/test_step_param_space_iter.py index 957409fe..b611c688 100644 --- a/test/openjd/model_v1/test_step_param_space_iter.py +++ b/test/openjd/model_v1/test_step_param_space_iter.py @@ -658,6 +658,51 @@ def test_an_unparseable_element_is_still_rejected(self) -> None: self._rendered(["not-a-float"]) +class TestWholeFieldFloatRangeExpressionKeepsSpelling: + """A FLOAT task parameter whose ``range`` is one whole-field expression gives + each float the spelling it carries, as a range list element does (Expression + Language §1.3.4). openjd-model 0.16.0 (openjd-rs#540); before, the list was + rebuilt from the numbers, so ``["4.20", "5.30"]`` rendered ``4.2`` and + ``5.3``. (Before openjd-model 0.16.0, openjd-rs#538, a native list of + numeric strings couldn't be given for LIST[FLOAT] at all.)""" + + @staticmethod + def _rendered(parameter_definitions: list[dict], rng: Any, values: dict) -> list[str]: + template = { + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "parameterDefinitions": parameter_definitions, + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [{"name": "F", "type": "FLOAT", "range": rng}] + }, + "script": { + "actions": {"onRun": {"command": "echo", "args": ["{{Task.Param.F}}"]}} + }, + } + ], + } + job = create_job( + job_template=decode_job_template(template=template, supported_extensions=["EXPR"]), + job_parameter_values=values, + ) + return [params["F"].value for params in StepParameterSpaceIterator(step=job.steps[0])] + + @pytest.mark.parametrize( + "rng", ["{{ Param.LF }}", ["{{ Param.LF }}"]], ids=["whole field", "list element"] + ) + def test_a_list_parameter(self, rng: Any) -> None: + defs = [{"name": "LF", "type": "LIST[FLOAT]"}] + assert self._rendered(defs, rng, {"LF": ["4.20", "5.30"]}) == ["4.20", "5.30"] + + def test_a_float_parameter_in_a_list_expression(self) -> None: + defs = [{"name": "X", "type": "FLOAT"}] + assert self._rendered(defs, "{{ [Param.X] }}", {"X": "1.50"}) == ["1.50"] + + class TestChunkIntContains: """``__contains__`` must round-trip values yielded by a chunked iterator. The iterator yields ``TaskParameterValue`` instances @@ -1011,16 +1056,19 @@ def test_yielded_values_round_trip_through_contains(self) -> None: class TestNoncontiguousChunkRendering: - """A NONCONTIGUOUS chunk renders its values the way the v0 reference's - ``IntRangeExpr.from_list`` does: commit to the first gap as the step, and - consume a consecutive pair atomically. openjd-model 0.8.0 (openjd-rs#398). - 0.7.1 used a compressor that only collapsed runs of three or more agreeing - gaps, so ``[1, 2, 4, 6]`` rendered as ``1,2-6:2``. Every spelling parses to - the same integers, which is why nothing caught the divergence; the rendered - text is what a task sees in ``{{Task.Param.Frame}}``. - - Expected values are the v0 reference's output for the same template, checked - by running both paths side by side when this test was written. + """A NONCONTIGUOUS chunk renders its values in the canonical form of + Template Schemas §3.4.1.1.1, which the chunk definition (§3.4.1.5) requires. + The rendered text is what a task sees in ``{{Task.Param.Frame}}``. + + openjd-model 0.14.0 (openjd-rs#469) writes range expressions in canonical + form, which starts a run at the first value not yet in one and keeps it only + when it has three or more values, so ``[1, 2, 4, 6]`` renders ``1,2-6:2``. + From openjd-model 0.8.0 (openjd-rs#398) until then, the binding matched the + pure-Python reference's ``IntRangeExpr.from_list``, which commits to the + first gap as the step and consumes a consecutive pair atomically, rendering + ``1,2,4,6``. The pure-Python reference still does that, so the first two + cases below differ from it; that is a gap in the reference, not here. + Every spelling parses to the same integers. """ @staticmethod @@ -1062,11 +1110,11 @@ def _frames(range_values: list[str], default_task_count: int) -> list[str]: @pytest.mark.parametrize( "range_values, expected", [ - pytest.param(["1", "2", "4", "6"], "1,2,4,6", id="pair then even progression"), - pytest.param(["1", "3", "4", "5"], "1,3,4,5", id="gap then consecutive run"), + pytest.param(["1", "2", "4", "6"], "1,2-6:2", id="pair then even progression"), + pytest.param(["1", "3", "4", "5"], "1,3-5", id="gap then consecutive run"), pytest.param(["7", "8"], "7,8", id="lone pair"), pytest.param(["1", "3", "5", "7"], "1-7:2", id="control: uniform step compresses"), ], ) - def test_chunk_renders_like_the_reference(self, range_values: list[str], expected: str) -> None: + def test_chunk_renders_in_canonical_form(self, range_values: list[str], expected: str) -> None: assert self._frames(range_values, default_task_count=4) == [expected] diff --git a/test/openjd/sessions/test_failed_actions.py b/test/openjd/sessions/test_failed_actions.py new file mode 100644 index 00000000..a0303f93 --- /dev/null +++ b/test/openjd/sessions/test_failed_actions.py @@ -0,0 +1,197 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""How ``Session`` handles a failed action, through openjd-sessions 0.10.0. + +- openjd-rs#532: when ``run_task``, ``enter_environment`` or + ``exit_environment`` fails before its subprocess starts, for example because a parameter value can't be coerced, + the action fails with the error's message and the session becomes + ending-only, as the pure-Python ``openjd.sessions`` does. Before, the binding + saw an error it discards, so ``action_status`` stayed ``None`` and the + session went back to ``READY``. +- openjd-rs#524: an action that fails leaves the session ending-only, even + after ``exit_environment(keep_session_running=True)`` succeeds. Before, that + exit returned the session to ``READY``. +- openjd-rs#536: when ``exit_environment`` can't build its symbol table from + the ``resolved_symtab`` it is given, it runs ``onExit`` with the table the + environment was entered with. Before, ``onExit`` didn't run. +- openjd-rs#530: white space around a number in an INT or FLOAT parameter + value is ignored. Before, such a value failed before the action started, + which the binding didn't report, so the task silently didn't run. + +What a session does with a call it refuses is in ``test_refused_calls.py``. +""" + +from __future__ import annotations + +import sys +import time +from pathlib import Path +from typing import Any + +from openjd._openjd_rs import ( + ActionState, + Session, + SessionState, + _reconstruct_serialized_symtab, + create_environment, + decode_environment_template, + deserialize_step, +) + + +def _python(code: str) -> dict[str, Any]: + return {"command": sys.executable, "args": ["-c", code]} + + +def _step_script(action: dict[str, Any]) -> Any: + return deserialize_step({"name": "S", "script": {"actions": {"onRun": action}}}).script + + +def _environment(**actions: dict[str, Any]) -> Any: + return create_environment( + decode_environment_template( + template={ + "specificationVersion": "environment-2023-09", + "environment": {"name": "E", "script": {"actions": actions}}, + } + ) + ) + + +def _settle(session: Session) -> None: + deadline = time.monotonic() + 30 + while session.state == SessionState.RUNNING and time.monotonic() < deadline: + time.sleep(0.05) + assert session.state != SessionState.RUNNING, "action did not finish within 30s" + + +def _session(tmp_path: Path, **kwargs: Any) -> Session: + return Session( + session_id="s", job_parameter_values={}, session_root_directory=str(tmp_path), **kwargs + ) + + +def test_a_task_that_fails_before_it_starts_fails_the_action(tmp_path: Path) -> None: + session = _session(tmp_path) + try: + session.run_task( + step_script=_step_script({"command": "echo", "args": ["{{Task.Param.F}}"]}), + task_parameter_values={"F": {"type": "INT", "value": "abc"}}, + ) + _settle(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.fail_message == ( + "Failed to coerce parameter 'Task.RawParam.F': " + "Cannot convert 'abc' to int: invalid digit found in string" + ) + assert session.state == SessionState.READY_ENDING + finally: + session.cleanup() + + +def test_an_environment_that_fails_before_it_starts_fails_the_action(tmp_path: Path) -> None: + """The environment still counts as entered, so it can be exited, and its + exit fails before ``onExit`` starts the same way and counts as exited.""" + session = Session( + session_id="s", + job_parameter_values={"P": {"type": "PATH", "value": "/a/" + "x" * 32_800}}, + session_root_directory=str(tmp_path), + ) + try: + identifier = session.enter_environment( + environment=_environment(onEnter=_python(""), onExit=_python("")) + ) + _settle(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.fail_message == ( + "Failed to apply path mapping to Param.P: " + "the path is 32803 bytes long, more than the limit of 32768 bytes" + ) + assert session.state == SessionState.READY_ENDING + assert session.environments_entered == [identifier] + session.exit_environment(identifier=identifier) + _settle(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert session.environments_entered == [] + finally: + session.cleanup() + + +def test_a_failed_task_keeps_the_session_ending_only(tmp_path: Path) -> None: + session = _session(tmp_path) + try: + identifier = session.enter_environment(environment=_environment(onEnter=_python(""))) + _settle(session) + session.run_task(step_script=_step_script(_python("raise SystemExit(1)"))) + _settle(session) + assert session.state == SessionState.READY_ENDING + session.exit_environment(identifier=identifier, keep_session_running=True) + _settle(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert session.state == SessionState.READY_ENDING + finally: + session.cleanup() + + +def test_on_exit_runs_with_the_enter_time_table(tmp_path: Path) -> None: + marker = tmp_path / "on-exit-ran" + session = _session(tmp_path) + try: + identifier = session.enter_environment( + environment=_environment( + onEnter=_python(""), + onExit=_python(f"open({str(marker)!r}, 'w').close()"), + ) + ) + _settle(session) + not_a_table = _reconstruct_serialized_symtab("{}") + session.exit_environment(identifier=identifier, resolved_symtab=not_a_table) + _settle(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert marker.exists() + assert session.state == SessionState.READY + finally: + session.cleanup() + + +def test_white_space_around_a_numeric_parameter_value_is_ignored(tmp_path: Path) -> None: + out = tmp_path / "out.txt" + session = Session( + session_id="s", + job_parameter_values={"I": {"type": "INT", "value": " 7 "}}, + session_root_directory=str(tmp_path), + ) + try: + session.run_task( + step_script=_step_script( + { + "command": sys.executable, + "args": [ + "-c", + "import sys; open(sys.argv[1], 'w').write(sys.argv[2] + '|' + sys.argv[3])", + str(out), + "{{Param.I}}", + "{{Task.Param.F}}", + ], + } + ), + task_parameter_values={"F": {"type": "FLOAT", "value": " 1.50 "}}, + ) + _settle(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert out.read_text() == "7|1.50" + finally: + session.cleanup() diff --git a/test/openjd/sessions/test_path_mapping_rules.py b/test/openjd/sessions/test_path_mapping_rules.py new file mode 100644 index 00000000..0fdb6ce9 --- /dev/null +++ b/test/openjd/sessions/test_path_mapping_rules.py @@ -0,0 +1,62 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``Session`` rejects an invalid path mapping rule. + +Since openjd-sessions 0.9.0 (openjd-rs#472) a session keeps its rules in a +``PathMapper``, which requires each POSIX or WINDOWS ``source_path`` to be +absolute (How Jobs Are Run, "Path Mapping"). Before, a relative rule was +accepted and matched relative paths. The rules are checked when the session is +constructed and when ``extend_path_mapping_rules`` adds to them. +""" + +from __future__ import annotations + +import time + +import pytest + +from openjd import _openjd_rs +from openjd._openjd_rs import Session +from openjd.expr import PathFormat, PathMappingRule + +# Registered at runtime by `register_renamed_exception`; the type stub doesn't +# declare the exception classes. +SessionError: type[Exception] = getattr(_openjd_rs, "SessionError") + +_GOOD = PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/a", destination_path="/b" +) +_RELATIVE = PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="rel", destination_path="/b" +) +_MESSAGE = ( + "Invalid path mapping rules: path mapping rule 1: source_path 'rel' must be an " + "absolute POSIX path, starting with '/'" +) + + +def test_construction_rejects_a_relative_source_path() -> None: + with pytest.raises(SessionError) as exc_info: + Session( + session_id=f"path-mapping-{time.time_ns()}", + job_parameter_values={}, + path_mapping_rules=[_GOOD, _RELATIVE], + ) + assert str(exc_info.value) == _MESSAGE + + +def test_extend_rejects_a_relative_source_path() -> None: + """The index counts from the first existing rule, so the one added rule + after one existing rule is rule 1.""" + session = Session( + session_id=f"path-mapping-{time.time_ns()}", + job_parameter_values={}, + path_mapping_rules=[_GOOD], + ) + try: + with pytest.raises(SessionError) as exc_info: + session.extend_path_mapping_rules([_RELATIVE]) + assert str(exc_info.value) == _MESSAGE + finally: + session.cleanup() diff --git a/test/openjd/sessions/test_refused_calls.py b/test/openjd/sessions/test_refused_calls.py new file mode 100644 index 00000000..ef3b9488 --- /dev/null +++ b/test/openjd/sessions/test_refused_calls.py @@ -0,0 +1,602 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A call the openjd-sessions crate refuses raises, as the Rust method returns ``Err``. + +The crate refuses a call before it changes anything: the session state, the +last action's status and the environments entered stay as they were. Each test +drives a sequence through the binding and checks the crate's documented result: +the exception class, its message (the crate's ``SessionError`` text), the state, +and that ``action_status`` still describes the previous action. + +Before, the binding handed every call to a background thread and discarded the +crate's ``Result``, so a refused call returned as if an action had started, ran +nothing, and left the previous action's status showing. openjd-sessions 0.10.0 +(openjd-rs#524) made that common: a failed action keeps the session ending-only +even after a successful ``exit_environment(keep_session_running=True)``. + +``clear_ending_only`` is the crate's way out of ending-only (openjd-rs#524). +""" + +from __future__ import annotations + +import sys +import time +from pathlib import Path +from typing import Any, Callable, Optional + +import pytest + +from openjd import _openjd_rs +from openjd._openjd_rs import ( + ActionState, + ActionStatus, + Session, + SessionState, + create_environment, + decode_environment_template, + deserialize_step, +) + +# Registered at runtime by `register_renamed_exception`; the type stub doesn't +# declare the exception classes. +SessionError: type[Exception] = getattr(_openjd_rs, "SessionError") + + +def _python(code: str) -> dict[str, Any]: + return {"command": sys.executable, "args": ["-c", code]} + + +def _step_script(code: str) -> Any: + return deserialize_step({"name": "S", "script": {"actions": {"onRun": _python(code)}}}).script + + +def _environment(name: str = "E", *, extensions: Optional[list[str]] = None, **actions: Any) -> Any: + template: dict[str, Any] = { + "specificationVersion": "environment-2023-09", + "environment": {"name": name, "script": {"actions": actions}}, + } + if extensions: + template["extensions"] = extensions + return create_environment( + decode_environment_template(template=template, supported_extensions=extensions) + ) + + +def _settle(session: Session) -> None: + deadline = time.monotonic() + 10 + busy = (SessionState.RUNNING, SessionState.CANCELING) + while session.state in busy and time.monotonic() < deadline: + time.sleep(0.02) + assert session.state not in busy, "action did not finish within 10s" + + +def _status(session: Session) -> Optional[tuple]: + status: Optional[ActionStatus] = session.action_status + if status is None: + return None + return (status.state, status.exit_code, status.fail_message, status.status_message) + + +@pytest.fixture +def session(tmp_path: Path): + s = Session(session_id="s", job_parameter_values={}, session_root_directory=str(tmp_path)) + try: + yield s + finally: + s.cleanup() + + +def _assert_refused( + session: Session, + call: Callable[[], object], + message: str, + error: type[Exception] = SessionError, +) -> None: + """``call`` raises ``error(message)`` and changes nothing.""" + state = session.state + status = _status(session) + entered = session.environments_entered + with pytest.raises(error) as exc_info: + call() + assert str(exc_info.value) == message + assert session.state == state + assert _status(session) == status + assert session.environments_entered == entered + + +def _make_ending_only(session: Session) -> str: + """Enter an environment, fail a task in it, and exit it with + ``keep_session_running=True``. Returns the identifier of a second, outer + environment that is still entered.""" + outer = session.enter_environment(environment=_environment("Outer", onEnter=_python(""))) + _settle(session) + inner = session.enter_environment(environment=_environment("Inner", onEnter=_python(""))) + _settle(session) + session.run_task(step_script=_step_script("raise SystemExit(1)")) + _settle(session) + session.exit_environment(identifier=inner, keep_session_running=True) + _settle(session) + status = session.action_status + assert status is not None and status.state == ActionState.SUCCESS + assert session.state == SessionState.READY_ENDING + return outer + + +class TestOnAnEndingOnlySession: + """openjd-rs#524: an ending-only session accepts only ``exit_environment`` + and ``cleanup``. The others raise ``InvalidState``.""" + + _MESSAGE = "Session must be in READY state, current: READY_ENDING" + + def test_run_task(self, session: Session, tmp_path: Path) -> None: + marker = tmp_path / "ran" + _make_ending_only(session) + _assert_refused( + session, + lambda: session.run_task( + step_script=_step_script(f"open({str(marker)!r}, 'w').close()") + ), + self._MESSAGE, + ) + time.sleep(0.2) + assert not marker.exists() + + def test_enter_environment(self, session: Session) -> None: + _make_ending_only(session) + _assert_refused( + session, + lambda: session.enter_environment(environment=_environment(onEnter=_python(""))), + self._MESSAGE, + ) + + def test_run_subprocess(self, session: Session) -> None: + _make_ending_only(session) + _assert_refused( + session, + lambda: session.run_subprocess(command=sys.executable, args=["-c", ""]), + self._MESSAGE, + ) + + def test_exit_environment_is_still_accepted(self, session: Session) -> None: + outer = _make_ending_only(session) + session.exit_environment(identifier=outer) + _settle(session) + status = session.action_status + assert status is not None and status.state == ActionState.SUCCESS + assert session.environments_entered == [] + assert session.state == SessionState.READY_ENDING + + +class TestClearEndingOnly: + def test_returns_the_session_to_ready(self, session: Session) -> None: + """The last action's status and the environments entered stay as + they were, and the session accepts new actions again.""" + outer = _make_ending_only(session) + status = _status(session) + session.clear_ending_only() + assert session.state == SessionState.READY + assert _status(session) == status + assert session.environments_entered == [outer] + session.run_task(step_script=_step_script("")) + _settle(session) + new_status = session.action_status + assert new_status is not None and new_status.state == ActionState.SUCCESS + assert session.state == SessionState.READY + + def test_a_failed_action_makes_it_ending_only_again(self, session: Session) -> None: + _make_ending_only(session) + session.clear_ending_only() + session.run_task(step_script=_step_script("raise SystemExit(1)")) + _settle(session) + assert session.state == SessionState.READY_ENDING + + def test_refused_when_the_session_is_ready(self, session: Session) -> None: + _assert_refused( + session, + session.clear_ending_only, + "Session must be in READY_ENDING state, current: READY", + ) + + def test_refused_while_an_action_runs(self, session: Session) -> None: + session.run_task(step_script=_step_script("import time; time.sleep(30)")) + try: + assert session.state == SessionState.RUNNING + _assert_refused( + session, + session.clear_ending_only, + "Session must be in READY_ENDING state, current: RUNNING", + ) + finally: + session.cancel_action() + _settle(session) + + +class TestWhileAnActionRuns: + """The crate refuses every action while one is running. Before, the + binding raised ``RuntimeError("An action is already running")``; it now + gives the crate's ``InvalidState`` message. ``SessionError`` is a + ``RuntimeError``, so a caller catching ``RuntimeError`` still does.""" + + @pytest.mark.parametrize( + "call, message", + [ + pytest.param( + lambda s: s.run_task(step_script=_step_script("")), + "Session must be in READY state, current: RUNNING", + id="run_task", + ), + pytest.param( + lambda s: s.enter_environment(environment=_environment(onEnter=_python(""))), + "Session must be in READY state, current: RUNNING", + id="enter_environment", + ), + pytest.param( + lambda s: s.exit_environment(identifier="E"), + "Session must be in READY or READY_ENDING state, current: RUNNING", + id="exit_environment", + ), + pytest.param( + lambda s: s.run_subprocess(command=sys.executable, args=["-c", ""]), + "Session must be in READY state, current: RUNNING", + id="run_subprocess", + ), + ], + ) + def test_a_second_action( + self, session: Session, call: Callable[[Session], object], message: str + ) -> None: + session.run_task(step_script=_step_script("import time; time.sleep(30)")) + try: + assert session.state == SessionState.RUNNING + assert issubclass(SessionError, RuntimeError) + _assert_refused(session, lambda: call(session), message) + finally: + session.cancel_action() + _settle(session) + + +class TestOtherRefusals: + def test_exit_an_unknown_environment(self, session: Session) -> None: + _assert_refused( + session, + lambda: session.exit_environment(identifier="nope"), + "Unknown environment identifier: nope", + ) + + def test_exit_out_of_order(self, session: Session) -> None: + first = session.enter_environment(environment=_environment("A", onEnter=_python(""))) + _settle(session) + second = session.enter_environment(environment=_environment("B", onEnter=_python(""))) + _settle(session) + _assert_refused( + session, + lambda: session.exit_environment(identifier=first), + f"Must exit the most recently entered environment first. Expected {second}, got {first}", + ) + + def test_enter_a_duplicate_identifier(self, session: Session) -> None: + session.enter_environment(environment=_environment(onEnter=_python("")), identifier="A") + _settle(session) + _assert_refused( + session, + lambda: session.enter_environment( + environment=_environment(onEnter=_python("")), identifier="A" + ), + "Environment A has already been entered in this Session.", + ) + + def test_enter_a_second_wrap_environment(self, session: Session) -> None: + def wrap_environment(name: str) -> Any: + hooks = ("onWrapEnvEnter", "onWrapTaskRun", "onWrapEnvExit") + return _environment( + name, extensions=["EXPR", "WRAP_ACTIONS"], **{h: _python("") for h in hooks} + ) + + session.enter_environment(environment=wrap_environment("W1")) + _settle(session) + _assert_refused( + session, + lambda: session.enter_environment(environment=wrap_environment("W2")), + "RFC 0008: a session may have at most one Environment defining wrap hooks " + "(onWrapEnvEnter / onWrapTaskRun / onWrapEnvExit). Environment 'W1' already " + "defines wrap hooks; cannot also enter 'W2'.", + ) + + @pytest.mark.parametrize( + "kwargs, message", + [ + pytest.param({"command": " "}, "command must be a non-empty string", id="blank"), + pytest.param( + {"command": sys.executable, "timeout": 0.0}, + "timeout must be positive", + id="zero timeout", + ), + ], + ) + def test_run_subprocess_arguments( + self, session: Session, kwargs: dict[str, Any], message: str + ) -> None: + _assert_refused(session, lambda: session.run_subprocess(**kwargs), message) + + @pytest.mark.parametrize( + "timeout, shown", [(-1.0, "-1"), (float("nan"), "NaN"), (float("inf"), "inf")] + ) + def test_run_subprocess_timeout_with_no_duration( + self, session: Session, timeout: float, shown: str + ) -> None: + """A timeout with no ``Duration`` raises ``ValueError`` before the + crate sees it. Before, the binding's conversion panicked + (``PanicException``, which ``except Exception`` doesn't catch).""" + _assert_refused( + session, + lambda: session.run_subprocess(command=sys.executable, timeout=timeout), + f"timeout must be a finite, non-negative number of seconds, got {shown}", + ValueError, + ) + + +_SLEEP = "import time; time.sleep(30)" + + +class TestCancelAction: + def test_with_no_action_running(self, session: Session) -> None: + """The crate's ``InvalidState``, whether or not an action has run.""" + message = "Session must be in RUNNING state, current: READY" + _assert_refused(session, session.cancel_action, message) + session.run_task(step_script=_step_script("")) + _settle(session) + _assert_refused(session, session.cancel_action, message) + + def test_a_second_cancel_while_canceling(self, session: Session, tmp_path: Path) -> None: + """The crate refuses a cancel while one is in progress. Before, the + binding delivered it through the cancel handle and returned, ignoring + its time limit.""" + marker = tmp_path / "started" + ignore_notify = ( + "import signal, time\n" + "for name in ('SIGTERM', 'SIGBREAK'):\n" + " if hasattr(signal, name):\n" + " signal.signal(getattr(signal, name), signal.SIG_IGN)\n" + f"open({str(marker)!r}, 'w').close()\n" + "time.sleep(30)" + ) + script = deserialize_step( + { + "name": "S", + "script": { + "actions": { + "onRun": { + **_python(ignore_notify), + "cancelation": { + "mode": "NOTIFY_THEN_TERMINATE", + "notifyPeriodInSeconds": 3, + }, + } + } + }, + } + ).script + session.run_task(step_script=script) + deadline = time.monotonic() + 10 + while not marker.exists() and time.monotonic() < deadline: + time.sleep(0.02) + assert marker.exists() + session.cancel_action() + assert session.state == SessionState.CANCELING + _assert_refused( + session, + lambda: session.cancel_action(time_limit=0.0), + "Session must be in RUNNING state, current: CANCELING", + ) + _settle(session) + status = session.action_status + assert status is not None and status.state == ActionState.CANCELED + + def test_a_time_limit_with_no_duration(self, session: Session) -> None: + _assert_refused( + session, + lambda: session.cancel_action(time_limit=-1.0), + "time_limit must be a finite, non-negative number of seconds, got -1", + ValueError, + ) + + @pytest.mark.parametrize( + "start", + [ + pytest.param(lambda s: s.run_task(step_script=_step_script(_SLEEP)), id="run_task"), + pytest.param( + lambda s: s.run_subprocess(command=sys.executable, args=["-c", _SLEEP]), + id="run_subprocess", + ), + pytest.param( + lambda s: s.enter_environment(environment=_environment(onEnter=_python(_SLEEP))), + id="enter_environment", + ), + ], + ) + def test_straight_after_the_call_that_started_the_action( + self, session: Session, start: Callable[[Session], object] + ) -> None: + """The crate reports an action ``Running`` just before it installs + its cancel token. A cancel made as soon as the call returns still + reaches the action; before, it nearly always raised "Cannot cancel: + no action is running" and the action ran to the end.""" + for _ in range(5): + start(session) + began = time.monotonic() + session.cancel_action() + _settle(session) + assert time.monotonic() - began < 8 + status = session.action_status + assert status is not None and status.state == ActionState.CANCELED + assert session.state == SessionState.READY_ENDING + session.clear_ending_only() + + +class TestAcceptedCalls: + """An accepted call returns once its action is running, with + ``action_status`` describing the new action, not the previous one.""" + + def test_run_task_reports_the_new_action(self, session: Session) -> None: + session.run_task(step_script=_step_script("raise SystemExit(1)")) + _settle(session) + session.clear_ending_only() + session.run_task(step_script=_step_script("import time; time.sleep(0.5)")) + status = session.action_status + assert status is not None + assert status.state == ActionState.RUNNING + assert session.state == SessionState.RUNNING + _settle(session) + status = session.action_status + assert status is not None and status.state == ActionState.SUCCESS + + def test_an_environment_with_no_script_has_ended_on_return(self, session: Session) -> None: + """Nothing runs, so the action has ended by the time the call + returns, and the session is back in its slot for the next call.""" + environment = create_environment( + decode_environment_template( + template={ + "specificationVersion": "environment-2023-09", + "environment": {"name": "E", "variables": {"X": "1"}}, + } + ) + ) + identifier = session.enter_environment(environment=environment) + status = session.action_status + assert status is not None and status.state == ActionState.SUCCESS + assert session.state == SessionState.READY + session.exit_environment(identifier=identifier) + assert session.environments_entered == [] + + +_CONCURRENT_CLEANUP = r""" +import logging, os, sys, threading +logging.basicConfig(level=logging.INFO, stream=open(os.devnull, "w")) +from openjd import _openjd_rs +from openjd._openjd_rs import Session +SessionError = getattr(_openjd_rs, "SessionError") +for i in range(50): + s = Session(session_id=f"s{i}", job_parameter_values={}, session_root_directory=sys.argv[1]) + stop = threading.Event() + def hammer(): + while not stop.is_set(): + try: + s.clear_ending_only() + except SessionError: + pass + t = threading.Thread(target=hammer, daemon=True) + t.start() + s.cleanup() + stop.set() + t.join() +print("done") +""" + + +def test_cleanup_with_a_concurrent_call_and_logging_on(tmp_path: Path) -> None: + """``cleanup`` holds the session lock while the crate logs, and each log + record needs the GIL. A concurrent call that waited for the lock while + holding the GIL deadlocked. Every lock of the session is now taken with + the GIL released. This runs in a fresh interpreter so the INFO level is + in force when the binding's logger first looks it up.""" + import subprocess + + result = subprocess.run( + [sys.executable, "-c", _CONCURRENT_CLEANUP, str(tmp_path)], + capture_output=True, + text=True, + timeout=25, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "done" + + +_CANCEL_ON_BANNER = r""" +import logging, sys, time +from openjd import _openjd_rs +from openjd._openjd_rs import Session, SessionState, deserialize_step +SessionError = getattr(_openjd_rs, "SessionError") +results = [] +class CancelOnBanner(logging.Handler): + def emit(self, record): + if "Running Task" in record.getMessage() and not results: + began = time.monotonic() + try: + session.cancel_action() + results.append("canceled") + except SessionError as e: + results.append(str(e)) + results.append(time.monotonic() - began < 0.5) +logging.getLogger().addHandler(CancelOnBanner()) +logging.getLogger().setLevel(logging.INFO) +session = Session(session_id="s", job_parameter_values={}, session_root_directory=sys.argv[1]) +script = deserialize_step( + {"name": "S", "script": {"actions": {"onRun": {"command": sys.executable, "args": ["-c", ""]}}}} +).script +session.run_task(step_script=script) +while session.state == SessionState.RUNNING: + time.sleep(0.02) +session.cleanup() +print(results) +""" + + +def test_cancel_from_the_action_thread_before_it_starts(tmp_path: Path) -> None: + """The crate logs the "Running Task" banner on the action thread between + reporting the action ``Running`` and installing its cancel token. A + logging handler that cancels there can't wait for the token, which can't + appear while it runs, so it raises at once instead of hanging. Runs in a + fresh interpreter, as above.""" + import subprocess + + result = subprocess.run( + [sys.executable, "-c", _CANCEL_ON_BANNER, str(tmp_path)], + capture_output=True, + text=True, + timeout=25, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "['Cannot cancel: the action has not started yet', True]" + + +_CANCEL_WHILE_THE_ACTION_THREAD_WAITS = r""" +import logging, sys, time +from openjd._openjd_rs import ActionState, Session, SessionState, deserialize_step +class SlowOnBanner(logging.Handler): + def emit(self, record): + if "Running Task" in record.getMessage(): + time.sleep(2.5) +logging.getLogger().addHandler(SlowOnBanner()) +logging.getLogger().setLevel(logging.INFO) +session = Session(session_id="s", job_parameter_values={}, session_root_directory=sys.argv[1]) +script = deserialize_step( + {"name": "S", "script": {"actions": {"onRun": { + "command": sys.executable, "args": ["-c", "import time; time.sleep(30)"]}}}} +).script +session.run_task(step_script=script) +session.cancel_action() +while session.state in (SessionState.RUNNING, SessionState.CANCELING): + time.sleep(0.02) +print(session.action_status.state.name) +session.cleanup() +""" + + +def test_cancel_while_the_action_thread_is_slow_to_start(tmp_path: Path) -> None: + """The gap between the crate reporting ``Running`` and installing the + cancel token includes logging the action's banner, which needs the GIL + and runs every Python logging handler. Here a handler takes 2.5 s. A + cancel from another thread waits the gap out rather than giving up; with + a 2 s limit it used to raise "Cannot cancel: the action has not started + yet", and the task ran to the end.""" + import subprocess + + result = subprocess.run( + [sys.executable, "-c", _CANCEL_WHILE_THE_ACTION_THREAD_WAITS, str(tmp_path)], + capture_output=True, + text=True, + timeout=25, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "CANCELED" diff --git a/test/openjd/sessions/test_working_directory.py b/test/openjd/sessions/test_working_directory.py new file mode 100644 index 00000000..8dcafae5 --- /dev/null +++ b/test/openjd/sessions/test_working_directory.py @@ -0,0 +1,24 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""How ``Session`` names its working directory.""" + +from __future__ import annotations + +import os +import re + +from openjd._openjd_rs import Session + + +def test_the_working_directory_is_named_with_8_random_characters() -> None: + """Since openjd-sessions 0.8.2 (openjd-rs#444) the session working directory + is named with exactly 8 characters from ``[a-z0-9_]`` and no part of the + session ID, as the pure-Python reference's ``mkdtemp(prefix="")`` names it. + Before, the name was the session ID followed by 32 hex digits.""" + session = Session(session_id="my-session-id", job_parameter_values={}) + try: + name = os.path.basename(session.working_directory) + assert re.fullmatch(r"[a-z0-9_]{8}", name), name + finally: + session.cleanup()