Skip to content

Commit 59b68e8

Browse files
committed
revert previous change + create different ivy cache per test to fix concurrent writes in CI
1 parent 7a659af commit 59b68e8

3 files changed

Lines changed: 35 additions & 10 deletions

File tree

‎sagemaker-mlops/src/sagemaker/mlops/feature_store/feature_processor/_spark_factory.py‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@
1313
"""Contains factory classes for instantiating Spark objects."""
1414
from __future__ import absolute_import
1515

16-
import os
17-
import tempfile
1816
from functools import lru_cache
1917
from typing import List, Tuple, Dict
2018

@@ -136,10 +134,6 @@ def _get_spark_configs(self, is_training_job) -> List[Tuple[str, str]]:
136134
"spark.jars.packages",
137135
",".join(fp_spark_packages),
138136
),
139-
(
140-
"spark.jars.ivy",
141-
os.path.join(tempfile.mkdtemp(), ".ivy2"),
142-
),
143137
)
144138
)
145139

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
"""Conftest for feature processor integration tests."""
14+
import os
15+
import tempfile
16+
17+
import pytest
18+
19+
from sagemaker.mlops.feature_store.feature_processor._spark_factory import SparkSessionFactory
20+
21+
22+
@pytest.fixture(autouse=True, scope="session")
23+
def isolate_ivy_cache():
24+
"""Give each pytest-xdist worker its own Ivy cache to prevent concurrent cache corruption."""
25+
ivy_dir = os.path.join(tempfile.mkdtemp(), ".ivy2")
26+
original = SparkSessionFactory._get_spark_configs
27+
28+
def _patched_get_spark_configs(self, is_training_job):
29+
configs = original(self, is_training_job)
30+
configs.append(("spark.jars.ivy", ivy_dir))
31+
return configs
32+
33+
SparkSessionFactory._get_spark_configs = _patched_get_spark_configs
34+
yield
35+
SparkSessionFactory._get_spark_configs = original

‎sagemaker-mlops/tests/unit/sagemaker/mlops/feature_store/feature_processor/test_spark_session_factory.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,6 @@ def test_spark_session_factory_configuration():
7979
]
8080
) in spark_configs.get("spark.jars.packages")
8181

82-
assert spark_configs.get("spark.jars.ivy") is not None
83-
assert ".ivy2" in spark_configs.get("spark.jars.ivy")
84-
8582

8683
def test_spark_session_factory_configuration_on_training_job():
8784
env_helper = Mock()
@@ -93,7 +90,6 @@ def test_spark_session_factory_configuration_on_training_job():
9390

9491
assert all(tup[0] != "spark.jars" for tup in spark_config)
9592
assert all(tup[0] != "spark.jars.packages" for tup in spark_config)
96-
assert all(tup[0] != "spark.jars.ivy" for tup in spark_config)
9793

9894

9995
@patch("pyspark.context.SparkContext.getOrCreate")

0 commit comments

Comments
 (0)