|
12 | 12 | # language governing permissions and limitations under the License. |
13 | 13 | """Unit tests for BedrockModelBuilder.""" |
14 | 14 |
|
| 15 | +import io |
15 | 16 | import json |
| 17 | +import tarfile |
| 18 | + |
16 | 19 | import pytest |
17 | 20 | from unittest.mock import Mock, patch |
18 | 21 | from botocore.exceptions import ClientError |
19 | 22 |
|
20 | 23 | from sagemaker.core.resources import TrainingJob |
21 | 24 | from sagemaker.core.shapes import ModelArtifacts, OutputDataConfig |
22 | 25 | from sagemaker.core.utils.utils import Unassigned |
23 | | -from sagemaker.serve.bedrock_model_builder import BedrockModelBuilder, _is_nova_model |
| 26 | +from sagemaker.serve.bedrock_model_builder import ( |
| 27 | + BedrockModelBuilder, |
| 28 | + _is_nova_model, |
| 29 | + _tar_member_relative_key, |
| 30 | +) |
24 | 31 |
|
25 | 32 | MODULE = "sagemaker.serve.bedrock_model_builder" |
26 | 33 |
|
@@ -1625,3 +1632,126 @@ def test_oss_reuse_preserves_user_tags(self): |
1625 | 1632 | assert user_tag in tags |
1626 | 1633 | source_tag = {"key": "sagemaker.amazonaws.com/model-source", "value": "s3://bucket/path/"} |
1627 | 1634 | assert source_tag in tags |
| 1635 | + |
| 1636 | + |
| 1637 | +# ── _extract_tar_gz_to_s3 ─────────────────────────────────────────────────── |
| 1638 | + |
| 1639 | + |
| 1640 | +def _tar_gz_bytes(members): |
| 1641 | + """Build an in-memory tar.gz from (name, content) pairs. |
| 1642 | +
|
| 1643 | + A content of None adds a directory entry instead of a file. |
| 1644 | + """ |
| 1645 | + buf = io.BytesIO() |
| 1646 | + with tarfile.open(fileobj=buf, mode="w:gz") as tar: |
| 1647 | + for name, content in members: |
| 1648 | + info = tarfile.TarInfo(name=name) |
| 1649 | + if content is None: |
| 1650 | + info.type = tarfile.DIRTYPE |
| 1651 | + tar.addfile(info) |
| 1652 | + else: |
| 1653 | + info.size = len(content) |
| 1654 | + tar.addfile(info, io.BytesIO(content)) |
| 1655 | + return buf.getvalue() |
| 1656 | + |
| 1657 | + |
| 1658 | +def _extract_with_members(members): |
| 1659 | + """Run _extract_tar_gz_to_s3 against a fake S3 client and return put_object keys.""" |
| 1660 | + s3_client = Mock() |
| 1661 | + s3_client.list_objects_v2.return_value = {"KeyCount": 0} |
| 1662 | + s3_client.get_object.return_value = {"Body": io.BytesIO(_tar_gz_bytes(members))} |
| 1663 | + b = object.__new__(BedrockModelBuilder) |
| 1664 | + b.boto_session = Mock() |
| 1665 | + b.boto_session.client.return_value = s3_client |
| 1666 | + |
| 1667 | + uri = b._extract_tar_gz_to_s3("s3://bucket/jobs/run/output/model.tar.gz") |
| 1668 | + |
| 1669 | + assert uri == "s3://bucket/jobs/run/output/extracted/" |
| 1670 | + return {c.kwargs["Key"]: c.kwargs["Body"] for c in s3_client.put_object.call_args_list} |
| 1671 | + |
| 1672 | + |
| 1673 | +class TestTarMemberRelativeKey: |
| 1674 | + @pytest.mark.parametrize( |
| 1675 | + "name,expected", |
| 1676 | + [ |
| 1677 | + ("config.json", "config.json"), |
| 1678 | + ("checkpoints/hf/config.json", "checkpoints/hf/config.json"), |
| 1679 | + ("./config.json", "config.json"), |
| 1680 | + ("./checkpoints//hf/./config.json", "checkpoints/hf/config.json"), |
| 1681 | + ("model..safetensors", "model..safetensors"), |
| 1682 | + ("a\\b.json", "a\\b.json"), |
| 1683 | + ], |
| 1684 | + ) |
| 1685 | + def test_normalizes_relative_names(self, name, expected): |
| 1686 | + assert _tar_member_relative_key(name) == expected |
| 1687 | + |
| 1688 | + @pytest.mark.parametrize( |
| 1689 | + "name", |
| 1690 | + [ |
| 1691 | + "", |
| 1692 | + ".", |
| 1693 | + "./", |
| 1694 | + "/config.json", |
| 1695 | + "\\config.json", |
| 1696 | + "C:/config.json", |
| 1697 | + "../config.json", |
| 1698 | + "a/../../config.json", |
| 1699 | + "a\\..\\config.json", |
| 1700 | + ], |
| 1701 | + ) |
| 1702 | + def test_rejects_unsupported_names(self, name): |
| 1703 | + assert _tar_member_relative_key(name) is None |
| 1704 | + |
| 1705 | + |
| 1706 | +class TestExtractTarGzToS3: |
| 1707 | + def test_uploads_members_under_extract_prefix(self): |
| 1708 | + keys = _extract_with_members( |
| 1709 | + [("config.json", b"{}"), ("./checkpoints/hf/model.safetensors", b"weights")] |
| 1710 | + ) |
| 1711 | + |
| 1712 | + assert keys == { |
| 1713 | + "jobs/run/output/extracted/config.json": b"{}", |
| 1714 | + "jobs/run/output/extracted/checkpoints/hf/model.safetensors": b"weights", |
| 1715 | + } |
| 1716 | + |
| 1717 | + def test_skips_directory_members(self): |
| 1718 | + keys = _extract_with_members( |
| 1719 | + [("./", None), ("./checkpoints", None), ("./checkpoints/config.json", b"{}")] |
| 1720 | + ) |
| 1721 | + |
| 1722 | + assert keys == {"jobs/run/output/extracted/checkpoints/config.json": b"{}"} |
| 1723 | + |
| 1724 | + def test_names_normalizing_to_same_key_keep_last_member(self): |
| 1725 | + keys = _extract_with_members([("./config.json", b"first"), ("config.json", b"second")]) |
| 1726 | + |
| 1727 | + assert keys == {"jobs/run/output/extracted/config.json": b"second"} |
| 1728 | + |
| 1729 | + def test_skips_members_with_unsupported_paths(self, caplog): |
| 1730 | + with caplog.at_level("WARNING"): |
| 1731 | + keys = _extract_with_members( |
| 1732 | + [("config.json", b"{}"), ("../other/file.txt", b"x"), ("/abs/file.txt", b"y")] |
| 1733 | + ) |
| 1734 | + |
| 1735 | + assert keys == {"jobs/run/output/extracted/config.json": b"{}"} |
| 1736 | + assert "Skipped 2 archive member(s)" in caplog.text |
| 1737 | + |
| 1738 | + def test_raises_with_skipped_count_when_all_members_skipped(self): |
| 1739 | + with pytest.raises(RuntimeError, match="2 archive member\\(s\\) were skipped"): |
| 1740 | + _extract_with_members([("../file.txt", b"x"), ("/abs/file.txt", b"y")]) |
| 1741 | + |
| 1742 | + def test_raises_when_archive_has_no_files(self): |
| 1743 | + with pytest.raises(RuntimeError, match="No files found"): |
| 1744 | + _extract_with_members([("./", None)]) |
| 1745 | + |
| 1746 | + def test_skips_extraction_when_prefix_exists(self): |
| 1747 | + s3_client = Mock() |
| 1748 | + s3_client.list_objects_v2.return_value = {"KeyCount": 1} |
| 1749 | + b = object.__new__(BedrockModelBuilder) |
| 1750 | + b.boto_session = Mock() |
| 1751 | + b.boto_session.client.return_value = s3_client |
| 1752 | + |
| 1753 | + uri = b._extract_tar_gz_to_s3("s3://bucket/jobs/model.tar.gz") |
| 1754 | + |
| 1755 | + assert uri == "s3://bucket/jobs/extracted/" |
| 1756 | + s3_client.get_object.assert_not_called() |
| 1757 | + s3_client.put_object.assert_not_called() |
0 commit comments