test: serve S3 tests from a local moto server

pyproject.toml

@@ -22,6 +22,7 @@ test = [
   "aiohttp",
   "deflate",
   "isal",
+  "moto[s3,server]",
   "paramiko",
   "pytest-rerunfailures",
   "rangehttpserver",

tests/conftest.py

@@ -87,6 +87,57 @@ def tests_directory() -> str:
     return os.path.dirname(os.path.realpath(__file__))
 
 
+@pytest.fixture(scope="session")
+def s3_server():
+    """
+    A local (in-process) S3 server, so that the S3 tests don't depend on the
+    network or on somebody else paying for the data transfer.
+
+    Yields ``(bucket_url, storage_options)``, where the bucket already contains
+    two copies of ``uproot-HZZ.root``, named ``uproot-HZZ-1.root`` and
+    ``uproot-HZZ-2.root``.
+    """
+    pytest.importorskip("s3fs")
+    moto_server = pytest.importorskip("moto.server")
+    import s3fs
+
+    if not hasattr(moto_server.ThreadedMotoServer, "get_host_and_port"):
+        # On free-threaded Windows, moto[server] -> docker -> pywin32 has no
+        # wheels, so the resolver falls back to a years-old moto
+        pytest.skip("moto is too old to report which port it is listening on")
+
+    bucket = "uproot-test"
+    server = moto_server.ThreadedMotoServer(ip_address="127.0.0.1", port=0)
+    server.start()
+
+    try:
+        _, port = server.get_host_and_port()
+        storage_options = {
+            # moto does not check these, but botocore insists on having them
+            "key": "testing",
+            "secret": "testing",
+            "client_kwargs": {"endpoint_url": f"http://127.0.0.1:{port}"},
+        }
+
+        with pytest.MonkeyPatch.context() as monkeypatch:
+            # botocore raises NoRegionError if it can't find a region anywhere
+            monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
+
+            # don't reuse (or poison) filesystem instances from other tests
+            s3fs.S3FileSystem.clear_instance_cache()
+            fs = s3fs.S3FileSystem(**storage_options)
+            fs.mkdir(bucket)
+            local_path = skhep_testdata.data_path("uproot-HZZ.root")
+            for name in ("uproot-HZZ-1.root", "uproot-HZZ-2.root"):
+                fs.put(local_path, f"{bucket}/{name}")
+
+            yield f"s3://{bucket}", storage_options
+
+            s3fs.S3FileSystem.clear_instance_cache()
+    finally:
+        server.stop()
+
+
 @pytest.fixture(scope="module")
 def xrootd_server(tmpdir_factory):
     pytest.importorskip("XRootD")

tests/test_0692_fsspec_reading.py

@@ -85,24 +85,23 @@ def test_open_fsspec_local():
         assert len(data) == 40
 
 
-@pytest.mark.network
 @pytest.mark.parametrize(
     "handler",
     [
         uproot.source.fsspec.FSSpecSource,
         None,
     ],
 )
-def test_open_fsspec_s3(handler):
-    pytest.importorskip("s3fs")
+def test_open_fsspec_s3(handler, s3_server):
+    bucket_url, storage_options = s3_server
 
     with uproot.open(
-        "s3://pivarski-princeton/pythia_ppZee_run17emb.picoDst.root:PicoDst",
-        anon=True,
+        f"{bucket_url}/uproot-HZZ-1.root:events",
         handler=handler,
+        **storage_options,
     ) as f:
-        data = f["Event/Event.mEventId"].array(library="np")
-        assert len(data) == 8004
+        data = f["Muon_Px"].array(library="np")
+        assert len(data) == 2421
 
 
 @pytest.mark.parametrize(
@@ -455,21 +454,21 @@ def test_fsspec_globbing_xrootd_no_files(handler):
         None,
     ],
 )
-def test_fsspec_globbing_s3(handler):
-    pytest.importorskip("s3fs")
+def test_fsspec_globbing_s3(handler, s3_server):
+    bucket_url, storage_options = s3_server
 
     iterator = uproot.iterate(
-        {"s3://pivarski-princeton/pythia_ppZee_run17emb.*.root": "PicoDst"},
-        ["Event/Event.mEventId"],
-        anon=True,
+        {f"{bucket_url}/uproot-HZZ-*.root": "events"},
+        ["Muon_Px"],
         handler=handler,
+        **storage_options,
     )
 
-    # if more files are added that match the glob, this test needs to be updated
+    # the bucket is populated by the s3_server fixture with exactly two matches
     arrays = [array for array in iterator]
-    assert len(arrays) == 1
+    assert len(arrays) == 2
     for array in arrays:
-        assert len(array) == 8004
+        assert len(array) == 2421
 
 
 @pytest.mark.parametrize(

tests/test_0916_read_from_s3.py

@@ -1,29 +1,26 @@
 # BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE
 
-import socket
-
 import pytest
 
 import uproot
 
 pytest.importorskip("s3fs")
 
 
-@pytest.mark.network
-def test_s3_fail():
-    with pytest.raises((FileNotFoundError, TimeoutError, socket.timeout)):
-        # Sometimes this raises a timeout error that doesn't go away for a long time, we might as well skip it.
+def test_s3_fail(s3_server):
+    bucket_url, storage_options = s3_server
+    with pytest.raises(FileNotFoundError):
         with uproot.source.fsspec.FSSpecSource(
-            "s3://pivarski-princeton/does-not-exist", anon=True
+            f"{bucket_url}/does-not-exist", **storage_options
         ) as source:
             uproot._util.tobytes(source.chunk(0, 100).raw_data)
 
 
-@pytest.mark.network
-def test_read_s3():
+def test_read_s3(s3_server):
+    bucket_url, storage_options = s3_server
     with uproot.open(
-        "s3://pivarski-princeton/pythia_ppZee_run17emb.picoDst.root:PicoDst",
-        anon=True,
+        f"{bucket_url}/uproot-HZZ-1.root:events",
+        **storage_options,
     ) as f:
-        data = f["Event/Event.mEventId"].array(library="np")
-        assert len(data) == 8004
+        data = f["Muon_Px"].array(library="np")
+        assert len(data) == 2421