Support azure-datalake-store 1.x in AzureDataLakeHook (#73819)

* Support azure-datalake-store 1.x in AzureDataLakeHook

* Require azure-datalake-store>=1.0.1 and drop 0.0.x support

Author: haseebmalik18

providers/microsoft/azure/README.rst

@@ -55,13 +55,13 @@ PIP package                                 Version required
 ==========================================  ===================
 ``apache-airflow``                          ``>=2.11.0``
 ``apache-airflow-providers-common-compat``  ``>=1.13.0``
-``adlfs``                                   ``>=2023.10.0``
+``adlfs``                                   ``>=2026.5.0``
 ``aiohttp``                                 ``>=3.14.0``
 ``azure-batch``                             ``>=15.0.0``
 ``azure-ai-projects``                       ``>=2.2.0``
 ``azure-cosmos``                            ``>=4.15.0``
 ``azure-mgmt-cosmosdb``                     ``>=9.9.0``
-``azure-datalake-store``                    ``>=0.0.45,<1``
+``azure-datalake-store``                    ``>=1.0.1``
 ``azure-identity``                          ``>=1.25.3``
 ``azure-keyvault-secrets``                  ``>=4.10.0``
 ``azure-mgmt-datalake-store``               ``>=0.5.0``

providers/microsoft/azure/docs/index.rst

@@ -111,13 +111,13 @@ PIP package                                 Version required
 ==========================================  ===================
 ``apache-airflow``                          ``>=2.11.0``
 ``apache-airflow-providers-common-compat``  ``>=1.13.0``
-``adlfs``                                   ``>=2023.10.0``
+``adlfs``                                   ``>=2026.5.0``
 ``aiohttp``                                 ``>=3.14.0``
 ``azure-batch``                             ``>=15.0.0``
 ``azure-ai-projects``                       ``>=2.2.0``
 ``azure-cosmos``                            ``>=4.15.0``
 ``azure-mgmt-cosmosdb``                     ``>=9.9.0``
-``azure-datalake-store``                    ``>=0.0.45,<1``
+``azure-datalake-store``                    ``>=1.0.1``
 ``azure-identity``                          ``>=1.25.3``
 ``azure-keyvault-secrets``                  ``>=4.10.0``
 ``azure-mgmt-datalake-store``               ``>=0.5.0``

providers/microsoft/azure/pyproject.toml

@@ -61,16 +61,14 @@ requires-python = ">=3.10"
 dependencies = [
     "apache-airflow>=2.11.0",
     "apache-airflow-providers-common-compat>=1.13.0",
-    "adlfs>=2023.10.0",
+    "adlfs>=2026.5.0",
     "aiohttp>=3.14.0",
     "azure-batch>=15.0.0",
     # 2.2.0 adds force deletion support for Hosted agents and agent versions.
     "azure-ai-projects>=2.2.0",
     "azure-cosmos>=4.15.0",
     "azure-mgmt-cosmosdb>=9.9.0",
-    # 1.x removed lib.auth used by AzureDataLakeHook; remove the cap once the hook supports 1.x,
-    # tracked at https://github.com/apache/airflow/issues/73795
-    "azure-datalake-store>=0.0.45,<1",
+    "azure-datalake-store>=1.0.1",
     "azure-identity>=1.25.3",
     "azure-keyvault-secrets>=4.10.0",
     "azure-mgmt-datalake-store>=0.5.0",

providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_lake.py

@@ -21,7 +21,7 @@
 from typing import Any, cast
 
 from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError
-from azure.datalake.store import core, lib, multithread
+from azure.datalake.store import core, multithread
 from azure.identity import ClientSecretCredential, DefaultAzureCredential
 from azure.storage.filedatalake import (
     DataLakeDirectoryClient,
@@ -34,13 +34,12 @@
 
 from airflow.providers.common.compat.sdk import AirflowException, BaseHook
 from airflow.providers.microsoft.azure.utils import (
-    AzureIdentityCredentialAdapter,
     add_managed_identity_connection_widgets,
     get_field,
     get_sync_default_azure_credential,
 )
 
-Credentials = ClientSecretCredential | AzureIdentityCredentialAdapter | DefaultAzureCredential
+Credentials = ClientSecretCredential | DefaultAzureCredential
 
 
 class AzureDataLakeHook(BaseHook):
@@ -122,11 +121,15 @@ def get_conn(self) -> core.AzureDLFileSystem:
             credential: Credentials
             tenant = self._get_field(extras, "tenant")
             if tenant:
-                credential = lib.auth(tenant_id=tenant, client_secret=conn.password, client_id=conn.login)
+                credential = ClientSecretCredential(
+                    tenant_id=tenant,
+                    client_id=cast("str", conn.login),
+                    client_secret=cast("str", conn.password),
+                )
             else:
                 managed_identity_client_id = self._get_field(extras, "managed_identity_client_id")
                 workload_identity_tenant_id = self._get_field(extras, "workload_identity_tenant_id")
-                credential = AzureIdentityCredentialAdapter(
+                credential = get_sync_default_azure_credential(
                     managed_identity_client_id=managed_identity_client_id,
                     workload_identity_tenant_id=workload_identity_tenant_id,
                 )

providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_lake.py

@@ -73,44 +73,36 @@ def setup_connections(self, create_mock_connections):
             )
         )
 
-    @mock.patch(f"{MODULE}.lib", autospec=True)
-    def test_conn(self, mock_lib):
-        from azure.datalake.store import core
-
+    @mock.patch(f"{MODULE}.core.AzureDLFileSystem", autospec=True)
+    @mock.patch(f"{MODULE}.ClientSecretCredential", autospec=True)
+    def test_conn_with_tenant_uses_client_secret_credential(self, mock_credential, mock_filesystem):
         from airflow.providers.microsoft.azure.hooks.data_lake import AzureDataLakeHook
 
         hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key")
-        assert hook._conn is None
-        assert hook.conn_id == "adl_test_key"
-        assert isinstance(hook.get_conn(), core.AzureDLFileSystem)
-        assert mock_lib.auth.called
 
-    @pytest.mark.usefixtures("connection_without_tenant")
-    @mock.patch(f"{MODULE}.lib")
-    @mock.patch(f"{MODULE}.AzureIdentityCredentialAdapter")
-    def test_fallback_to_azure_identity_credential_adppter_when_tenant_is_not_provided(
-        self,
-        mock_azure_identity_credential_adapter,
-        mock_datalake_store_lib,
-    ):
-        from azure.datalake.store import core
+        assert hook.get_conn() is mock_filesystem.return_value
+        mock_credential.assert_called_once_with(
+            tenant_id="tenant", client_id="client_id", client_secret="client secret"
+        )
+        mock_filesystem.assert_called_once_with(mock_credential.return_value, store_name="accountname")
 
+    @pytest.mark.usefixtures("connection_without_tenant")
+    @mock.patch(f"{MODULE}.core.AzureDLFileSystem", autospec=True)
+    @mock.patch(f"{MODULE}.get_sync_default_azure_credential", autospec=True)
+    def test_conn_without_tenant_uses_default_azure_credential(self, mock_credential, mock_filesystem):
         from airflow.providers.microsoft.azure.hooks.data_lake import AzureDataLakeHook
 
         hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key_without_tenant")
-        assert hook._conn is None
-        assert hook.conn_id == "adl_test_key_without_tenant"
-        assert isinstance(hook.get_conn(), core.AzureDLFileSystem)
-        mock_azure_identity_credential_adapter.assert_called()
-        args = mock_azure_identity_credential_adapter.call_args
-        assert args.kwargs["managed_identity_client_id"] is None
-        assert args.kwargs["workload_identity_tenant_id"] is None
-        mock_datalake_store_lib.auth.assert_not_called()
+
+        assert hook.get_conn() is mock_filesystem.return_value
+        mock_credential.assert_called_once_with(
+            managed_identity_client_id=None, workload_identity_tenant_id=None
+        )
+        mock_filesystem.assert_called_once_with(mock_credential.return_value, store_name="accountname")
 
     @pytest.mark.usefixtures("connection")
     @mock.patch(f"{MODULE}.core.AzureDLFileSystem", autospec=True)
-    @mock.patch(f"{MODULE}.lib", autospec=True)
-    def test_check_for_blob(self, mock_lib, mock_filesystem):
+    def test_check_for_blob(self, mock_filesystem):
         from airflow.providers.microsoft.azure.hooks.data_lake import AzureDataLakeHook
 
         mocked_glob = mock_filesystem.return_value.glob
@@ -120,8 +112,7 @@ def test_check_for_blob(self, mock_lib, mock_filesystem):
 
     @pytest.mark.usefixtures("connection")
     @mock.patch(f"{MODULE}.multithread.ADLUploader", autospec=True)
-    @mock.patch(f"{MODULE}.lib", autospec=True)
-    def test_upload_file(self, mock_lib, mock_uploader):
+    def test_upload_file(self, mock_uploader):
         from airflow.providers.microsoft.azure.hooks.data_lake import AzureDataLakeHook
 
         hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key")
@@ -145,8 +136,7 @@ def test_upload_file(self, mock_lib, mock_uploader):
 
     @pytest.mark.usefixtures("connection")
     @mock.patch(f"{MODULE}.multithread.ADLDownloader", autospec=True)
-    @mock.patch(f"{MODULE}.lib", autospec=True)
-    def test_download_file(self, mock_lib, mock_downloader):
+    def test_download_file(self, mock_downloader):
         from airflow.providers.microsoft.azure.hooks.data_lake import AzureDataLakeHook
 
         hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key")
@@ -170,8 +160,7 @@ def test_download_file(self, mock_lib, mock_downloader):
 
     @pytest.mark.usefixtures("connection")
     @mock.patch(f"{MODULE}.core.AzureDLFileSystem", autospec=True)
-    @mock.patch(f"{MODULE}.lib", autospec=True)
-    def test_list_glob(self, mock_lib, mock_fs):
+    def test_list_glob(self, mock_fs):
         from airflow.providers.microsoft.azure.hooks.data_lake import AzureDataLakeHook
 
         hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key")
@@ -180,8 +169,7 @@ def test_list_glob(self, mock_lib, mock_fs):
 
     @pytest.mark.usefixtures("connection")
     @mock.patch(f"{MODULE}.core.AzureDLFileSystem", autospec=True)
-    @mock.patch(f"{MODULE}.lib", autospec=True)
-    def test_list_walk(self, mock_lib, mock_fs):
+    def test_list_walk(self, mock_fs):
         from airflow.providers.microsoft.azure.hooks.data_lake import AzureDataLakeHook
 
         hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key")
@@ -190,8 +178,7 @@ def test_list_walk(self, mock_lib, mock_fs):
 
     @pytest.mark.usefixtures("connection")
     @mock.patch(f"{MODULE}.core.AzureDLFileSystem", autospec=True)
-    @mock.patch(f"{MODULE}.lib", autospec=True)
-    def test_remove(self, mock_lib, mock_fs):
+    def test_remove(self, mock_fs):
         from airflow.providers.microsoft.azure.hooks.data_lake import AzureDataLakeHook
 
         hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key")

pyproject.toml

@@ -1674,7 +1674,6 @@ airbyte-api = "1.x breaks provider tests (#69081)"
 mysql-connector-python = "Releases without wheels for >=3.12 (#60889, #66026)"
 anthropic = "Provider SDK migration is version-sensitive (#72072, #72094)"
 azure-ai-projects = "2.5+ needs openai>=3, broke main (#73621, #73627)"
-azure-datalake-store = "1.x removed lib.auth used by AzureDataLakeHook (#73795)"
 
 [tool.uv.pip]
 # Synchroonize with scripts/ci/prek/upgrade_important_versions.py