Merge pull request #36 from comet-ml/CRThaze/s3proxy-azureblob-managed-identity

feat(azureblob): support Azure Managed/Workload Identity

Author: CRThaze

.github/workflows/lint-render.yaml

@@ -30,6 +30,7 @@ jobs:
           - test-values/s3.yaml
           - test-values/azureblob.yaml
           - test-values/azureblob-existing-secret.yaml
+          - test-values/azureblob-managed-identity.yaml
           - test-values/gcs.yaml
           - test-values/b2.yaml
           - test-values/openstack-swift.yaml

charts/s3proxy/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.2
+version: 0.4.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/s3proxy/README.md.gotmpl

@@ -26,7 +26,7 @@ The chart tracks S3Proxy (`andrewgaul/s3proxy`) through `appVersion` (currently
 S3Proxy `3.0.0` deprecated the Apache jclouds storage backends (`s3`, `aws-s3`, `azureblob`, `filesystem`, `transient`) in favor of SDK / NIO2 providers. Upstream has announced that `3.3.0` is the last release to bundle jclouds and that future releases "will lack its Atmos and B2 storage backends" (no jclouds-free release has shipped yet; `3.3.0` remains the latest). The jclouds providers still work on 3.x, but are deprecated:
 
 - `filesystem` / `transient`: already default to the non-deprecated `*-nio2` variants (`nio2: true`).
-- `azureblob`: **defaults to `provider: azureblob-sdk`** (the Azure SDK provider, which signs correctly against custom endpoints such as Azurite). The legacy jclouds `azureblob` provider is deprecated and mis-signs against custom endpoints; set `provider: azureblob` only if you specifically need it. On real Azure, `azureblob-sdk` may require `config.backends.azureblob.regions` for bucket creation.
+- `azureblob`: **defaults to `provider: azureblob-sdk`** (the Azure SDK provider, which signs correctly against custom endpoints such as Azurite). The legacy jclouds `azureblob` provider is deprecated and mis-signs against custom endpoints; set `provider: azureblob` only if you specifically need it. On real Azure, `azureblob-sdk` may require `config.backends.azureblob.regions` for bucket creation. For AKS Managed/Workload Identity (no static key), set `config.backends.azureblob.managedIdentity: true` — see "Example 3b" below.
 - `s3`, `googleCloudStorage`, `openstackSwift`: SDK providers exist upstream (`aws-s3-sdk`, `google-cloud-storage-sdk`, `openstack-swift-sdk`). `rackspaceCloudfiles` is OpenStack-Swift-compatible and may be served by `openstack-swift-sdk`. Migrating the chart defaults to the SDK providers is tracked separately.
 - `b2` (and Atmos, if ever added) are jclouds-only with **no SDK successor**. These are the backends upstream has said future releases will drop.
 
@@ -163,6 +163,40 @@ Install:
 helm install s3proxy-azure ./{{ template "chart.name" . }} -f values-azure.yaml
 ```
 
+### Example 3b: Azure Blob with Managed / Workload Identity (no static key)
+
+On AKS with Workload Identity, set `managedIdentity: true` instead of an account
+key or SAS token. The chart then renders both `jclouds.identity` and
+`jclouds.credential` empty, which makes the `azureblob-sdk` provider fall back to
+Azure's `DefaultAzureCredential` — it reads the federated token AKS injects
+(`AZURE_CLIENT_ID` / `AZURE_TENANT_ID` / `AZURE_FEDERATED_TOKEN_FILE`). The storage
+account name goes in the endpoint, never in the identity.
+
+```yaml
+# values-azure-mi.yaml
+config:
+  backends:
+    filesystem:
+      enabled: false
+    azureblob:
+      enabled: true
+      provider: "azureblob-sdk"   # required: the legacy jclouds provider can't do token auth
+      managedIdentity: true
+      account: "mystorageaccount" # used for the endpoint; NOT emitted as jclouds.identity
+      # endpoint: "https://mystorageaccount.blob.core.windows.net"  # or set explicitly
+
+# Bind the pod to the AKS user-assigned identity:
+serviceAccount:
+  create: true
+  annotations:
+    azure.workload.identity/client-id: "<managed-identity-client-id>"
+podLabels:
+  azure.workload.identity/use: "true"
+
+persistence:
+  enabled: false
+```
+
 ### Example 4: Google Cloud Storage Backend
 
 ```yaml

charts/s3proxy/templates/configmap.yaml

@@ -87,6 +87,19 @@ s3proxy.large-object-mocking=true
     {{- end }}
   {{- end }}
 {{- end }}
+{{- /* Fail fast on an Azure managed-identity config that can't authenticate, since
+       the failure otherwise surfaces only at runtime as an opaque Azure SDK error. */}}
+{{- if and .Values.config.backends.azureblob.enabled .Values.config.backends.azureblob.managedIdentity }}
+  {{- if ne .Values.config.backends.azureblob.provider "azureblob-sdk" }}
+    {{- fail "config.backends.azureblob.managedIdentity requires provider=azureblob-sdk; the legacy jclouds azureblob provider cannot use Azure token auth." }}
+  {{- end }}
+  {{- if or .Values.config.backends.azureblob.key.value .Values.config.backends.azureblob.key.existingSecret .Values.config.backends.azureblob.sasToken.value }}
+    {{- fail "config.backends.azureblob.managedIdentity conflicts with a static credential; a non-empty key.*/sasToken.* disables DefaultAzureCredential. Unset them." }}
+  {{- end }}
+  {{- if and (not .Values.config.backends.azureblob.account) (not .Values.config.backends.azureblob.endpoint) }}
+    {{- fail "config.backends.azureblob.managedIdentity needs the storage endpoint; set config.backends.azureblob.account (default endpoint) or endpoint." }}
+  {{- end }}
+{{- end }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -174,15 +187,23 @@ data:
     # (otherwise "no jclouds.regions configured" -> InvalidLocationConstraint).
     jclouds.regions={{ .Values.config.backends.azureblob.regions }}
   {{- end }}
-  {{- if .Values.config.backends.azureblob.account }}
+  {{- if .Values.config.backends.azureblob.managedIdentity }}
+    # Managed/Workload Identity: azureblob-sdk falls back to Azure's
+    # DefaultAzureCredential (reads the AKS-injected AZURE_CLIENT_ID/TENANT_ID/
+    # FEDERATED_TOKEN_FILE) only when BOTH identity and credential are empty. The
+    # account name lives in jclouds.endpoint above, never in the identity.
+    jclouds.identity=
+    jclouds.credential=
+  {{- else }}
+    {{- if .Values.config.backends.azureblob.account }}
     jclouds.identity={{ .Values.config.backends.azureblob.account }}
-  {{- end }}
-  {{- /* With a key (key.value or key.existingSecret) the initContainer merges
-         jclouds.credential in from a Secret. S3Proxy requires the property in every
-         backend file, so emit it empty when neither is set (SAS-token or Azure
-         Workload Identity auth). */}}
-  {{- if and (not .Values.config.backends.azureblob.key.value) (not .Values.config.backends.azureblob.key.existingSecret) }}
+    {{- end }}
+    {{- /* With a key (key.value or key.existingSecret) the initContainer merges
+           jclouds.credential in from a Secret. S3Proxy requires the property in every
+           backend file, so emit it empty when neither is set (SAS-token auth). */}}
+    {{- if and (not .Values.config.backends.azureblob.key.value) (not .Values.config.backends.azureblob.key.existingSecret) }}
     jclouds.credential=
+    {{- end }}
   {{- end }}
   {{- range $index, $bucket := .Values.config.backends.azureblob.bucketLocators }}
     s3proxy.bucket-locator.{{ add $index 1 }}={{ $bucket }}

charts/s3proxy/values.yaml

@@ -234,6 +234,8 @@ config:
       enabled: false
       # -- Provider type. Defaults to `azureblob-sdk` (Azure SDK): it signs correctly against custom endpoints (Azurite, Azure Gov/China, private endpoints) and is the non-deprecated provider on S3Proxy 3.x. The legacy jclouds `azureblob` provider is deprecated upstream and mis-signs against custom endpoints; on real Azure `azureblob-sdk` may require `regions` to be set for bucket creation.
       provider: "azureblob-sdk"
+      # -- Use Azure Managed/Workload Identity instead of a static account key or SAS token. Requires `provider: azureblob-sdk`; mutually exclusive with `key.*` and `sasToken.*`. When true the chart renders both `jclouds.identity` and `jclouds.credential` empty, which makes azureblob-sdk fall back to Azure's `DefaultAzureCredential` (consuming the AKS workload-identity env `AZURE_CLIENT_ID`/`AZURE_TENANT_ID`/`AZURE_FEDERATED_TOKEN_FILE`). Set `account` (or `endpoint`) for the storage endpoint, annotate the ServiceAccount with `azure.workload.identity/client-id` (`serviceAccount.annotations`) and label the pod `azure.workload.identity/use: "true"` (`podLabels`).
+      managedIdentity: false
       # -- Storage account name
       account: ""
       # -- Storage account key configuration

test-values/azureblob-managed-identity.yaml

@@ -0,0 +1,25 @@
+# Render/lint scenario: Azure Blob with Managed/Workload Identity (no static key).
+# Proves the chart can render both jclouds.identity and jclouds.credential EMPTY so
+# azureblob-sdk uses DefaultAzureCredential — the account name lives in the endpoint,
+# never in jclouds.identity. Also exercises the workload-identity SA/pod wiring.
+config:
+  auth:
+    type: aws-v4
+    identity: test-access-key
+    secret: test-secret-key
+  backends:
+    filesystem:
+      enabled: false
+    azureblob:
+      enabled: true
+      provider: azureblob-sdk
+      managedIdentity: true
+      account: teststorageaccount
+serviceAccount:
+  create: true
+  annotations:
+    azure.workload.identity/client-id: "00000000-0000-0000-0000-000000000000"
+podLabels:
+  azure.workload.identity/use: "true"
+persistence:
+  enabled: false