Switch to DefaultAzureCredential (#40689)

* wip

* more environmentcredential removal

* adding token method to common and providing test config info

* changing queue oauth helper function

* adding AzurePowerShellCredential as the token to use for live tests

* sync change

* async change

* fixing analyze error

* change to test-template

* empty commit to fix pipeline

* changing immutablestoragewithversioningtests to use environment variables

* changing live tests to use AzurePipelinesCredentialBuilder

* removing adal4j

* adding support for running live tests locally to getTokenCredential

* making azurepowershellcredential last in the chain

* adjusting subscription configuration

* adjusting subscription configuration again

* changing immutablestorage copy tests to use sas, updating error returned by audience failure for files

* unused imports

* adding different credential builders for oauthcopysourcetests

* style

* listSharesEnableSnapshotVirtualDirectoryAccess fix

* Update StorageCommonTestUtils.java

* getTokenCredential fix and removing adal4j dependency from blob-batch

* fixing getAuthToken in blobs and files

* removing unused imports

---------

Co-authored-by: Vinay Gera <vigera@microsoft.com>

sdk/storage/azure-storage-blob-batch/pom.xml

@@ -110,12 +110,6 @@
       <version>3.4.38</version> <!-- {x-version-update;io.projectreactor:reactor-test;external_dependency} -->
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>com.microsoft.azure</groupId>
-      <artifactId>adal4j</artifactId>
-      <version>1.6.5</version> <!-- {x-version-update;com.microsoft.azure:adal4j;external_dependency} -->
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>cglib</groupId>
       <artifactId>cglib-nodep</artifactId>

sdk/storage/azure-storage-blob-batch/src/test/java/com/azure/storage/blob/batch/BlobBatchTestBase.java

@@ -12,7 +12,6 @@
 import com.azure.core.test.models.TestProxySanitizer;
 import com.azure.core.test.models.TestProxySanitizerType;
 import com.azure.core.util.CoreUtils;
-import com.azure.identity.EnvironmentCredentialBuilder;
 import com.azure.storage.blob.BlobContainerClient;
 import com.azure.storage.blob.BlobContainerClientBuilder;
 import com.azure.storage.blob.BlobServiceAsyncClient;
@@ -92,13 +91,7 @@ protected BlobServiceClient getOAuthServiceClient() {
 
         instrument(builder);
 
-        if (getTestMode() != TestMode.PLAYBACK) {
-            // AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET
-            return builder.credential(new EnvironmentCredentialBuilder().build()).buildClient();
-        } else {
-            // Running in playback, we don't have access to the AAD environment variables, just use SharedKeyCredential.
-            return builder.credential(ENVIRONMENT.getPrimaryAccount().getCredential()).buildClient();
-        }
+        return builder.credential(StorageCommonTestUtils.getTokenCredential(interceptorManager)).buildClient();
     }
 
     protected BlobServiceClient getServiceClient(TestAccount account) {

sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/KeyVaultKeyTests.java

@@ -15,9 +15,7 @@
 import com.azure.core.http.policy.RetryStrategy;
 import com.azure.core.http.policy.UserAgentPolicy;
 import com.azure.core.test.TestMode;
-import com.azure.core.test.utils.MockTokenCredential;
 import com.azure.core.util.Configuration;
-import com.azure.identity.ClientSecretCredentialBuilder;
 import com.azure.security.keyvault.keys.KeyClient;
 import com.azure.security.keyvault.keys.KeyClientBuilder;
 import com.azure.security.keyvault.keys.KeyServiceVersion;
@@ -27,6 +25,7 @@
 import com.azure.security.keyvault.keys.models.KeyVaultKey;
 import com.azure.storage.blob.BlobContainerClient;
 import com.azure.storage.common.implementation.Constants;
+import com.azure.storage.common.test.shared.StorageCommonTestUtils;
 import org.junit.jupiter.api.Test;
 
 import java.io.ByteArrayInputStream;
@@ -120,15 +119,7 @@ private HttpPipeline getHttpPipeline(KeyServiceVersion serviceVersion) {
         Configuration global = Configuration.getGlobalConfiguration().clone();
         TokenCredential credential;
 
-        if (getTestMode() != TestMode.PLAYBACK) {
-            credential = new ClientSecretCredentialBuilder()
-                .clientSecret(global.get("AZURE_CLIENT_SECRET"))
-                .clientId(global.get("AZURE_CLIENT_ID"))
-                .tenantId(global.get("AZURE_TENANT_ID"))
-                .build();
-        } else {
-            credential = new MockTokenCredential();
-        }
+        credential = StorageCommonTestUtils.getTokenCredential(interceptorManager);
 
         // Closest to API goes first, closest to wire goes last.
         final List<HttpPipelinePolicy> policies = new ArrayList<>();

sdk/storage/azure-storage-blob/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "java",
   "TagPrefix": "java/storage/azure-storage-blob",
-  "Tag": "java/storage/azure-storage-blob_b29c63bb55"
+  "Tag": "java/storage/azure-storage-blob_49a56bfab4"
 }

sdk/storage/azure-storage-blob/pom.xml

@@ -147,12 +147,6 @@
       <version>5.9.3</version> <!-- {x-version-update;org.junit.jupiter:junit-jupiter-params;external_dependency} -->
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>com.microsoft.azure</groupId>
-      <artifactId>adal4j</artifactId>
-      <version>1.6.5</version> <!-- {x-version-update;com.microsoft.azure:adal4j;external_dependency} -->
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.objenesis</groupId>
       <artifactId>objenesis</artifactId>

sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java

@@ -4,7 +4,6 @@
 package com.azure.storage.blob;
 
 import com.azure.core.client.traits.HttpTrait;
-import com.azure.core.credential.AccessToken;
 import com.azure.core.credential.TokenRequestContext;
 import com.azure.core.http.HttpClient;
 import com.azure.core.http.HttpHeaderName;
@@ -20,7 +19,6 @@
 import com.azure.core.util.CoreUtils;
 import com.azure.core.util.FluxUtil;
 import com.azure.core.util.logging.ClientLogger;
-import com.azure.identity.EnvironmentCredentialBuilder;
 import com.azure.storage.blob.models.BlobContainerItem;
 import com.azure.storage.blob.models.BlobErrorCode;
 import com.azure.storage.blob.models.BlobProperties;
@@ -217,14 +215,13 @@ protected static Mono<ByteBuffer> collectBytesInBuffer(Flux<ByteBuffer> content)
         return FluxUtil.collectBytesInByteBufferStream(content).map(ByteBuffer::wrap);
     }
 
-    protected static String getAuthToken() {
+    protected String getAuthToken() {
         if (ENVIRONMENT.getTestMode() == TestMode.PLAYBACK) {
             // we just need some string to satisfy SDK for playback mode. Recording framework handles this fine.
             return "recordingBearerToken";
         }
-        return new EnvironmentCredentialBuilder().build().getToken(new TokenRequestContext()
-                .setScopes(Collections.singletonList("https://storage.azure.com/.default")))
-            .map(AccessToken::getToken).block();
+        return StorageCommonTestUtils.getTokenCredential(interceptorManager).getTokenSync(new TokenRequestContext()
+                .setScopes(Collections.singletonList("https://storage.azure.com/.default"))).getToken();
     }
 
     protected String generateContainerName() {
@@ -331,7 +328,7 @@ protected BlobServiceClient getOAuthServiceClient() {
 
         instrument(builder);
 
-        return setOauthCredentials(builder).buildClient();
+        return builder.credential(StorageCommonTestUtils.getTokenCredential(interceptorManager)).buildClient();
     }
 
     protected BlobServiceAsyncClient getOAuthServiceAsyncClient() {
@@ -340,17 +337,7 @@ protected BlobServiceAsyncClient getOAuthServiceAsyncClient() {
 
         instrument(builder);
 
-        return setOauthCredentials(builder).buildAsyncClient();
-    }
-
-    protected BlobServiceClientBuilder setOauthCredentials(BlobServiceClientBuilder builder) {
-        if (ENVIRONMENT.getTestMode() != TestMode.PLAYBACK) {
-            // AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET
-            return builder.credential(new EnvironmentCredentialBuilder().build());
-        } else {
-            // Running in playback, we don't have access to the AAD environment variables, just use SharedKeyCredential.
-            return builder.credential(ENVIRONMENT.getPrimaryAccount().getCredential());
-        }
+        return builder.credential(StorageCommonTestUtils.getTokenCredential(interceptorManager)).buildAsyncClient();
     }
 
     protected BlobServiceClient getServiceClient(String endpoint) {
@@ -422,13 +409,7 @@ protected BlobServiceClientBuilder getServiceClientBuilderWithTokenCredential(St
             builder.addPolicy(policy);
         }
 
-        if (ENVIRONMENT.getTestMode() != TestMode.PLAYBACK) {
-            // AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET
-            builder.credential(new EnvironmentCredentialBuilder().build());
-        } else {
-            // Running in playback, we don't have access to the AAD environment variables, just use SharedKeyCredential.
-            builder.credential(ENVIRONMENT.getPrimaryAccount().getCredential());
-        }
+        builder.credential(StorageCommonTestUtils.getTokenCredential(interceptorManager));
 
         instrument(builder);
         return builder;
@@ -531,13 +512,7 @@ protected BlobContainerClientBuilder getContainerClientBuilderWithTokenCredentia
             builder.addPolicy(policy);
         }
 
-        if (ENVIRONMENT.getTestMode() != TestMode.PLAYBACK) {
-            // AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET
-            builder.credential(new EnvironmentCredentialBuilder().build());
-        } else {
-            // Running in playback, we don't have access to the AAD environment variables, just use SharedKeyCredential.
-            builder.credential(ENVIRONMENT.getPrimaryAccount().getCredential());
-        }
+        builder.credential(StorageCommonTestUtils.getTokenCredential(interceptorManager));
 
         instrument(builder);
         return builder;
@@ -600,13 +575,7 @@ protected BlobClientBuilder getBlobClientBuilderWithTokenCredential(String endpo
             builder.addPolicy(policy);
         }
 
-        if (ENVIRONMENT.getTestMode() != TestMode.PLAYBACK) {
-            // AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET
-            builder.credential(new EnvironmentCredentialBuilder().build());
-        } else {
-            // Running in playback, we don't have access to the AAD environment variables, just use SharedKeyCredential.
-            builder.credential(ENVIRONMENT.getPrimaryAccount().getCredential());
-        }
+        builder.credential(StorageCommonTestUtils.getTokenCredential(interceptorManager));
 
         instrument(builder);
         return builder;
@@ -693,13 +662,7 @@ protected SpecializedBlobClientBuilder getSpecializedBuilderWithTokenCredential(
             builder.addPolicy(policy);
         }
 
-        if (ENVIRONMENT.getTestMode() != TestMode.PLAYBACK) {
-            // AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET
-            builder.credential(new EnvironmentCredentialBuilder().build());
-        } else {
-            // Running in playback, we don't have access to the AAD environment variables, just use SharedKeyCredential.
-            builder.credential(ENVIRONMENT.getPrimaryAccount().getCredential());
-        }
+        builder.credential(StorageCommonTestUtils.getTokenCredential(interceptorManager));
 
         instrument(builder);
         return builder;

sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ImmutableStorageWithVersioningAsyncTests.java

@@ -12,7 +12,16 @@
 import com.azure.core.http.HttpResponse;
 import com.azure.core.http.policy.BearerTokenAuthenticationPolicy;
 import com.azure.core.test.TestMode;
+import com.azure.core.test.utils.MockTokenCredential;
+import com.azure.core.util.Configuration;
+import com.azure.core.util.CoreUtils;
 import com.azure.core.util.polling.PollerFlux;
+import com.azure.identity.AzureCliCredentialBuilder;
+import com.azure.identity.AzureDeveloperCliCredentialBuilder;
+import com.azure.identity.AzurePipelinesCredentialBuilder;
+import com.azure.identity.AzurePowerShellCredentialBuilder;
+import com.azure.identity.ChainedTokenCredentialBuilder;
+import com.azure.identity.DefaultAzureCredentialBuilder;
 import com.azure.identity.EnvironmentCredentialBuilder;
 import com.azure.storage.blob.models.BlobContainerProperties;
 import com.azure.storage.blob.models.BlobCopyInfo;
@@ -28,7 +37,6 @@
 import com.azure.storage.blob.models.ListBlobContainersOptions;
 import com.azure.storage.blob.models.ListBlobsOptions;
 import com.azure.storage.blob.models.ParallelTransferOptions;
-import com.azure.storage.blob.models.PublicAccessType;
 import com.azure.storage.blob.options.AppendBlobCreateOptions;
 import com.azure.storage.blob.options.BlobBeginCopyOptions;
 import com.azure.storage.blob.options.BlobBreakLeaseOptions;
@@ -48,7 +56,6 @@
 import com.azure.storage.common.sas.AccountSasService;
 import com.azure.storage.common.sas.AccountSasSignatureValues;
 import com.azure.storage.common.test.shared.extensions.LiveOnly;
-import com.azure.storage.common.test.shared.extensions.PlaybackOnly;
 import com.azure.storage.common.test.shared.extensions.RequiredServiceVersion;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -90,7 +97,7 @@ public class ImmutableStorageWithVersioningAsyncTests extends BlobTestBase {
     private static final String RESOURCE_GROUP_NAME = ENVIRONMENT.getResourceGroupName();
     private static final String SUBSCRIPTION_ID = ENVIRONMENT.getSubscriptionId();
     private static final String API_VERSION = "2021-04-01";
-    private static final TokenCredential CREDENTIAL = new EnvironmentCredentialBuilder().build();
+    private static final TokenCredential CREDENTIAL = getTokenCredential(ENVIRONMENT.getTestMode());
     private static final BearerTokenAuthenticationPolicy CREDENTIAL_POLICY =
         new BearerTokenAuthenticationPolicy(CREDENTIAL, "https://management.azure.com/.default");
     private BlobContainerAsyncClient vlwContainer;
@@ -250,6 +257,43 @@ public static void cleanupSpec() throws MalformedURLException {
         }
     }
 
+    public static TokenCredential getTokenCredential(TestMode testMode) {
+        if (testMode == TestMode.RECORD) {
+            return new DefaultAzureCredentialBuilder().build();
+        } else if (testMode == TestMode.LIVE) {
+            Configuration config = Configuration.getGlobalConfiguration();
+
+            ChainedTokenCredentialBuilder builder = new ChainedTokenCredentialBuilder()
+                .addLast(new EnvironmentCredentialBuilder().build())
+                .addLast(new AzureCliCredentialBuilder().build())
+                .addLast(new AzureDeveloperCliCredentialBuilder().build());
+
+            String serviceConnectionId = config.get("AZURESUBSCRIPTION_SERVICE_CONNECTION_ID");
+            String clientId = config.get("AZURESUBSCRIPTION_CLIENT_ID");
+            String tenantId = config.get("AZURESUBSCRIPTION_TENANT_ID");
+            String systemAccessToken = config.get("SYSTEM_ACCESSTOKEN");
+
+            if (!CoreUtils.isNullOrEmpty(serviceConnectionId)
+                && !CoreUtils.isNullOrEmpty(clientId)
+                && !CoreUtils.isNullOrEmpty(tenantId)
+                && !CoreUtils.isNullOrEmpty(systemAccessToken)) {
+
+                builder.addLast(new AzurePipelinesCredentialBuilder()
+                    .systemAccessToken(systemAccessToken)
+                    .clientId(clientId)
+                    .tenantId(tenantId)
+                    .serviceConnectionId(serviceConnectionId)
+                    .build());
+            }
+
+            builder.addLast(new AzurePowerShellCredentialBuilder().build());
+
+            return builder.build();
+        } else { //playback or not set
+            return new MockTokenCredential();
+        }
+    }
+
     @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2020-10-02")
     @Test
     public void setImmutabilityPolicyMin() {
@@ -514,7 +558,7 @@ public void containerProperties() {
             .verifyComplete();
 
         StepVerifier.create(vlwContainer.getServiceAsyncClient().listBlobContainers(
-            new ListBlobContainersOptions().setPrefix(vlwContainer.getBlobContainerName())))
+                new ListBlobContainersOptions().setPrefix(vlwContainer.getBlobContainerName())))
             .assertNext(r -> assertTrue(r.getProperties().isImmutableStorageWithVersioningEnabled()))
             .verifyComplete();
     }
@@ -646,14 +690,9 @@ private static Stream<Arguments> blobUploadSupplier() {
         return Stream.of(Arguments.of(1L), Arguments.of((Long) null));
     }
 
-    // Live mode does not support copy from URL with immutability policy due to public access
-    // Revisit this and remove public access once default credential is enabled
     @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2020-10-02")
     @Test
-    @PlaybackOnly()
     public void syncCopy() {
-        vlwContainer.setAccessPolicy(PublicAccessType.CONTAINER, null).block();
-        sleepIfRunningAgainstService(30000); // Give time for the policy to take effect
         BlockBlobAsyncClient destination = vlwContainer.getBlobAsyncClient(generateBlobName()).getBlockBlobAsyncClient();
         OffsetDateTime expiryTime = testResourceNamer.now().plusDays(2);
         // The service rounds Immutability Policy Expiry to the nearest second.
@@ -662,7 +701,10 @@ public void syncCopy() {
             .setExpiryTime(expiryTime)
             .setPolicyMode(BlobImmutabilityPolicyMode.UNLOCKED);
 
-        destination.copyFromUrlWithResponse(new BlobCopyFromUrlOptions(vlwBlob.getBlobUrl())
+        String sas = vlwBlob.generateSas(new BlobServiceSasSignatureValues(testResourceNamer.now().plusDays(1),
+            new BlobSasPermission().setTagsPermission(true).setReadPermission(true)));
+
+        destination.copyFromUrlWithResponse(new BlobCopyFromUrlOptions(vlwBlob.getBlobUrl() + "?" + sas)
             .setImmutabilityPolicy(immutabilityPolicy)
             .setLegalHold(true)).block();
 
@@ -673,9 +715,6 @@ public void syncCopy() {
                 assertTrue(r.hasLegalHold());
             })
             .verifyComplete();
-
-        // cleanup:
-        vlwContainer.setAccessPolicy(null, null).block();
     }
 
     @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2020-10-02")
@@ -691,8 +730,8 @@ public void copy() {
 
         PollerFlux<BlobCopyInfo, Void> poller = setPlaybackPollerFluxPollInterval(
             destination.beginCopy(new BlobBeginCopyOptions(vlwBlob.getBlobUrl())
-            .setImmutabilityPolicy(immutabilityPolicy)
-            .setLegalHold(true)));
+                .setImmutabilityPolicy(immutabilityPolicy)
+                .setLegalHold(true)));
         poller.blockLast();
 
         StepVerifier.create(destination.getProperties())