Skip to content

Commit 2090e82

Browse files
JesseLovelaceJesseLovelace
authored
STS Azure sample (GoogleCloudPlatform#7529)
* STS Azure samples * use python in run script * adjust run script * fix python script * fix variable reference * fix lint * address review comments * fix azure acc example
1 parent 4fe195c commit 2090e82

File tree

5 files changed

+189
-25
lines changed

5 files changed

+189
-25
lines changed

‎.kokoro/tests/run_tests.sh‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,6 @@ if [[ "$SCRIPT_DEBUG" != "true" ]]; then
6969
SECRET_FILES=("java-docs-samples-service-account.json" \
7070
"java-aiplatform-samples-secrets.txt" \
7171
"java-automl-samples-secrets.txt" \
72-
"java-aws-samples-secrets.txt" \
7372
"java-bigtable-samples-secrets.txt" \
7473
"java-cloud-sql-samples-secrets.txt" \
7574
"java-cts-v4-samples-secrets.txt" \
@@ -89,6 +88,14 @@ if [[ "$SCRIPT_DEBUG" != "true" ]]; then
8988
source "${KOKORO_GFILE_DIR}/secrets/$SECRET"
9089
fi
9190
done
91+
92+
export STS_AWS_SECRET=`gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=java-storagetransfer-aws`
93+
export AWS_ACCESS_KEY_ID=`S="$STS_AWS_SECRET" python3 -c 'import json,sys,os;obj=json.loads(os.getenv("S"));print (obj["AccessKeyId"]);'`
94+
export AWS_SECRET_ACCESS_KEY=`S="$STS_AWS_SECRET" python3 -c 'import json,sys,os;obj=json.loads(os.getenv("S"));print (obj["SecretAccessKey"]);'`
95+
export STS_AZURE_SECRET=`gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=java-storagetransfer-azure`
96+
export AZURE_STORAGE_ACCOUNT=`S="$STS_AZURE_SECRET" python3 -c 'import json,sys,os;obj=json.loads(os.getenv("S"));print (obj["StorageAccount"]);'`
97+
export AZURE_CONNECTION_STRING=`S="$STS_AZURE_SECRET" python3 -c 'import json,sys,os;obj=json.loads(os.getenv("S"));print (obj["ConnectionString"]);'`
98+
export AZURE_SAS_TOKEN=`S="$STS_AZURE_SECRET" python3 -c 'import json,sys,os;obj=json.loads(os.getenv("S"));print (obj["SAS"]);'`
9299

93100
# Activate service account
94101
gcloud auth activate-service-account \

‎storage-transfer/pom.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,5 +103,16 @@
103103
<scope>test</scope>
104104
</dependency>
105105

106+
<dependency>
107+
<groupId>com.azure</groupId>
108+
<artifactId>azure-storage-blob</artifactId>
109+
<version>12.20.1</version>
110+
</dependency>
111+
<dependency>
112+
<groupId>com.azure</groupId>
113+
<artifactId>azure-identity</artifactId>
114+
<version>1.7.2</version>
115+
</dependency>
116+
106117
</dependencies>
107118
</project>

‎storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/QuickstartSample.java‎

Lines changed: 27 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -44,31 +44,36 @@ public static void main(String[] args) throws Exception {
4444
public static void quickStartSample(
4545
String projectId, String gcsSourceBucket, String gcsSinkBucket) throws Exception {
4646

47-
StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.create();
47+
// Initialize client that will be used to send requests. This client only needs to be created
48+
// once, and can be reused for multiple requests. After completing all of your requests, call
49+
// the "close" method on the client to safely clean up any remaining background resources,
50+
// or use "try-with-close" statement to do this automatically.
51+
try (StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.create()) {
4852

49-
TransferJob transferJob =
50-
TransferJob.newBuilder()
51-
.setProjectId(projectId)
52-
.setTransferSpec(
53-
TransferSpec.newBuilder()
54-
.setGcsDataSource(GcsData.newBuilder().setBucketName(gcsSourceBucket))
55-
.setGcsDataSink(GcsData.newBuilder().setBucketName(gcsSinkBucket)))
56-
.setStatus(TransferJob.Status.ENABLED)
57-
.build();
53+
TransferJob transferJob =
54+
TransferJob.newBuilder()
55+
.setProjectId(projectId)
56+
.setTransferSpec(
57+
TransferSpec.newBuilder()
58+
.setGcsDataSource(GcsData.newBuilder().setBucketName(gcsSourceBucket))
59+
.setGcsDataSink(GcsData.newBuilder().setBucketName(gcsSinkBucket)))
60+
.setStatus(TransferJob.Status.ENABLED)
61+
.build();
5862

59-
TransferJob response =
60-
storageTransfer.createTransferJob(
61-
CreateTransferJobRequest.newBuilder().setTransferJob(transferJob).build());
63+
TransferJob response =
64+
storageTransfer.createTransferJob(
65+
CreateTransferJobRequest.newBuilder().setTransferJob(transferJob).build());
6266

63-
storageTransfer
64-
.runTransferJobAsync(
65-
RunTransferJobRequest.newBuilder()
66-
.setProjectId(projectId)
67-
.setJobName(response.getName())
68-
.build())
69-
.get();
70-
System.out.println(
71-
"Created and ran transfer job between two GCS buckets with name " + response.getName());
67+
storageTransfer
68+
.runTransferJobAsync(
69+
RunTransferJobRequest.newBuilder()
70+
.setProjectId(projectId)
71+
.setJobName(response.getName())
72+
.build())
73+
.get();
74+
System.out.println(
75+
"Created and ran transfer job between two GCS buckets with name " + response.getName());
76+
}
7277
}
7378
}
7479
// [END storagetransfer_quickstart]

‎storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/TransferFromAzure.java‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Copyright 2022 Google Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage.storagetransfer.samples;
18+
19+
// [START storagetransfer_transfer_from_azure]
20+
import com.google.storagetransfer.v1.proto.StorageTransferServiceClient;
21+
import com.google.storagetransfer.v1.proto.TransferProto;
22+
import com.google.storagetransfer.v1.proto.TransferProto.RunTransferJobRequest;
23+
import com.google.storagetransfer.v1.proto.TransferTypes.AzureBlobStorageData;
24+
import com.google.storagetransfer.v1.proto.TransferTypes.AzureCredentials;
25+
import com.google.storagetransfer.v1.proto.TransferTypes.GcsData;
26+
import com.google.storagetransfer.v1.proto.TransferTypes.TransferJob;
27+
import com.google.storagetransfer.v1.proto.TransferTypes.TransferJob.Status;
28+
import com.google.storagetransfer.v1.proto.TransferTypes.TransferSpec;
29+
import java.io.IOException;
30+
import java.util.concurrent.ExecutionException;
31+
32+
public class TransferFromAzure {
33+
public static void main(String[] args)
34+
throws IOException, ExecutionException, InterruptedException {
35+
// TODO(developer): Replace these variables before running the sample.
36+
// Your Google Cloud Project ID
37+
String projectId = "my-project-id";
38+
39+
// Your Azure Storage Account name
40+
String azureStorageAccount = "my-azure-account";
41+
42+
// The Azure source container to transfer data from
43+
String azureSourceContainer = "my-source-container";
44+
45+
// The GCS bucket to transfer data to
46+
String gcsSinkBucket = "my-sink-bucket";
47+
48+
transferFromAzureBlobStorage(
49+
projectId, azureStorageAccount, azureSourceContainer, gcsSinkBucket);
50+
}
51+
52+
/**
53+
* Creates and runs a transfer job to transfer all data from an Azure container to a GCS bucket.
54+
*/
55+
public static void transferFromAzureBlobStorage(String projectId, String azureStorageAccount,
56+
String azureSourceContainer, String gcsSinkBucket)
57+
throws IOException, ExecutionException, InterruptedException {
58+
59+
// Your Azure SAS token, should be accessed via environment variable
60+
String azureSasToken = System.getenv("AZURE_SAS_TOKEN");
61+
62+
TransferSpec transferSpec = TransferSpec.newBuilder()
63+
.setAzureBlobStorageDataSource(
64+
AzureBlobStorageData.newBuilder()
65+
.setAzureCredentials(AzureCredentials.newBuilder()
66+
.setSasToken(azureSasToken)
67+
.build())
68+
.setContainer(azureSourceContainer)
69+
.setStorageAccount(azureStorageAccount))
70+
.setGcsDataSink(GcsData.newBuilder().setBucketName(gcsSinkBucket).build())
71+
.build();
72+
73+
TransferJob transferJob =
74+
TransferJob.newBuilder()
75+
.setProjectId(projectId)
76+
.setStatus(Status.ENABLED)
77+
.setTransferSpec(transferSpec)
78+
.build();
79+
80+
// Initialize client that will be used to send requests. This client only needs to be created
81+
// once, and can be reused for multiple requests. After completing all of your requests, call
82+
// the "close" method on the client to safely clean up any remaining background resources,
83+
// or use "try-with-close" statement to do this automatically.
84+
try (StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.create()) {
85+
// Create the transfer job
86+
TransferJob response =
87+
storageTransfer.createTransferJob(TransferProto.CreateTransferJobRequest.newBuilder()
88+
.setTransferJob(transferJob)
89+
.build());
90+
91+
// Run the created job
92+
storageTransfer
93+
.runTransferJobAsync(
94+
RunTransferJobRequest.newBuilder()
95+
.setProjectId(projectId)
96+
.setJobName(response.getName())
97+
.build())
98+
.get();
99+
100+
System.out.println(
101+
"Created and ran a transfer job from "
102+
+ azureSourceContainer
103+
+ " to "
104+
+ gcsSinkBucket
105+
+ " with "
106+
+ "name "
107+
+ response.getName());
108+
}
109+
110+
}
111+
}
112+
// [END storagetransfer_transfer_from_azure]

‎storage-transfer/src/test/java/com/google/cloud/storage/storagetransfer/samples/test/ITStoragetransferSamplesTest.java‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
2525
import com.amazonaws.services.s3.model.ObjectListing;
2626
import com.amazonaws.services.s3.model.S3ObjectSummary;
27+
import com.azure.storage.blob.BlobContainerClient;
28+
import com.azure.storage.blob.BlobServiceClient;
29+
import com.azure.storage.blob.BlobServiceClientBuilder;
2730
import com.google.api.services.storagetransfer.v1.Storagetransfer;
2831
import com.google.api.services.storagetransfer.v1.model.Date;
2932
import com.google.api.services.storagetransfer.v1.model.GcsData;
@@ -48,6 +51,7 @@
4851
import com.google.cloud.storage.storagetransfer.samples.QuickstartSample;
4952
import com.google.cloud.storage.storagetransfer.samples.TransferBetweenPosix;
5053
import com.google.cloud.storage.storagetransfer.samples.TransferFromAws;
54+
import com.google.cloud.storage.storagetransfer.samples.TransferFromAzure;
5155
import com.google.cloud.storage.storagetransfer.samples.TransferFromPosix;
5256
import com.google.cloud.storage.storagetransfer.samples.TransferFromS3CompatibleSource;
5357
import com.google.cloud.storage.storagetransfer.samples.TransferToNearline;
@@ -60,17 +64,22 @@
6064
import com.google.cloud.storage.testing.RemoteStorageHelper;
6165
import com.google.cloud.testing.junit4.MultipleAttemptsRule;
6266
import com.google.cloud.testing.junit4.StdOutCaptureRule;
67+
import com.google.common.base.Strings;
6368
import com.google.common.collect.ImmutableList;
69+
import com.google.gson.Gson;
6470
import com.google.storagetransfer.v1.proto.StorageTransferServiceClient;
6571
import com.google.storagetransfer.v1.proto.TransferProto;
6672
import com.google.storagetransfer.v1.proto.TransferProto.GetGoogleServiceAccountRequest;
6773
import com.google.storagetransfer.v1.proto.TransferTypes;
74+
import java.io.Reader;
6875
import java.nio.file.Files;
76+
import java.nio.file.Paths;
6977
import java.text.SimpleDateFormat;
7078
import java.util.ArrayList;
7179
import java.util.Arrays;
7280
import java.util.Iterator;
7381
import java.util.List;
82+
import java.util.Map;
7483
import java.util.UUID;
7584
import java.util.concurrent.ExecutionException;
7685
import java.util.concurrent.TimeUnit;
@@ -87,8 +96,14 @@ public class ITStoragetransferSamplesTest {
8796
private static final String SINK_GCS_BUCKET = "sts-test-bucket-sink" + UUID.randomUUID();
8897
private static final String SOURCE_GCS_BUCKET = "sts-test-bucket-source" + UUID.randomUUID();
8998
private static final String AMAZON_BUCKET = "sts-amazon-bucket" + UUID.randomUUID();
99+
private static final String AZURE_BUCKET = "sts-azure-bucket" + UUID.randomUUID();
100+
private static String AZURE_CONNECTION_STRING = System.getenv("AZURE_CONNECTION_STRING");
101+
private static String AZURE_STORAGE_ACCOUNT = System.getenv("AZURE_STORAGE_ACCOUNT");
102+
private static String AZURE_SAS_TOKEN = System.getenv("AZURE_SAS_TOKEN");
90103
private static Storage storage;
91104
private static AmazonS3 s3;
105+
private static BlobServiceClient blobServiceClient;
106+
private static BlobContainerClient blobContainerClient;
92107
private static StorageTransferServiceClient sts;
93108

94109
@Rule public MultipleAttemptsRule multipleAttemptsRule = new MultipleAttemptsRule(5);
@@ -131,6 +146,12 @@ public static void beforeClass() throws Exception {
131146
s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.US_WEST_1).build();
132147

133148
s3.createBucket(AMAZON_BUCKET);
149+
150+
blobServiceClient = new BlobServiceClientBuilder()
151+
.connectionString(AZURE_CONNECTION_STRING)
152+
.sasToken(AZURE_SAS_TOKEN)
153+
.buildClient();
154+
blobContainerClient = blobServiceClient.createBlobContainer(AZURE_BUCKET);
134155
}
135156

136157
private static void grantBucketsStsPermissions(String serviceAccount, String bucket)
@@ -213,9 +234,8 @@ public static void afterClass() throws ExecutionException, InterruptedException
213234
RemoteStorageHelper.forceDelete(storage, SINK_GCS_BUCKET, 1, TimeUnit.MINUTES);
214235
RemoteStorageHelper.forceDelete(storage, SOURCE_GCS_BUCKET, 1, TimeUnit.MINUTES);
215236
}
216-
237+
blobContainerClient.delete();
217238
cleanAmazonBucket();
218-
219239
sts.shutdownNow();
220240
}
221241

@@ -474,4 +494,13 @@ public void testTransferFromS3CompatibleSource() throws Exception {
474494
assertThat(sampleOutput).contains("transferJobs/");
475495
deleteTransferJob(sampleOutput);
476496
}
497+
498+
@Test
499+
public void testTransferFromAzure() throws Exception {
500+
TransferFromAzure.transferFromAzureBlobStorage(
501+
PROJECT_ID, AZURE_STORAGE_ACCOUNT, AZURE_BUCKET, SINK_GCS_BUCKET);
502+
String sampleOutput = stdOutCaptureRule.getCapturedOutputAsUtf8String();
503+
assertThat(sampleOutput).contains("transferJobs/");
504+
deleteTransferJob(sampleOutput);
505+
}
477506
}

0 commit comments

Comments
 (0)