Skip to content

Commit ffbae23

Browse files
lvvvvvflesv
authored and
minherz
committed
add export to bq feature (GoogleCloudPlatform#2281)
* Subject: add export to BigQuery Type: new feature * Fix comments on passing args * fix copyright time * fix gethistory failure Co-authored-by: Les Vogel <[email protected]>
1 parent 50df3ae commit ffbae23

2 files changed

Lines changed: 87 additions & 0 deletions

File tree

‎asset/src/main/java/com/example/asset/ExportAssetsBigqueryExample.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.asset;
18+
19+
// [START asset_quickstart_export_assets_bigquery]
20+
// Imports the Google Cloud client library
21+
22+
import com.google.cloud.ServiceOptions;
23+
import com.google.cloud.asset.v1.AssetServiceClient;
24+
import com.google.cloud.asset.v1.BigQueryDestination;
25+
import com.google.cloud.asset.v1.ExportAssetsRequest;
26+
import com.google.cloud.asset.v1.ExportAssetsResponse;
27+
import com.google.cloud.asset.v1.OutputConfig;
28+
import com.google.cloud.asset.v1.ProjectName;
29+
30+
public class ExportAssetsBigqueryExample {
31+
32+
// Use the default project Id.
33+
private static final String projectId = ServiceOptions.getDefaultProjectId();
34+
35+
// Export assets for a project.
36+
// @param args path where the results will be exported to.
37+
public static void exportBigQuery(String bigqueryDataset, String bigqueryTable) throws Exception {
38+
try (AssetServiceClient client = AssetServiceClient.create()) {
39+
ProjectName parent = ProjectName.of(projectId);
40+
OutputConfig outputConfig =
41+
OutputConfig.newBuilder()
42+
.setBigqueryDestination(
43+
BigQueryDestination.newBuilder()
44+
.setDataset(bigqueryDataset)
45+
.setTable(bigqueryTable)
46+
.setForce(true)
47+
.build())
48+
.build();
49+
ExportAssetsRequest request =
50+
ExportAssetsRequest.newBuilder()
51+
.setParent(parent.toString())
52+
.setOutputConfig(outputConfig)
53+
.build();
54+
ExportAssetsResponse response = client.exportAssetsAsync(request).get();
55+
System.out.println(response);
56+
}
57+
}
58+
}
59+
// [END asset_quickstart_export_assets_bigquery]

‎asset/src/test/java/com/example/asset/QuickStartIT.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,14 @@
1818

1919
import static com.google.common.truth.Truth.assertThat;
2020

21+
import com.google.cloud.ServiceOptions;
22+
import com.google.cloud.bigquery.BigQuery;
23+
import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption;
24+
import com.google.cloud.bigquery.BigQueryOptions;
25+
import com.google.cloud.bigquery.Dataset;
26+
import com.google.cloud.bigquery.DatasetId;
27+
import com.google.cloud.bigquery.DatasetInfo;
28+
import com.google.cloud.bigquery.testing.RemoteBigQueryHelper;
2129
import com.google.cloud.storage.BlobInfo;
2230
import com.google.cloud.storage.Bucket;
2331
import com.google.cloud.storage.BucketInfo;
@@ -38,8 +46,10 @@
3846
@SuppressWarnings("checkstyle:abbreviationaswordinname")
3947
public class QuickStartIT {
4048
private static final String bucketName = UUID.randomUUID().toString();
49+
private static final String datasetName = RemoteBigQueryHelper.generateDatasetName();
4150
private ByteArrayOutputStream bout;
4251
private PrintStream out;
52+
private BigQuery bigquery;
4353

4454
private static final void deleteBucket(String bucketName) {
4555
Storage storage = StorageOptions.getDefaultInstance().getService();
@@ -62,6 +72,10 @@ private static final void createBucket(String bucketName) {
6272
@Before
6373
public void setUp() {
6474
createBucket(bucketName);
75+
bigquery = BigQueryOptions.getDefaultInstance().getService();
76+
if (bigquery.getDataset(datasetName) == null) {
77+
Dataset dataset = bigquery.create(DatasetInfo.newBuilder(datasetName).build());
78+
}
6579
bout = new ByteArrayOutputStream();
6680
out = new PrintStream(bout);
6781
System.setOut(out);
@@ -72,6 +86,8 @@ public void tearDown() {
7286
String consoleOutput = bout.toString();
7387
System.setOut(null);
7488
deleteBucket(bucketName);
89+
DatasetId datasetId = DatasetId.of(bigquery.getOptions().getProjectId(), datasetName);
90+
bigquery.delete(datasetId, DatasetDeleteOption.deleteContents());
7591
}
7692

7793
@Test
@@ -82,8 +98,20 @@ public void testExportAssetExample() throws Exception {
8298
assertThat(got).contains(String.format("uri: \"%s\"", assetDumpPath));
8399
}
84100

101+
@Test
102+
public void testExportAssetBigqueryExample() throws Exception {
103+
String dataset =
104+
String.format("projects/%s/datasets/%s", ServiceOptions.getDefaultProjectId(), datasetName);
105+
String table = "java_test";
106+
ExportAssetsBigqueryExample.exportBigQuery(dataset, table);
107+
String got = bout.toString();
108+
assertThat(got).contains(String.format("dataset: \"%s\"", dataset));
109+
}
110+
85111
@Test
86112
public void testBatchGetAssetsHistory() throws Exception {
113+
// Wait 10 seconds to let bucket creation event go to CAI
114+
Thread.sleep(10000);
87115
String bucketAssetName = String.format("//storage.googleapis.com/%s", bucketName);
88116
BatchGetAssetsHistoryExample.main(bucketAssetName);
89117
String got = bout.toString();

0 commit comments

Comments
 (0)