Skip to content

Commit 929b2ce

Browse files
VeronicaWassonstephaniewang526
VeronicaWasson
and
stephaniewang526
authored
docs(samples): jsonstreamwriter samples (#756)
* docs(samples): jsonstreamwriter samples * Add copyright notice * Remove allowUnknownFields parameter * Added retry with exponential backoff * Revert "Added retry with exponential backoff" Remove the backoff logic to keep the sample code simpler. * Addressed PR review feedback. - Simplify code, remove duplicate-record example. - Split all snippets and tests into separate classes. - Add comments and javadocs links. - Clean up imports. - Add region tags. - Catch only specific exceptions. - Run linter and fmt-maven-plugin. * docs(samples): Fix mismatched region tag * Update samples/snippets/src/main/java/com/example/bigquerystorage/WriteCommittedStream.java * Update samples/snippets/src/main/java/com/example/bigquerystorage/WritePendingStream.java * Update samples/snippets/src/main/java/com/example/bigquerystorage/WritePendingStream.java * Update samples/snippets/src/main/java/com/example/bigquerystorage/WritePendingStream.java * Update samples/snippets/src/main/java/com/example/bigquerystorage/WriteToDefaultStream.java * Update samples/snippets/src/main/java/com/example/bigquerystorage/WriteToDefaultStream.java * Update samples/snippets/src/main/java/com/example/bigquerystorage/WriteToDefaultStream.java * Update samples/snippets/src/main/java/com/example/bigquerystorage/WriteToDefaultStream.java * Update samples/snippets/src/main/java/com/example/bigquerystorage/WritePendingStream.java * Update samples/snippets/src/main/java/com/example/bigquerystorage/WriteCommittedStream.java * Update samples/snippets/src/main/java/com/example/bigquerystorage/WriteCommittedStream.java * Update samples/snippets/src/main/java/com/example/bigquerystorage/WriteCommittedStream.java * Update samples/snippets/src/main/java/com/example/bigquerystorage/WritePendingStream.java * docs(samples): Create test resouces Create temporary dataset and table for sample integration tests Co-authored-by: Stephanie Wang <stephaniewang526@users.noreply.github.com>
1 parent 1c2b5c1 commit 929b2ce

File tree

6 files changed

+558
-0
lines changed

6 files changed

+558
-0
lines changed

‎samples/snippets/src/main/java/com/example/bigquerystorage/WriteCommittedStream.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquerystorage;
18+
19+
// [START bigquerystorage_jsonstreamwriter_committed]
20+
import com.google.api.core.ApiFuture;
21+
import com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse;
22+
import com.google.cloud.bigquery.storage.v1beta2.BigQueryWriteClient;
23+
import com.google.cloud.bigquery.storage.v1beta2.CreateWriteStreamRequest;
24+
import com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter;
25+
import com.google.cloud.bigquery.storage.v1beta2.TableName;
26+
import com.google.cloud.bigquery.storage.v1beta2.WriteStream;
27+
import com.google.protobuf.Descriptors.DescriptorValidationException;
28+
import java.io.IOException;
29+
import java.util.concurrent.ExecutionException;
30+
import org.json.JSONArray;
31+
import org.json.JSONObject;
32+
33+
public class WriteCommittedStream {
34+
35+
public static void runWriteCommittedStream()
36+
throws DescriptorValidationException, InterruptedException, IOException {
37+
// TODO(developer): Replace these variables before running the sample.
38+
String projectId = "MY_PROJECT_ID";
39+
String datasetName = "MY_DATASET_NAME";
40+
String tableName = "MY_TABLE_NAME";
41+
42+
writeCommittedStream(projectId, datasetName, tableName);
43+
}
44+
45+
public static void writeCommittedStream(String projectId, String datasetName, String tableName)
46+
throws DescriptorValidationException, InterruptedException, IOException {
47+
48+
try (BigQueryWriteClient client = BigQueryWriteClient.create()) {
49+
// Initialize a write stream for the specified table.
50+
// For more information on WriteStream.Type, see:
51+
// https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1beta2/WriteStream.Type.html
52+
WriteStream stream = WriteStream.newBuilder().setType(WriteStream.Type.COMMITTED).build();
53+
TableName parentTable = TableName.of(projectId, datasetName, tableName);
54+
CreateWriteStreamRequest createWriteStreamRequest =
55+
CreateWriteStreamRequest.newBuilder()
56+
.setParent(parentTable.toString())
57+
.setWriteStream(stream)
58+
.build();
59+
WriteStream writeStream = client.createWriteStream(createWriteStreamRequest);
60+
61+
// Use the JSON stream writer to send records in JSON format.
62+
// For more information about JsonStreamWriter, see:
63+
// https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriter.html
64+
try (JsonStreamWriter writer =
65+
JsonStreamWriter.newBuilder(writeStream.getName(), writeStream.getTableSchema(), client)
66+
.build()) {
67+
// Append 10 JSON objects to the stream.
68+
for (int i = 0; i < 10; i++) {
69+
// Create a JSON object that is compatible with the table schema.
70+
JSONObject record = new JSONObject();
71+
record.put("col1", String.format("record %03d", i));
72+
JSONArray jsonArr = new JSONArray();
73+
jsonArr.put(record);
74+
75+
// To detect duplicate records, pass the index as the record offset.
76+
// To disable deduplication, omit the offset or use WriteStream.Type.DEFAULT.
77+
ApiFuture<AppendRowsResponse> future = writer.append(jsonArr, i);
78+
AppendRowsResponse response = future.get();
79+
}
80+
}
81+
System.out.println("Appended records successfully.");
82+
} catch (ExecutionException e) {
83+
// If the wrapped exception is a StatusRuntimeException, check the state of the operation.
84+
// If the state is INTERNAL, CANCELLED, or ABORTED, you can retry. For more information, see:
85+
// https://grpc.github.io/grpc-java/javadoc/io/grpc/StatusRuntimeException.html
86+
System.out.println("Failed to append records. \n" + e.toString());
87+
}
88+
}
89+
}
90+
// [END bigquerystorage_jsonstreamwriter_committed]

‎samples/snippets/src/main/java/com/example/bigquerystorage/WritePendingStream.java‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquerystorage;
18+
19+
// [START bigquerystorage_jsonstreamwriter_pending]
20+
import com.google.api.core.ApiFuture;
21+
import com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse;
22+
import com.google.cloud.bigquery.storage.v1beta2.BatchCommitWriteStreamsRequest;
23+
import com.google.cloud.bigquery.storage.v1beta2.BatchCommitWriteStreamsResponse;
24+
import com.google.cloud.bigquery.storage.v1beta2.BigQueryWriteClient;
25+
import com.google.cloud.bigquery.storage.v1beta2.CreateWriteStreamRequest;
26+
import com.google.cloud.bigquery.storage.v1beta2.FinalizeWriteStreamResponse;
27+
import com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter;
28+
import com.google.cloud.bigquery.storage.v1beta2.TableName;
29+
import com.google.cloud.bigquery.storage.v1beta2.WriteStream;
30+
import com.google.protobuf.Descriptors.DescriptorValidationException;
31+
import java.io.IOException;
32+
import java.util.concurrent.ExecutionException;
33+
import org.json.JSONArray;
34+
import org.json.JSONObject;
35+
36+
public class WritePendingStream {
37+
38+
public static void runWritePendingStream()
39+
throws DescriptorValidationException, InterruptedException, IOException {
40+
// TODO(developer): Replace these variables before running the sample.
41+
String projectId = "MY_PROJECT_ID";
42+
String datasetName = "MY_DATASET_NAME";
43+
String tableName = "MY_TABLE_NAME";
44+
45+
writePendingStream(projectId, datasetName, tableName);
46+
}
47+
48+
public static void writePendingStream(String projectId, String datasetName, String tableName)
49+
throws DescriptorValidationException, InterruptedException, IOException {
50+
try (BigQueryWriteClient client = BigQueryWriteClient.create()) {
51+
// Initialize a write stream for the specified table.
52+
// For more information on WriteStream.Type, see:
53+
// https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1beta2/WriteStream.Type.html
54+
WriteStream stream = WriteStream.newBuilder().setType(WriteStream.Type.PENDING).build();
55+
TableName parentTable = TableName.of(projectId, datasetName, tableName);
56+
CreateWriteStreamRequest createWriteStreamRequest =
57+
CreateWriteStreamRequest.newBuilder()
58+
.setParent(parentTable.toString())
59+
.setWriteStream(stream)
60+
.build();
61+
WriteStream writeStream = client.createWriteStream(createWriteStreamRequest);
62+
63+
// Use the JSON stream writer to send records in JSON format.
64+
// For more information about JsonStreamWriter, see:
65+
// https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriter.html
66+
try (JsonStreamWriter writer =
67+
JsonStreamWriter.newBuilder(writeStream.getName(), writeStream.getTableSchema(), client)
68+
.build()) {
69+
// Append 10 JSON objects to the stream.
70+
for (int i = 0; i < 10; i++) {
71+
// Create a JSON object that is compatible with the table schema.
72+
JSONObject record = new JSONObject();
73+
record.put("col1", String.format("batch-record %03d", i));
74+
JSONArray jsonArr = new JSONArray();
75+
jsonArr.put(record);
76+
77+
ApiFuture<AppendRowsResponse> future = writer.append(jsonArr);
78+
AppendRowsResponse response = future.get();
79+
}
80+
FinalizeWriteStreamResponse finalizeResponse =
81+
client.finalizeWriteStream(writeStream.getName());
82+
System.out.println("Rows written: " + finalizeResponse.getRowCount());
83+
}
84+
85+
// Commit the streams.
86+
BatchCommitWriteStreamsRequest commitRequest =
87+
BatchCommitWriteStreamsRequest.newBuilder()
88+
.setParent(parentTable.toString())
89+
.addWriteStreams(writeStream.getName())
90+
.build();
91+
BatchCommitWriteStreamsResponse commitResponse =
92+
client.batchCommitWriteStreams(commitRequest);
93+
System.out.println("Appended and committed records successfully.");
94+
} catch (ExecutionException e) {
95+
// If the wrapped exception is a StatusRuntimeException, check the state of the operation.
96+
// If the state is INTERNAL, CANCELLED, or ABORTED, you can retry. For more information, see:
97+
// https://grpc.github.io/grpc-java/javadoc/io/grpc/StatusRuntimeException.html
98+
System.out.println(e);
99+
}
100+
}
101+
}
102+
// [END bigquerystorage_jsonstreamwriter_pending]

‎samples/snippets/src/main/java/com/example/bigquerystorage/WriteToDefaultStream.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquerystorage;
18+
19+
// [START bigquerystorage_jsonstreamwriter_default]
20+
import com.google.api.core.ApiFuture;
21+
import com.google.cloud.bigquery.BigQuery;
22+
import com.google.cloud.bigquery.BigQueryOptions;
23+
import com.google.cloud.bigquery.Schema;
24+
import com.google.cloud.bigquery.Table;
25+
import com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse;
26+
import com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter;
27+
import com.google.cloud.bigquery.storage.v1beta2.TableName;
28+
import com.google.protobuf.Descriptors.DescriptorValidationException;
29+
import java.io.IOException;
30+
import java.util.concurrent.ExecutionException;
31+
import org.json.JSONArray;
32+
import org.json.JSONObject;
33+
34+
public class WriteToDefaultStream {
35+
36+
public static void runWriteToDefaultStream()
37+
throws DescriptorValidationException, InterruptedException, IOException {
38+
// TODO(developer): Replace these variables before running the sample.
39+
String projectId = "MY_PROJECT_ID";
40+
String datasetName = "MY_DATASET_NAME";
41+
String tableName = "MY_TABLE_NAME";
42+
43+
writeToDefaultStream(projectId, datasetName, tableName);
44+
}
45+
46+
public static void writeToDefaultStream(String projectId, String datasetName, String tableName)
47+
throws DescriptorValidationException, InterruptedException, IOException {
48+
BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
49+
Table table = bigquery.getTable(datasetName, tableName);
50+
TableName parentTable = TableName.of(projectId, datasetName, tableName);
51+
Schema schema = table.getDefinition().getSchema();
52+
53+
// Use the JSON stream writer to send records in JSON format.
54+
// For more information about JsonStreamWriter, see:
55+
// https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1beta2/JstreamWriter.html
56+
try (JsonStreamWriter writer =
57+
JsonStreamWriter.newBuilder(parentTable.toString(), schema).createDefaultStream().build()) {
58+
// Append 10 JSON objects to the stream.
59+
for (int i = 0; i < 10; i++) {
60+
// Create a JSON object that is compatible with the table schema.
61+
JSONObject record = new JSONObject();
62+
record.put("col1", String.format("record %03d", i));
63+
JSONArray jsonArr = new JSONArray();
64+
jsonArr.put(record);
65+
66+
ApiFuture<AppendRowsResponse> future = writer.append(jsonArr);
67+
AppendRowsResponse response = future.get();
68+
}
69+
System.out.println("Appended records successfully.");
70+
} catch (ExecutionException e) {
71+
// If the wrapped exception is a StatusRuntimeException, check the state of the operation.
72+
// If the state is INTERNAL, CANCELLED, or ABORTED, you can retry. For more information, see:
73+
// https://grpc.github.io/grpc-java/javadoc/io/grpc/StatusRuntimeException.html
74+
System.out.println("Failed to append records. \n" + e.toString());
75+
}
76+
}
77+
}
78+
// [END bigquerystorage_jsonstreamwriter_default]

‎samples/snippets/src/test/java/com/example/bigquerystorage/WriteCommittedStreamIT.java‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquerystorage;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
import static junit.framework.TestCase.assertNotNull;
21+
22+
import com.google.cloud.bigquery.BigQuery;
23+
import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption;
24+
import com.google.cloud.bigquery.BigQueryOptions;
25+
import com.google.cloud.bigquery.DatasetId;
26+
import com.google.cloud.bigquery.DatasetInfo;
27+
import com.google.cloud.bigquery.Field;
28+
import com.google.cloud.bigquery.Schema;
29+
import com.google.cloud.bigquery.StandardSQLTypeName;
30+
import com.google.cloud.bigquery.StandardTableDefinition;
31+
import com.google.cloud.bigquery.TableId;
32+
import com.google.cloud.bigquery.TableInfo;
33+
import java.io.ByteArrayOutputStream;
34+
import java.io.PrintStream;
35+
import java.util.UUID;
36+
import org.junit.After;
37+
import org.junit.Before;
38+
import org.junit.BeforeClass;
39+
import org.junit.Test;
40+
import org.junit.runner.RunWith;
41+
import org.junit.runners.JUnit4;
42+
43+
@RunWith(JUnit4.class)
44+
public class WriteCommittedStreamIT {
45+
46+
private static final String GOOGLE_CLOUD_PROJECT = System.getenv("GOOGLE_CLOUD_PROJECT");
47+
48+
private ByteArrayOutputStream bout;
49+
private PrintStream out;
50+
private BigQuery bigquery;
51+
private String datasetName;
52+
private String tableName;
53+
54+
private static void requireEnvVar(String varName) {
55+
assertNotNull(
56+
"Environment variable " + varName + " is required to perform these tests.",
57+
System.getenv(varName));
58+
}
59+
60+
@BeforeClass
61+
public static void checkRequirements() {
62+
requireEnvVar("GOOGLE_CLOUD_PROJECT");
63+
}
64+
65+
@Before
66+
public void setUp() {
67+
bout = new ByteArrayOutputStream();
68+
out = new PrintStream(bout);
69+
System.setOut(out);
70+
71+
bigquery = BigQueryOptions.getDefaultInstance().getService();
72+
73+
// Create a new dataset and table for each test.
74+
datasetName = "WRITE_STREAM_TEST" + UUID.randomUUID().toString().substring(0, 8);
75+
tableName = "COMMITTED_STREAM_TEST" + UUID.randomUUID().toString().substring(0, 8);
76+
Schema schema = Schema.of(Field.of("col1", StandardSQLTypeName.STRING));
77+
bigquery.create(DatasetInfo.newBuilder(datasetName).build());
78+
TableInfo tableInfo =
79+
TableInfo.newBuilder(TableId.of(datasetName, tableName), StandardTableDefinition.of(schema))
80+
.build();
81+
bigquery.create(tableInfo);
82+
}
83+
84+
@After
85+
public void tearDown() {
86+
bigquery.delete(
87+
DatasetId.of(GOOGLE_CLOUD_PROJECT, datasetName), DatasetDeleteOption.deleteContents());
88+
System.setOut(null);
89+
}
90+
91+
@Test
92+
public void testWriteCommittedStream() throws Exception {
93+
WriteCommittedStream.writeCommittedStream(GOOGLE_CLOUD_PROJECT, datasetName, tableName);
94+
assertThat(bout.toString()).contains("Appended records successfully.");
95+
}
96+
}

0 commit comments

Comments
 (0)