Skip to content

Commit 5dfd523

Browse files
VeronicaWassongcf-owl-bot[bot]
VeronicaWasson
and
gcf-owl-bot[bot]
authored
docs(samples): Add WriteAPI BUFFERED mode sample (#1338)
* docs(samples): Add WriteAPI BUFFERED mode sample * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Verify the rows were added to the table * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Veronica Wasson <3992422+MikeWasson@users.noreply.github.com> Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent 9370eb8 commit 5dfd523

File tree

3 files changed

+206
-0
lines changed

3 files changed

+206
-0
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-bigquerystora
112112
| Parallel Write Committed Stream | [source code](https://github.com/googleapis/java-bigquerystorage/blob/main/samples/snippets/src/main/java/com/example/bigquerystorage/ParallelWriteCommittedStream.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquerystorage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquerystorage/ParallelWriteCommittedStream.java) |
113113
| Storage Arrow Sample | [source code](https://github.com/googleapis/java-bigquerystorage/blob/main/samples/snippets/src/main/java/com/example/bigquerystorage/StorageArrowSample.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquerystorage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquerystorage/StorageArrowSample.java) |
114114
| Storage Sample | [source code](https://github.com/googleapis/java-bigquerystorage/blob/main/samples/snippets/src/main/java/com/example/bigquerystorage/StorageSample.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquerystorage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquerystorage/StorageSample.java) |
115+
| Write Buffered Stream | [source code](https://github.com/googleapis/java-bigquerystorage/blob/main/samples/snippets/src/main/java/com/example/bigquerystorage/WriteBufferedStream.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquerystorage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquerystorage/WriteBufferedStream.java) |
115116
| Write Committed Stream | [source code](https://github.com/googleapis/java-bigquerystorage/blob/main/samples/snippets/src/main/java/com/example/bigquerystorage/WriteCommittedStream.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquerystorage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquerystorage/WriteCommittedStream.java) |
116117
| Write Pending Stream | [source code](https://github.com/googleapis/java-bigquerystorage/blob/main/samples/snippets/src/main/java/com/example/bigquerystorage/WritePendingStream.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquerystorage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquerystorage/WritePendingStream.java) |
117118
| Write To Default Stream | [source code](https://github.com/googleapis/java-bigquerystorage/blob/main/samples/snippets/src/main/java/com/example/bigquerystorage/WriteToDefaultStream.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquerystorage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquerystorage/WriteToDefaultStream.java) |

‎samples/snippets/src/main/java/com/example/bigquerystorage/WriteBufferedStream.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquerystorage;
18+
19+
// [START bigquerystorage_jsonstreamwriter_buffered]
20+
import com.google.api.core.ApiFuture;
21+
import com.google.cloud.bigquery.storage.v1beta2.AppendRowsResponse;
22+
import com.google.cloud.bigquery.storage.v1beta2.BigQueryWriteClient;
23+
import com.google.cloud.bigquery.storage.v1beta2.CreateWriteStreamRequest;
24+
import com.google.cloud.bigquery.storage.v1beta2.FlushRowsRequest;
25+
import com.google.cloud.bigquery.storage.v1beta2.FlushRowsResponse;
26+
import com.google.cloud.bigquery.storage.v1beta2.JsonStreamWriter;
27+
import com.google.cloud.bigquery.storage.v1beta2.TableName;
28+
import com.google.cloud.bigquery.storage.v1beta2.WriteStream;
29+
import com.google.protobuf.Descriptors.DescriptorValidationException;
30+
import com.google.protobuf.Int64Value;
31+
import java.io.IOException;
32+
import java.util.concurrent.ExecutionException;
33+
import org.json.JSONArray;
34+
import org.json.JSONObject;
35+
36+
public class WriteBufferedStream {
37+
38+
public static void runWriteBufferedStream()
39+
throws DescriptorValidationException, InterruptedException, IOException {
40+
// TODO(developer): Replace these variables before running the sample.
41+
String projectId = "MY_PROJECT_ID";
42+
String datasetName = "MY_DATASET_NAME";
43+
String tableName = "MY_TABLE_NAME";
44+
45+
writeBufferedStream(projectId, datasetName, tableName);
46+
}
47+
48+
public static void writeBufferedStream(String projectId, String datasetName, String tableName)
49+
throws DescriptorValidationException, InterruptedException, IOException {
50+
try (BigQueryWriteClient client = BigQueryWriteClient.create()) {
51+
// Initialize a write stream for the specified table.
52+
// For more information on WriteStream.Type, see:
53+
// https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1beta2/WriteStream.Type.html
54+
WriteStream stream = WriteStream.newBuilder().setType(WriteStream.Type.BUFFERED).build();
55+
TableName parentTable = TableName.of(projectId, datasetName, tableName);
56+
CreateWriteStreamRequest createWriteStreamRequest =
57+
CreateWriteStreamRequest.newBuilder()
58+
.setParent(parentTable.toString())
59+
.setWriteStream(stream)
60+
.build();
61+
WriteStream writeStream = client.createWriteStream(createWriteStreamRequest);
62+
63+
// Use the JSON stream writer to send records in JSON format.
64+
// For more information about JsonStreamWriter, see:
65+
// https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriter.html
66+
try (JsonStreamWriter writer =
67+
JsonStreamWriter.newBuilder(writeStream.getName(), writeStream.getTableSchema())
68+
.build()) {
69+
// Write two batches to the stream, each with 10 JSON records.
70+
for (int i = 0; i < 2; i++) {
71+
JSONArray jsonArr = new JSONArray();
72+
for (int j = 0; j < 10; j++) {
73+
// Create a JSON object that is compatible with the table schema.
74+
JSONObject record = new JSONObject();
75+
record.put("col1", String.format("buffered-record %03d", i));
76+
jsonArr.put(record);
77+
}
78+
ApiFuture<AppendRowsResponse> future = writer.append(jsonArr);
79+
AppendRowsResponse response = future.get();
80+
}
81+
82+
// Flush the buffer.
83+
FlushRowsRequest flushRowsRequest =
84+
FlushRowsRequest.newBuilder()
85+
.setWriteStream(writeStream.getName())
86+
.setOffset(Int64Value.of(10 * 2 - 1)) // Advance the cursor to the latest record.
87+
.build();
88+
FlushRowsResponse flushRowsResponse = client.flushRows(flushRowsRequest);
89+
// You can continue to write to the stream after flushing the buffer.
90+
}
91+
System.out.println("Appended and committed records successfully.");
92+
} catch (ExecutionException e) {
93+
// If the wrapped exception is a StatusRuntimeException, check the state of the operation.
94+
// If the state is INTERNAL, CANCELLED, or ABORTED, you can retry. For more information, see:
95+
// https://grpc.github.io/grpc-java/javadoc/io/grpc/StatusRuntimeException.html
96+
System.out.println(e);
97+
}
98+
}
99+
}
100+
// [END bigquerystorage_jsonstreamwriter_buffered]

‎samples/snippets/src/test/java/com/example/bigquerystorage/WriteBufferedStreamIT.java‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquerystorage;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
import static junit.framework.TestCase.assertNotNull;
21+
22+
import com.google.cloud.bigquery.BigQuery;
23+
import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption;
24+
import com.google.cloud.bigquery.BigQueryOptions;
25+
import com.google.cloud.bigquery.DatasetId;
26+
import com.google.cloud.bigquery.DatasetInfo;
27+
import com.google.cloud.bigquery.Field;
28+
import com.google.cloud.bigquery.QueryJobConfiguration;
29+
import com.google.cloud.bigquery.Schema;
30+
import com.google.cloud.bigquery.StandardSQLTypeName;
31+
import com.google.cloud.bigquery.StandardTableDefinition;
32+
import com.google.cloud.bigquery.TableId;
33+
import com.google.cloud.bigquery.TableInfo;
34+
import com.google.cloud.bigquery.TableResult;
35+
import java.io.ByteArrayOutputStream;
36+
import java.io.PrintStream;
37+
import java.util.UUID;
38+
import org.junit.After;
39+
import org.junit.Before;
40+
import org.junit.BeforeClass;
41+
import org.junit.Test;
42+
import org.junit.runner.RunWith;
43+
import org.junit.runners.JUnit4;
44+
45+
@RunWith(JUnit4.class)
46+
public class WriteBufferedStreamIT {
47+
48+
private static final String GOOGLE_CLOUD_PROJECT = System.getenv("GOOGLE_CLOUD_PROJECT");
49+
50+
private ByteArrayOutputStream bout;
51+
private PrintStream out;
52+
private BigQuery bigquery;
53+
private String datasetName;
54+
private String tableName;
55+
56+
private static void requireEnvVar(String varName) {
57+
assertNotNull(
58+
"Environment variable " + varName + " is required to perform these tests.",
59+
System.getenv(varName));
60+
}
61+
62+
@BeforeClass
63+
public static void checkRequirements() {
64+
requireEnvVar("GOOGLE_CLOUD_PROJECT");
65+
}
66+
67+
@Before
68+
public void setUp() {
69+
bout = new ByteArrayOutputStream();
70+
out = new PrintStream(bout);
71+
System.setOut(out);
72+
73+
bigquery = BigQueryOptions.getDefaultInstance().getService();
74+
75+
// Create a new dataset and table for each test.
76+
datasetName = "WRITE_STREAM_TEST" + UUID.randomUUID().toString().substring(0, 8);
77+
tableName = "PENDING_STREAM_TEST" + UUID.randomUUID().toString().substring(0, 8);
78+
Schema schema = Schema.of(Field.of("col1", StandardSQLTypeName.STRING));
79+
bigquery.create(DatasetInfo.newBuilder(datasetName).build());
80+
TableInfo tableInfo =
81+
TableInfo.newBuilder(TableId.of(datasetName, tableName), StandardTableDefinition.of(schema))
82+
.build();
83+
bigquery.create(tableInfo);
84+
}
85+
86+
@After
87+
public void tearDown() {
88+
bigquery.delete(
89+
DatasetId.of(GOOGLE_CLOUD_PROJECT, datasetName), DatasetDeleteOption.deleteContents());
90+
System.setOut(null);
91+
}
92+
93+
@Test
94+
public void testWriteBufferedStream() throws Exception {
95+
WriteBufferedStream.writeBufferedStream(GOOGLE_CLOUD_PROJECT, datasetName, tableName);
96+
assertThat(bout.toString()).contains("Appended and committed records successfully.");
97+
98+
// Verify that the records are visible in the table.
99+
String query = "SELECT * FROM " + tableName;
100+
QueryJobConfiguration queryConfig =
101+
QueryJobConfiguration.newBuilder(query).setDefaultDataset(datasetName).build();
102+
TableResult result = bigquery.query(queryConfig);
103+
assertThat(result.getTotalRows()).isEqualTo(20);
104+
}
105+
}

0 commit comments

Comments
 (0)