Skip to content

Commit 2ae69b6

Browse files
yirutangyirutang
authored
fix: remove stream ttl in client library, since there is no very clear TTL defined. (#627)
* fix: Due to backend issues resolved, we no longer need to wait for 5 seconds between reconnection * fix: Remove the unnecessary GetWriteStream call to check stream TTL, since the stream TTL definition becomes complicate. * add ignore method diff
1 parent 6347c13 commit 2ae69b6

File tree

8 files changed

+36
-215
lines changed

8 files changed

+36
-215
lines changed

‎google-cloud-bigquerystorage/clirr-ignored-differences.xml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,14 @@
3333
<className>com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptor</className>
3434
<method>com.google.protobuf.Descriptors$Descriptor ConvertBQTableSchemaToProtoDescriptor(com.google.cloud.bigquery.storage.v1alpha2.Table$TableSchema)</method>
3535
</difference>
36-
</differences>
36+
<difference>
37+
<className>com/google/cloud/bigquery/storage/v1alpha2/JsonStreamWriter</className>
38+
<differenceType>7002</differenceType>
39+
<method>java.lang.Boolean expired()</method>
40+
</difference>
41+
<difference>
42+
<className>com/google/cloud/bigquery/storage/v1alpha2/StreamWriter</className>
43+
<differenceType>7002</differenceType>
44+
<method>java.lang.Boolean expired()</method>
45+
</difference>
46+
</differences>

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/JsonStreamWriter.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -260,11 +260,6 @@ public void close() {
260260
this.streamWriter.close();
261261
}
262262

263-
/** Returns if a stream has expired. */
264-
public Boolean expired() {
265-
return this.streamWriter.expired();
266-
}
267-
268263
private class JsonStreamWriterOnSchemaUpdateRunnable extends OnSchemaUpdateRunnable {
269264
private JsonStreamWriter jsonStreamWriter;
270265
/**

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/JsonWriterCache.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -113,11 +113,7 @@ public JsonStreamWriter getTableWriter(String tableName)
113113
synchronized (this) {
114114
writer = jsonWriterCache.getIfPresent(tableName);
115115
if (writer != null) {
116-
if (!writer.expired()) {
117-
return writer;
118-
} else {
119-
writer.close();
120-
}
116+
return writer;
121117
}
122118
writeStream = CreateNewWriteStream(tableName);
123119
writer = CreateNewWriter(writeStream);

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/StreamWriter.java‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@
5757
import java.util.regex.Matcher;
5858
import java.util.regex.Pattern;
5959
import org.threeten.bp.Duration;
60-
import org.threeten.bp.Instant;
6160

6261
/**
6362
* A BigQuery Stream Writer that can be used to write data into BigQuery Table.
@@ -118,9 +117,6 @@ public class StreamWriter implements AutoCloseable {
118117
private final AtomicBoolean activeAlarm;
119118
private ScheduledFuture<?> currentAlarmFuture;
120119

121-
private Instant createTime;
122-
private Duration streamTTL = Duration.ofDays(1);
123-
124120
private Integer currentRetries = 0;
125121

126122
// Used for schema updates
@@ -182,19 +178,6 @@ private StreamWriter(Builder builder)
182178
}
183179

184180
refreshAppend();
185-
Stream.WriteStream stream =
186-
stub.getWriteStream(Storage.GetWriteStreamRequest.newBuilder().setName(streamName).build());
187-
createTime =
188-
Instant.ofEpochSecond(
189-
stream.getCreateTime().getSeconds(), stream.getCreateTime().getNanos());
190-
if (stream.getType() == Stream.WriteStream.Type.PENDING && stream.hasCommitTime()) {
191-
throw new IllegalStateException(
192-
"Cannot write to a stream that is already committed: " + streamName);
193-
}
194-
if (createTime.plus(streamTTL).compareTo(Instant.now()) < 0) {
195-
throw new IllegalStateException(
196-
"Cannot write to a stream that is already expired: " + streamName);
197-
}
198181
}
199182

200183
/** Stream name we are writing to. */
@@ -212,11 +195,6 @@ OnSchemaUpdateRunnable getOnSchemaUpdateRunnable() {
212195
return this.onSchemaUpdateRunnable;
213196
}
214197

215-
/** Returns if a stream has expired. */
216-
public Boolean expired() {
217-
return createTime.plus(streamTTL).compareTo(Instant.now()) < 0;
218-
}
219-
220198
private void setException(Throwable t) {
221199
exceptionLock.lock();
222200
if (this.streamException == null) {

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/WriterCache.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -135,11 +135,7 @@ public StreamWriter getTableWriter(String tableName, Descriptor userSchema)
135135
if (tableEntry != null) {
136136
writer = tableEntry.getIfPresent(userSchema);
137137
if (writer != null) {
138-
if (!writer.expired()) {
139-
return writer;
140-
} else {
141-
writer.close();
142-
}
138+
return writer;
143139
}
144140
compat.check(tableName, userSchema);
145141
streamName = CreateNewStream(tableName);

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/DirectWriterTest.java‎

Lines changed: 15 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import com.google.cloud.bigquery.storage.v1alpha2.Storage.AppendRowsRequest;
2929
import com.google.common.collect.Sets;
3030
import com.google.protobuf.AbstractMessage;
31-
import com.google.protobuf.Timestamp;
3231
import java.io.IOException;
3332
import java.util.Arrays;
3433
import java.util.List;
@@ -50,7 +49,6 @@
5049
import org.junit.runners.JUnit4;
5150
import org.mockito.Mock;
5251
import org.mockito.MockitoAnnotations;
53-
import org.threeten.bp.Instant;
5452

5553
@RunWith(JUnit4.class)
5654
public class DirectWriterTest {
@@ -115,18 +113,6 @@ void WriterCreationResponseMock(String testStreamName, Set<Long> responseOffsets
115113
Stream.WriteStream.newBuilder().setName(testStreamName).build();
116114
mockBigQueryWrite.addResponse(expectedResponse);
117115

118-
// Response from GetWriteStream
119-
Instant time = Instant.now();
120-
Timestamp timestamp =
121-
Timestamp.newBuilder().setSeconds(time.getEpochSecond()).setNanos(time.getNano()).build();
122-
Stream.WriteStream expectedResponse2 =
123-
Stream.WriteStream.newBuilder()
124-
.setName(testStreamName)
125-
.setType(Stream.WriteStream.Type.COMMITTED)
126-
.setCreateTime(timestamp)
127-
.build();
128-
mockBigQueryWrite.addResponse(expectedResponse2);
129-
130116
for (Long offset : responseOffsets) {
131117
Storage.AppendRowsResponse response =
132118
Storage.AppendRowsResponse.newBuilder().setOffset(offset).build();
@@ -144,18 +130,6 @@ void JsonWriterCreationResponseMock(String testStreamName, Set<Long> responseOff
144130
.build();
145131
mockBigQueryWrite.addResponse(expectedResponse);
146132

147-
// Response from GetWriteStream
148-
Instant time = Instant.now();
149-
Timestamp timestamp =
150-
Timestamp.newBuilder().setSeconds(time.getEpochSecond()).setNanos(time.getNano()).build();
151-
Stream.WriteStream expectedResponse2 =
152-
Stream.WriteStream.newBuilder()
153-
.setName(testStreamName)
154-
.setType(Stream.WriteStream.Type.COMMITTED)
155-
.setCreateTime(timestamp)
156-
.build();
157-
mockBigQueryWrite.addResponse(expectedResponse2);
158-
159133
for (Long offset : responseOffsets) {
160134
Storage.AppendRowsResponse response =
161135
Storage.AppendRowsResponse.newBuilder().setOffset(offset).build();
@@ -183,19 +157,15 @@ public void testJsonWriteSuccess() throws Exception {
183157
ApiFuture<Long> ret = DirectWriter.append(TEST_TABLE, jsonArr);
184158
assertEquals(Long.valueOf(0L), ret.get());
185159
List<AbstractMessage> actualRequests = mockBigQueryWrite.getRequests();
186-
Assert.assertEquals(3, actualRequests.size());
160+
Assert.assertEquals(2, actualRequests.size());
187161
assertEquals(
188162
TEST_TABLE, ((Storage.CreateWriteStreamRequest) actualRequests.get(0)).getParent());
189-
assertEquals(
190-
Stream.WriteStream.Type.COMMITTED,
191-
((Storage.CreateWriteStreamRequest) actualRequests.get(0)).getWriteStream().getType());
192-
assertEquals(TEST_STREAM, ((Storage.GetWriteStreamRequest) actualRequests.get(1)).getName());
193163
assertEquals(
194164
m1.toByteString(),
195-
((AppendRowsRequest) actualRequests.get(2)).getProtoRows().getRows().getSerializedRows(0));
165+
((AppendRowsRequest) actualRequests.get(1)).getProtoRows().getRows().getSerializedRows(0));
196166
assertEquals(
197167
m2.toByteString(),
198-
((AppendRowsRequest) actualRequests.get(2)).getProtoRows().getRows().getSerializedRows(1));
168+
((AppendRowsRequest) actualRequests.get(1)).getProtoRows().getRows().getSerializedRows(1));
199169

200170
Storage.AppendRowsResponse response =
201171
Storage.AppendRowsResponse.newBuilder().setOffset(2).build();
@@ -205,7 +175,7 @@ public void testJsonWriteSuccess() throws Exception {
205175
assertEquals(Long.valueOf(2L), ret.get());
206176
assertEquals(
207177
m1.toByteString(),
208-
((AppendRowsRequest) actualRequests.get(3)).getProtoRows().getRows().getSerializedRows(0));
178+
((AppendRowsRequest) actualRequests.get(2)).getProtoRows().getRows().getSerializedRows(0));
209179
DirectWriter.clearCache();
210180
}
211181

@@ -220,13 +190,9 @@ public void testProtobufWriteSuccess() throws Exception {
220190
verify(schemaCheck).check(TEST_TABLE, FooType.getDescriptor());
221191
assertEquals(Long.valueOf(0L), ret.get());
222192
List<AbstractMessage> actualRequests = mockBigQueryWrite.getRequests();
223-
Assert.assertEquals(3, actualRequests.size());
193+
Assert.assertEquals(2, actualRequests.size());
224194
assertEquals(
225195
TEST_TABLE, ((Storage.CreateWriteStreamRequest) actualRequests.get(0)).getParent());
226-
assertEquals(
227-
Stream.WriteStream.Type.COMMITTED,
228-
((Storage.CreateWriteStreamRequest) actualRequests.get(0)).getWriteStream().getType());
229-
assertEquals(TEST_STREAM, ((Storage.GetWriteStreamRequest) actualRequests.get(1)).getName());
230196

231197
Storage.AppendRowsRequest.ProtoData.Builder dataBuilder =
232198
Storage.AppendRowsRequest.ProtoData.newBuilder();
@@ -241,7 +207,7 @@ public void testProtobufWriteSuccess() throws Exception {
241207
.setWriteStream(TEST_STREAM)
242208
.setProtoRows(dataBuilder.build())
243209
.build();
244-
assertEquals(expectRequest.toString(), actualRequests.get(2).toString());
210+
assertEquals(expectRequest.toString(), actualRequests.get(1).toString());
245211

246212
Storage.AppendRowsResponse response =
247213
Storage.AppendRowsResponse.newBuilder().setOffset(2).build();
@@ -254,7 +220,7 @@ public void testProtobufWriteSuccess() throws Exception {
254220
ProtoBufProto.ProtoRows.newBuilder().addSerializedRows(m1.toByteString()).build());
255221
expectRequest =
256222
Storage.AppendRowsRequest.newBuilder().setProtoRows(dataBuilder.build()).build();
257-
assertEquals(expectRequest.toString(), actualRequests.get(3).toString());
223+
assertEquals(expectRequest.toString(), actualRequests.get(2).toString());
258224

259225
// Write with a different schema.
260226
WriterCreationResponseMock(TEST_STREAM_2, Sets.newHashSet(Long.valueOf(0L)));
@@ -271,14 +237,10 @@ public void testProtobufWriteSuccess() throws Exception {
271237
.setWriteStream(TEST_STREAM_2)
272238
.setProtoRows(dataBuilder.build())
273239
.build();
274-
Assert.assertEquals(7, actualRequests.size());
240+
Assert.assertEquals(5, actualRequests.size());
275241
assertEquals(
276-
TEST_TABLE, ((Storage.CreateWriteStreamRequest) actualRequests.get(4)).getParent());
277-
assertEquals(
278-
Stream.WriteStream.Type.COMMITTED,
279-
((Storage.CreateWriteStreamRequest) actualRequests.get(4)).getWriteStream().getType());
280-
assertEquals(TEST_STREAM_2, ((Storage.GetWriteStreamRequest) actualRequests.get(5)).getName());
281-
assertEquals(expectRequest.toString(), actualRequests.get(6).toString());
242+
TEST_TABLE, ((Storage.CreateWriteStreamRequest) actualRequests.get(3)).getParent());
243+
assertEquals(expectRequest.toString(), actualRequests.get(4).toString());
282244

283245
DirectWriter.clearCache();
284246
}
@@ -433,13 +395,9 @@ public void testJsonProtobufWrite() throws Exception {
433395
verify(schemaCheck).check(TEST_TABLE, FooType.getDescriptor());
434396
assertEquals(Long.valueOf(0L), ret.get());
435397
List<AbstractMessage> actualRequests = mockBigQueryWrite.getRequests();
436-
Assert.assertEquals(3, actualRequests.size());
398+
Assert.assertEquals(2, actualRequests.size());
437399
assertEquals(
438400
TEST_TABLE, ((Storage.CreateWriteStreamRequest) actualRequests.get(0)).getParent());
439-
assertEquals(
440-
Stream.WriteStream.Type.COMMITTED,
441-
((Storage.CreateWriteStreamRequest) actualRequests.get(0)).getWriteStream().getType());
442-
assertEquals(TEST_STREAM, ((Storage.GetWriteStreamRequest) actualRequests.get(1)).getName());
443401

444402
Storage.AppendRowsRequest.ProtoData.Builder dataBuilder =
445403
Storage.AppendRowsRequest.ProtoData.newBuilder();
@@ -454,25 +412,19 @@ public void testJsonProtobufWrite() throws Exception {
454412
.setWriteStream(TEST_STREAM)
455413
.setProtoRows(dataBuilder.build())
456414
.build();
457-
assertEquals(expectRequest.toString(), actualRequests.get(2).toString());
415+
assertEquals(expectRequest.toString(), actualRequests.get(1).toString());
458416

459417
JsonWriterCreationResponseMock(TEST_STREAM, Sets.newHashSet(Long.valueOf(0L)));
460418
ret = DirectWriter.append(TEST_TABLE, jsonArr);
461419
assertEquals(Long.valueOf(0L), ret.get());
462420
actualRequests = mockBigQueryWrite.getRequests();
463-
Assert.assertEquals(6, actualRequests.size());
464-
assertEquals(
465-
TEST_TABLE, ((Storage.CreateWriteStreamRequest) actualRequests.get(3)).getParent());
466-
assertEquals(
467-
Stream.WriteStream.Type.COMMITTED,
468-
((Storage.CreateWriteStreamRequest) actualRequests.get(3)).getWriteStream().getType());
469-
assertEquals(TEST_STREAM, ((Storage.GetWriteStreamRequest) actualRequests.get(4)).getName());
421+
Assert.assertEquals(4, actualRequests.size());
470422
assertEquals(
471423
m1.toByteString(),
472-
((AppendRowsRequest) actualRequests.get(5)).getProtoRows().getRows().getSerializedRows(0));
424+
((AppendRowsRequest) actualRequests.get(3)).getProtoRows().getRows().getSerializedRows(0));
473425
assertEquals(
474426
m2.toByteString(),
475-
((AppendRowsRequest) actualRequests.get(5)).getProtoRows().getRows().getSerializedRows(1));
427+
((AppendRowsRequest) actualRequests.get(3)).getProtoRows().getRows().getSerializedRows(1));
476428

477429
DirectWriter.clearCache();
478430
}

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/JsonWriterCacheTest.java‎

Lines changed: 3 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import com.google.api.gax.grpc.testing.MockServiceHelper;
2727
import com.google.cloud.bigquery.storage.test.Test.*;
2828
import com.google.protobuf.AbstractMessage;
29-
import com.google.protobuf.Timestamp;
3029
import java.io.IOException;
3130
import java.util.Arrays;
3231
import java.util.List;
@@ -40,8 +39,6 @@
4039
import org.junit.runners.JUnit4;
4140
import org.mockito.Mock;
4241
import org.mockito.MockitoAnnotations;
43-
import org.threeten.bp.Instant;
44-
import org.threeten.bp.temporal.ChronoUnit;
4542

4643
@RunWith(JUnit4.class)
4744
public class JsonWriterCacheTest {
@@ -108,18 +105,6 @@ void WriterCreationResponseMock(String testStreamName) {
108105
.setTableSchema(TABLE_SCHEMA)
109106
.build();
110107
mockBigQueryWrite.addResponse(expectedResponse);
111-
112-
// Response from GetWriteStream
113-
Instant time = Instant.now();
114-
Timestamp timestamp =
115-
Timestamp.newBuilder().setSeconds(time.getEpochSecond()).setNanos(time.getNano()).build();
116-
Stream.WriteStream expectedResponse2 =
117-
Stream.WriteStream.newBuilder()
118-
.setName(testStreamName)
119-
.setType(Stream.WriteStream.Type.COMMITTED)
120-
.setCreateTime(timestamp)
121-
.build();
122-
mockBigQueryWrite.addResponse(expectedResponse2);
123108
}
124109

125110
@After
@@ -144,50 +129,15 @@ public void testCreateNewWriter() throws Exception {
144129
WriterCreationResponseMock(TEST_STREAM);
145130
JsonStreamWriter writer = cache.getTableWriter(TEST_TABLE);
146131
List<AbstractMessage> actualRequests = mockBigQueryWrite.getRequests();
147-
assertEquals(2, actualRequests.size());
132+
assertEquals(1, actualRequests.size());
148133
assertEquals(
149134
TEST_TABLE, ((Storage.CreateWriteStreamRequest) actualRequests.get(0)).getParent());
150-
assertEquals(
151-
Stream.WriteStream.Type.COMMITTED,
152-
((Storage.CreateWriteStreamRequest) actualRequests.get(0)).getWriteStream().getType());
153-
assertEquals(TEST_STREAM, ((Storage.GetWriteStreamRequest) actualRequests.get(1)).getName());
154135

155136
assertEquals(TEST_STREAM, writer.getStreamName());
156137
assertEquals(1, cache.cachedTableCount());
157138
cache.clear();
158139
}
159140

160-
@Test
161-
public void testWriterExpired() throws Exception {
162-
JsonWriterCache cache = JsonWriterCache.getTestInstance(client, 10);
163-
// Response from CreateWriteStream
164-
Stream.WriteStream expectedResponse =
165-
Stream.WriteStream.newBuilder().setName(TEST_STREAM).build();
166-
mockBigQueryWrite.addResponse(expectedResponse);
167-
168-
// Response from GetWriteStream
169-
Instant time = Instant.now().minus(2, ChronoUnit.DAYS);
170-
Timestamp timestamp =
171-
Timestamp.newBuilder().setSeconds(time.getEpochSecond()).setNanos(time.getNano()).build();
172-
Stream.WriteStream expectedResponse2 =
173-
Stream.WriteStream.newBuilder()
174-
.setName(TEST_STREAM)
175-
.setType(Stream.WriteStream.Type.COMMITTED)
176-
.setCreateTime(timestamp)
177-
.build();
178-
mockBigQueryWrite.addResponse(expectedResponse2);
179-
180-
try {
181-
JsonStreamWriter writer = cache.getTableWriter(TEST_TABLE);
182-
fail("Should fail");
183-
} catch (IllegalStateException e) {
184-
assertEquals(
185-
"Cannot write to a stream that is already expired: projects/p/datasets/d/tables/t/streams/s",
186-
e.getMessage());
187-
}
188-
cache.clear();
189-
}
190-
191141
@Test
192142
public void testWriterWithDifferentTable() throws Exception {
193143
JsonWriterCache cache = JsonWriterCache.getTestInstance(client, 2);
@@ -197,14 +147,11 @@ public void testWriterWithDifferentTable() throws Exception {
197147
JsonStreamWriter writer2 = cache.getTableWriter(TEST_TABLE_2);
198148

199149
List<AbstractMessage> actualRequests = mockBigQueryWrite.getRequests();
200-
assertEquals(4, actualRequests.size());
150+
assertEquals(2, actualRequests.size());
201151
assertEquals(
202152
TEST_TABLE, ((Storage.CreateWriteStreamRequest) actualRequests.get(0)).getParent());
203-
assertEquals(TEST_STREAM, ((Storage.GetWriteStreamRequest) actualRequests.get(1)).getName());
204153
assertEquals(
205-
TEST_TABLE_2, ((Storage.CreateWriteStreamRequest) actualRequests.get(2)).getParent());
206-
Assert.assertEquals(
207-
TEST_STREAM_21, ((Storage.GetWriteStreamRequest) actualRequests.get(3)).getName());
154+
TEST_TABLE_2, ((Storage.CreateWriteStreamRequest) actualRequests.get(1)).getParent());
208155
assertEquals(TEST_STREAM, writer1.getStreamName());
209156
assertEquals(TEST_STREAM_21, writer2.getStreamName());
210157
assertEquals(2, cache.cachedTableCount());

0 commit comments

Comments
 (0)