Skip to content

Commit 64bdda8

Browse files
PhongChuongPhongChuong
authored
feat: Add new fields to copy job statistics (#3205)
* feat: Add new fields to copy job statistics Specifically, [copiedRows, copiedLogicalBytes] are added to CopyStatistics. * Fix issue where IT verified copied rows twice instead of copied logical bytes. * Add additional test to verified non-zero copy statistic case.
1 parent 7a24d3e commit 64bdda8

File tree

3 files changed

+109
-2
lines changed

3 files changed

+109
-2
lines changed

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java‎

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.api.services.bigquery.model.JobStatistics2;
2222
import com.google.api.services.bigquery.model.JobStatistics3;
2323
import com.google.api.services.bigquery.model.JobStatistics4;
24+
import com.google.api.services.bigquery.model.JobStatistics5;
2425
import com.google.api.services.bigquery.model.QueryParameter;
2526
import com.google.cloud.StringEnumType;
2627
import com.google.cloud.StringEnumValue;
@@ -51,14 +52,36 @@ public abstract class JobStatistics implements Serializable {
5152
/** A Google BigQuery Copy Job statistics. */
5253
public static class CopyStatistics extends JobStatistics {
5354

54-
private static final long serialVersionUID = 8218325588441660938L;
55+
private static final long serialVersionUID = 8218325588441660939L;
56+
57+
private final Long copiedLogicalBytes;
58+
59+
private final Long copiedRows;
5560

5661
static final class Builder extends JobStatistics.Builder<CopyStatistics, Builder> {
5762

63+
private Long copiedLogicalBytes;
64+
65+
private Long copiedRows;
66+
5867
private Builder() {}
5968

6069
private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) {
6170
super(statisticsPb);
71+
if (statisticsPb.getCopy() != null) {
72+
this.copiedLogicalBytes = statisticsPb.getCopy().getCopiedLogicalBytes();
73+
this.copiedRows = statisticsPb.getCopy().getCopiedRows();
74+
}
75+
}
76+
77+
Builder setCopiedLogicalBytes(long copiedLogicalBytes) {
78+
this.copiedLogicalBytes = copiedLogicalBytes;
79+
return self();
80+
}
81+
82+
Builder setCopiedRows(long copiedRows) {
83+
this.copiedRows = copiedRows;
84+
return self();
6285
}
6386

6487
@Override
@@ -69,6 +92,25 @@ CopyStatistics build() {
6992

7093
private CopyStatistics(Builder builder) {
7194
super(builder);
95+
this.copiedLogicalBytes = builder.copiedLogicalBytes;
96+
this.copiedRows = builder.copiedRows;
97+
}
98+
99+
/** Returns number of logical bytes copied to the destination table. */
100+
public Long getCopiedLogicalBytes() {
101+
return copiedLogicalBytes;
102+
}
103+
104+
/** Returns number of rows copied to the destination table. */
105+
public Long getCopiedRows() {
106+
return copiedRows;
107+
}
108+
109+
@Override
110+
ToStringHelper toStringHelper() {
111+
return super.toStringHelper()
112+
.add("copiedLogicalBytes", copiedLogicalBytes)
113+
.add("copiedRows", copiedRows);
72114
}
73115

74116
@Override
@@ -81,7 +123,15 @@ public final boolean equals(Object obj) {
81123

82124
@Override
83125
public final int hashCode() {
84-
return baseHashCode();
126+
return Objects.hash(baseHashCode(), copiedLogicalBytes, copiedRows);
127+
}
128+
129+
@Override
130+
com.google.api.services.bigquery.model.JobStatistics toPb() {
131+
JobStatistics5 copyStatisticsPb = new JobStatistics5();
132+
copyStatisticsPb.setCopiedLogicalBytes(copiedLogicalBytes);
133+
copyStatisticsPb.setCopiedRows(copiedRows);
134+
return super.toPb().setCopy(copyStatisticsPb);
85135
}
86136

87137
static Builder newBuilder() {

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,11 +85,15 @@ public class JobStatisticsTest {
8585
private static final Long SLOTMS = 12545L;
8686
private static final String TRANSACTION_ID = UUID.randomUUID().toString().substring(0, 8);
8787
private static final String SESSION_ID = UUID.randomUUID().toString().substring(0, 8);
88+
private static final Long COPIED_ROW = 1L;
89+
private static final Long COPIED_LOGICAL_BYTES = 2L;
8890
private static final CopyStatistics COPY_STATISTICS =
8991
CopyStatistics.newBuilder()
9092
.setCreationTimestamp(CREATION_TIME)
9193
.setEndTime(END_TIME)
9294
.setStartTime(START_TIME)
95+
.setCopiedRows(COPIED_ROW)
96+
.setCopiedLogicalBytes(COPIED_LOGICAL_BYTES)
9397
.build();
9498
private static final ExtractStatistics EXTRACT_STATISTICS =
9599
ExtractStatistics.newBuilder()
@@ -262,6 +266,12 @@ public void testBuilder() {
262266
assertEquals(FILE_COUNT, EXTRACT_STATISTICS.getDestinationUriFileCounts());
263267
assertEquals(INPUT_BYTES, EXTRACT_STATISTICS.getInputBytes());
264268

269+
assertEquals(CREATION_TIME, COPY_STATISTICS.getCreationTime());
270+
assertEquals(START_TIME, COPY_STATISTICS.getStartTime());
271+
assertEquals(END_TIME, COPY_STATISTICS.getEndTime());
272+
assertEquals(COPIED_LOGICAL_BYTES, COPY_STATISTICS.getCopiedLogicalBytes());
273+
assertEquals(COPIED_ROW, COPY_STATISTICS.getCopiedRows());
274+
265275
assertEquals(CREATION_TIME, LOAD_STATISTICS.getCreationTime());
266276
assertEquals(START_TIME, LOAD_STATISTICS.getStartTime());
267277
assertEquals(END_TIME, LOAD_STATISTICS.getEndTime());
@@ -334,6 +344,7 @@ public void testBuilder() {
334344
public void testToPbAndFromPb() {
335345
compareExtractStatistics(
336346
EXTRACT_STATISTICS, ExtractStatistics.fromPb(EXTRACT_STATISTICS.toPb()));
347+
compareCopyStatistics(COPY_STATISTICS, CopyStatistics.fromPb(COPY_STATISTICS.toPb()));
337348
compareLoadStatistics(LOAD_STATISTICS, LoadStatistics.fromPb(LOAD_STATISTICS.toPb()));
338349
compareQueryStatistics(QUERY_STATISTICS, QueryStatistics.fromPb(QUERY_STATISTICS.toPb()));
339350
compareStatistics(COPY_STATISTICS, CopyStatistics.fromPb(COPY_STATISTICS.toPb()));
@@ -400,6 +411,13 @@ private void compareExtractStatistics(ExtractStatistics expected, ExtractStatist
400411
assertEquals(expected.getInputBytes(), value.getInputBytes());
401412
}
402413

414+
private void compareCopyStatistics(CopyStatistics expected, CopyStatistics value) {
415+
assertEquals(expected, value);
416+
compareStatistics(expected, value);
417+
assertEquals(expected.getCopiedLogicalBytes(), value.getCopiedLogicalBytes());
418+
assertEquals(expected.getCopiedRows(), value.getCopiedRows());
419+
}
420+
403421
private void compareLoadStatistics(LoadStatistics expected, LoadStatistics value) {
404422
assertEquals(expected, value);
405423
compareStatistics(expected, value);

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@
9090
import com.google.cloud.bigquery.JobId;
9191
import com.google.cloud.bigquery.JobInfo;
9292
import com.google.cloud.bigquery.JobStatistics;
93+
import com.google.cloud.bigquery.JobStatistics.CopyStatistics;
9394
import com.google.cloud.bigquery.JobStatistics.ExtractStatistics;
9495
import com.google.cloud.bigquery.JobStatistics.LoadStatistics;
9596
import com.google.cloud.bigquery.JobStatistics.QueryStatistics;
@@ -5042,11 +5043,18 @@ public void testCopyJob() throws InterruptedException, TimeoutException {
50425043
assertNotNull(createdTable);
50435044
assertEquals(DATASET, createdTable.getTableId().getDataset());
50445045
assertEquals(sourceTableName, createdTable.getTableId().getTable());
5046+
50455047
TableId destinationTable = TableId.of(DATASET, destinationTableName);
50465048
CopyJobConfiguration configuration = CopyJobConfiguration.of(destinationTable, sourceTable);
50475049
Job remoteJob = bigquery.create(JobInfo.of(configuration));
50485050
remoteJob = remoteJob.waitFor();
50495051
assertNull(remoteJob.getStatus().getError());
5052+
5053+
CopyStatistics copyStatistics = remoteJob.getStatistics();
5054+
assertNotNull(copyStatistics);
5055+
assertEquals(0, copyStatistics.getCopiedRows().longValue());
5056+
assertEquals(0, copyStatistics.getCopiedLogicalBytes().longValue());
5057+
50505058
Table remoteTable = bigquery.getTable(DATASET, destinationTableName);
50515059
assertNotNull(remoteTable);
50525060
assertEquals(destinationTable.getDataset(), remoteTable.getTableId().getDataset());
@@ -5056,6 +5064,37 @@ public void testCopyJob() throws InterruptedException, TimeoutException {
50565064
assertTrue(remoteTable.delete());
50575065
}
50585066

5067+
@Test
5068+
public void testCopyJobStatistics() throws InterruptedException, TimeoutException {
5069+
String sourceTableName = "test_copy_job_statistics_source_table";
5070+
String destinationTableName = "test_copy_job_statistics_destination_table";
5071+
5072+
QueryJobConfiguration createTable =
5073+
QueryJobConfiguration.newBuilder(
5074+
String.format(
5075+
"CREATE TABLE %s AS SELECT num FROM UNNEST(GENERATE_ARRAY(0,5)) as num",
5076+
sourceTableName))
5077+
.setDefaultDataset(DatasetId.of(DATASET))
5078+
.setUseLegacySql(false)
5079+
.build();
5080+
bigquery.query(createTable);
5081+
5082+
// Copy the created table.
5083+
TableId sourceTable = TableId.of(DATASET, sourceTableName);
5084+
TableId destinationTable = TableId.of(DATASET, destinationTableName);
5085+
CopyJobConfiguration configuration = CopyJobConfiguration.of(destinationTable, sourceTable);
5086+
Job remoteJob = bigquery.create(JobInfo.of(configuration));
5087+
remoteJob = remoteJob.waitFor();
5088+
assertNull(remoteJob.getStatus().getError());
5089+
5090+
CopyStatistics copyStatistics = remoteJob.getStatistics();
5091+
assertNotNull(copyStatistics);
5092+
assertEquals(6, copyStatistics.getCopiedRows().longValue());
5093+
// Assert != 0 since copied logical bytes is may return non-deterministic value due to how the
5094+
// data is represented.
5095+
assertNotEquals(0, copyStatistics.getCopiedLogicalBytes().longValue());
5096+
}
5097+
50595098
@Test
50605099
public void testSnapshotTableCopyJob() throws InterruptedException {
50615100
String sourceTableName = "test_copy_job_base_table";

0 commit comments

Comments
 (0)