Skip to content

Commit ebbd8f5

Browse files
prash-miprash-mi
authored
feat: Add decimal target type (#2166)
* Add getDecimalTargetTypes and getDecimalTargetTypes * Add getDecimalTargetTypes and getDecimalTargetTypes * Modified unit test cases for DecimalTargetTypes * Adding getDecimalTargetTypes and getDecimalTargetTypes as clirr exceptions. Removed previous exceptions * Adding testInsertWithDecimalTargetTypes IT
1 parent c98d22b commit ebbd8f5

File tree

5 files changed

+84
-28
lines changed

5 files changed

+84
-28
lines changed

‎google-cloud-bigquery/clirr-ignored-differences.xml‎

Lines changed: 5 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,14 @@
22
<!-- see http://www.mojohaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
33
<differences>
44
<!-- TODO: REMOVE AFTER RELEASE -->
5-
<difference>
6-
<differenceType>7012</differenceType>
7-
<className>com/google/cloud/bigquery/BigQuery</className>
8-
<method>com.google.cloud.bigquery.Connection createConnection(com.google.cloud.bigquery.ConnectionSettings)</method>
9-
</difference>
10-
<difference>
11-
<differenceType>7012</differenceType>
12-
<className>com/google/cloud/bigquery/BigQuery</className>
13-
<method>com.google.cloud.bigquery.Connection createConnection()</method>
14-
</difference>
15-
<difference>
16-
<differenceType>7012</differenceType>
17-
<className>com/google/cloud/bigquery/spi/v2/BigQueryRpc</className>
18-
<method>com.google.api.services.bigquery.model.Job createJobForQuery(com.google.api.services.bigquery.model.Job)</method>
19-
</difference>
20-
<difference>
21-
<differenceType>7012</differenceType>
22-
<className>com/google/cloud/bigquery/spi/v2/BigQueryRpc</className>
23-
<method>com.google.api.services.bigquery.model.Job getQueryJob(java.lang.String, java.lang.String, java.lang.String)</method>
24-
</difference>
25-
<difference>
5+
<difference>
266
<differenceType>7012</differenceType>
27-
<className>com/google/cloud/bigquery/spi/v2/BigQueryRpc</className>
28-
<method>com.google.api.services.bigquery.model.GetQueryResultsResponse getQueryResultsWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Integer)</method>
7+
<className>com/google/cloud/bigquery/LoadConfiguration</className>
8+
<method>java.util.List getDecimalTargetTypes()</method>
299
</difference>
3010
<difference>
3111
<differenceType>7012</differenceType>
32-
<className>com/google/cloud/bigquery/spi/v2/BigQueryRpc</className>
33-
<method>com.google.api.services.bigquery.model.TableDataList listTableDataWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Integer, java.lang.String)</method>
12+
<className>com/google/cloud/bigquery/LoadConfiguration$Builder</className>
13+
<method>com.google.cloud.bigquery.LoadConfiguration$Builder setDecimalTargetTypes(java.util.List)</method>
3414
</difference>
3515
</differences>

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,17 @@ interface Builder {
125125
*/
126126
Builder setUseAvroLogicalTypes(Boolean useAvroLogicalTypes);
127127

128+
/**
129+
* Defines the list of possible SQL data types to which the source decimal values are converted.
130+
* This list and the precision and the scale parameters of the decimal field determine the
131+
* target type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in
132+
* the specified list and if it supports the precision and the scale. STRING supports all
133+
* precision and scale values.
134+
*
135+
* @param decimalTargetTypes decimalTargetType or {@code null} for none
136+
*/
137+
Builder setDecimalTargetTypes(List<String> decimalTargetTypes);
138+
128139
LoadConfiguration build();
129140
}
130141

@@ -214,6 +225,15 @@ interface Builder {
214225
/** Returns True/False. Indicates whether the logical type is interpreted. */
215226
Boolean getUseAvroLogicalTypes();
216227

228+
/**
229+
* Returns the list of possible SQL data types to which the source decimal values are converted.
230+
* This list and the precision and the scale parameters of the decimal field determine the target
231+
* type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in the
232+
* specified list and if it supports the precision and the scale. STRING supports all precision
233+
* and scale values.
234+
*/
235+
List<String> getDecimalTargetTypes();
236+
217237
/** Returns a builder for the load configuration object. */
218238
Builder toBuilder();
219239
}

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ public final class WriteChannelConfiguration implements LoadConfiguration, Seria
5555
private final Clustering clustering;
5656
private final Boolean useAvroLogicalTypes;
5757
private final Map<String, String> labels;
58+
private List<String> decimalTargetTypes;
5859

5960
public static final class Builder implements LoadConfiguration.Builder {
6061

@@ -73,6 +74,7 @@ public static final class Builder implements LoadConfiguration.Builder {
7374
private Clustering clustering;
7475
private Boolean useAvroLogicalTypes;
7576
private Map<String, String> labels;
77+
private List<String> decimalTargetTypes;
7678

7779
private Builder() {}
7880

@@ -93,6 +95,7 @@ private Builder(WriteChannelConfiguration writeChannelConfiguration) {
9395
this.clustering = writeChannelConfiguration.clustering;
9496
this.useAvroLogicalTypes = writeChannelConfiguration.useAvroLogicalTypes;
9597
this.labels = writeChannelConfiguration.labels;
98+
this.decimalTargetTypes = writeChannelConfiguration.decimalTargetTypes;
9699
}
97100

98101
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) {
@@ -169,6 +172,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
169172
if (configurationPb.getLabels() != null) {
170173
this.labels = configurationPb.getLabels();
171174
}
175+
if (loadConfigurationPb.getDecimalTargetTypes() != null) {
176+
this.decimalTargetTypes = loadConfigurationPb.getDecimalTargetTypes();
177+
}
172178
}
173179

174180
@Override
@@ -262,6 +268,12 @@ public Builder setLabels(Map<String, String> labels) {
262268
return this;
263269
}
264270

271+
@Override
272+
public Builder setDecimalTargetTypes(List<String> decimalTargetTypes) {
273+
this.decimalTargetTypes = decimalTargetTypes;
274+
return this;
275+
}
276+
265277
@Override
266278
public WriteChannelConfiguration build() {
267279
return new WriteChannelConfiguration(this);
@@ -284,6 +296,7 @@ protected WriteChannelConfiguration(Builder builder) {
284296
this.clustering = builder.clustering;
285297
this.useAvroLogicalTypes = builder.useAvroLogicalTypes;
286298
this.labels = builder.labels;
299+
this.decimalTargetTypes = builder.decimalTargetTypes;
287300
}
288301

289302
@Override
@@ -372,6 +385,11 @@ public Map<String, String> getLabels() {
372385
return labels;
373386
}
374387

388+
@Override
389+
public List<String> getDecimalTargetTypes() {
390+
return decimalTargetTypes;
391+
}
392+
375393
@Override
376394
public Builder toBuilder() {
377395
return new Builder(this);
@@ -393,7 +411,8 @@ MoreObjects.ToStringHelper toStringHelper() {
393411
.add("timePartitioning", timePartitioning)
394412
.add("clustering", clustering)
395413
.add("useAvroLogicalTypes", useAvroLogicalTypes)
396-
.add("labels", labels);
414+
.add("labels", labels)
415+
.add("decimalTargetTypes", decimalTargetTypes);
397416
}
398417

399418
@Override
@@ -424,7 +443,8 @@ public int hashCode() {
424443
timePartitioning,
425444
clustering,
426445
useAvroLogicalTypes,
427-
labels);
446+
labels,
447+
decimalTargetTypes);
428448
}
429449

430450
WriteChannelConfiguration setProjectId(String projectId) {
@@ -495,6 +515,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
495515
if (labels != null) {
496516
jobConfiguration.setLabels(labels);
497517
}
518+
if (decimalTargetTypes != null) {
519+
loadConfigurationPb.setDecimalTargetTypes(decimalTargetTypes);
520+
}
498521
jobConfiguration.setLoad(loadConfigurationPb);
499522
return jobConfiguration;
500523
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ public class WriteChannelConfigurationTest {
5959
Clustering.newBuilder().setFields(ImmutableList.of("Foo", "Bar")).build();
6060
private static final Map<String, String> LABELS =
6161
ImmutableMap.of("test-job-name", "test-write-channel");
62+
private static final List<String> DECIMAL_TARGET_TYPES =
63+
ImmutableList.of("NUMERIC", "BIGNUMERIC");
6264
private static final WriteChannelConfiguration LOAD_CONFIGURATION_CSV =
6365
WriteChannelConfiguration.newBuilder(TABLE_ID)
6466
.setCreateDisposition(CREATE_DISPOSITION)
@@ -73,6 +75,7 @@ public class WriteChannelConfigurationTest {
7375
.setTimePartitioning(TIME_PARTITIONING)
7476
.setClustering(CLUSTERING)
7577
.setLabels(LABELS)
78+
.setDecimalTargetTypes(DECIMAL_TARGET_TYPES)
7679
.build();
7780

7881
private static final DatastoreBackupOptions BACKUP_OPTIONS =
@@ -104,6 +107,7 @@ public class WriteChannelConfigurationTest {
104107
.setTimePartitioning(TIME_PARTITIONING)
105108
.setClustering(CLUSTERING)
106109
.setUseAvroLogicalTypes(USERAVROLOGICALTYPES)
110+
.setDecimalTargetTypes(DECIMAL_TARGET_TYPES)
107111
.build();
108112

109113
@Test
@@ -169,7 +173,8 @@ public void testBuilder() {
169173
.setMaxBadRecords(MAX_BAD_RECORDS)
170174
.setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS)
171175
.setSchema(TABLE_SCHEMA)
172-
.setAutodetect(AUTODETECT);
176+
.setAutodetect(AUTODETECT)
177+
.setDecimalTargetTypes(DECIMAL_TARGET_TYPES);
173178
WriteChannelConfiguration loadConfigurationCSV = builder.build();
174179
assertEquals(TABLE_ID, loadConfigurationCSV.getDestinationTable());
175180
assertEquals(CREATE_DISPOSITION, loadConfigurationCSV.getCreateDisposition());
@@ -182,6 +187,7 @@ public void testBuilder() {
182187
assertEquals(TABLE_SCHEMA, loadConfigurationCSV.getSchema());
183188
assertEquals(SCHEMA_UPDATE_OPTIONS, loadConfigurationCSV.getSchemaUpdateOptions());
184189
assertEquals(AUTODETECT, loadConfigurationCSV.getAutodetect());
190+
assertEquals(DECIMAL_TARGET_TYPES, loadConfigurationCSV.getDecimalTargetTypes());
185191
builder.setFormatOptions(BACKUP_OPTIONS);
186192
WriteChannelConfiguration loadConfigurationBackup = builder.build();
187193
assertEquals(BACKUP_OPTIONS, loadConfigurationBackup.getDatastoreBackupOptions());
@@ -225,5 +231,6 @@ private void compareLoadConfiguration(
225231
assertEquals(expected.getClustering(), value.getClustering());
226232
assertEquals(expected.getUseAvroLogicalTypes(), value.getUseAvroLogicalTypes());
227233
assertEquals(expected.getLabels(), value.getLabels());
234+
assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes());
228235
}
229236
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4374,6 +4374,32 @@ public void testInsertFromFileWithLabels()
43744374
assertTrue(bigquery.delete(tableId));
43754375
}
43764376

4377+
@Test
4378+
public void testInsertWithDecimalTargetTypes()
4379+
throws InterruptedException, IOException, TimeoutException {
4380+
String destinationTableName = "test_insert_from_file_table_with_decimal_target_type";
4381+
TableId tableId = TableId.of(DATASET, destinationTableName);
4382+
WriteChannelConfiguration configuration =
4383+
WriteChannelConfiguration.newBuilder(tableId)
4384+
.setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
4385+
.setAutodetect(true)
4386+
.setDecimalTargetTypes(ImmutableList.of("STRING", "NUMERIC", "BIGNUMERIC"))
4387+
.build();
4388+
TableDataWriteChannel channel = bigquery.writer(configuration);
4389+
try {
4390+
channel.write(ByteBuffer.wrap("foo".getBytes(StandardCharsets.UTF_8)));
4391+
} finally {
4392+
channel.close();
4393+
}
4394+
Job job = channel.getJob().waitFor();
4395+
LoadJobConfiguration jobConfiguration = job.getConfiguration();
4396+
assertNull(job.getStatus().getError());
4397+
assertEquals(
4398+
ImmutableList.of("STRING", "NUMERIC", "BIGNUMERIC"),
4399+
jobConfiguration.getDecimalTargetTypes());
4400+
assertTrue(bigquery.delete(tableId));
4401+
}
4402+
43774403
@Test
43784404
public void testLocation() throws Exception {
43794405
String location = "EU";

0 commit comments

Comments
 (0)