Skip to content

Commit 200da6d

Browse files
allenc3allenc3
authored
feat: Adding JsonToProtoMessage.java (#400)
* feat: Finished JsonToProtoMessage, adding in tests * Added more test cases * Draft version of JsonToProtoMessage * Fixed float types in jsonTest.proto, and added a test that checks if repeated types are optional * Fix according to PR (added case insensitive feature, added support to int32 for BQDate) * Update JsonToProtoMessage to fix allowUnknownField behavior * Fix according to PR, used case insensitive treeset for allowUnknownFields check. * Fixed according to PR * Fix test errors * Change loopiing all proto fields to looping all json fields since # of json fields <= # of proto fields * Remove unuse variable * Remove unuse variable * Removed unnecessary set * Fixed according to PR, and added proto message empty test case. * Pushed for loop to be outside of the try catches to remove unnecessary for loops. * Made tests based on self created protos * Fix jsonTest.proto * Changed throwing empty JsonException to checking for the type of the values first. This prevents catching unexpected JsonExceptions and blocking the actual error message. * Fix by specifying version in parent pom only and remove * imports for JsonToProtoMessageTests * Add in parent pom * Refactor parent pom
1 parent 112224b commit 200da6d

File tree

5 files changed

+1158
-0
lines changed

5 files changed

+1158
-0
lines changed

‎google-cloud-bigquerystorage/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,11 @@
108108
<groupId>org.apache.commons</groupId>
109109
<artifactId>commons-lang3</artifactId>
110110
</dependency>
111+
<dependency>
112+
<groupId>org.json</groupId>
113+
<artifactId>json</artifactId>
114+
</dependency>
115+
111116

112117
<!-- Test dependencies -->
113118
<dependency>

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/JsonToProtoMessage.java‎

Lines changed: 323 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,323 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigquery.storage.v1alpha2;
17+
18+
import com.google.common.base.Preconditions;
19+
import com.google.common.collect.ImmutableMap;
20+
import com.google.protobuf.Descriptors.Descriptor;
21+
import com.google.protobuf.Descriptors.FieldDescriptor;
22+
import com.google.protobuf.DynamicMessage;
23+
import com.google.protobuf.Message;
24+
import com.google.protobuf.UninitializedMessageException;
25+
import org.json.JSONArray;
26+
import org.json.JSONException;
27+
import org.json.JSONObject;
28+
29+
/**
30+
* Converts Json data to protocol buffer messages given the protocol buffer descriptor. The protobuf
31+
* descriptor must have all fields lowercased.
32+
*/
33+
public class JsonToProtoMessage {
34+
private static ImmutableMap<FieldDescriptor.Type, String> FieldTypeToDebugMessage =
35+
new ImmutableMap.Builder<FieldDescriptor.Type, String>()
36+
.put(FieldDescriptor.Type.BOOL, "boolean")
37+
.put(FieldDescriptor.Type.BYTES, "string")
38+
.put(FieldDescriptor.Type.INT32, "int32")
39+
.put(FieldDescriptor.Type.DOUBLE, "double")
40+
.put(FieldDescriptor.Type.INT64, "int64")
41+
.put(FieldDescriptor.Type.STRING, "string")
42+
.put(FieldDescriptor.Type.MESSAGE, "object")
43+
.build();
44+
45+
/**
46+
* Converts Json data to protocol buffer messages given the protocol buffer descriptor.
47+
*
48+
* @param protoSchema
49+
* @param json
50+
* @param allowUnknownFields Ignores unknown JSON fields.
51+
* @throws IllegalArgumentException when JSON data is not compatible with proto descriptor.
52+
*/
53+
public static DynamicMessage convertJsonToProtoMessage(
54+
Descriptor protoSchema, JSONObject json, boolean allowUnknownFields)
55+
throws IllegalArgumentException {
56+
Preconditions.checkNotNull(json, "JSONObject is null.");
57+
Preconditions.checkNotNull(protoSchema, "Protobuf descriptor is null.");
58+
Preconditions.checkState(json.length() != 0, "JSONObject is empty.");
59+
60+
return convertJsonToProtoMessageImpl(
61+
protoSchema, json, "root", /*topLevel=*/ true, allowUnknownFields);
62+
}
63+
64+
/**
65+
* Converts Json data to protocol buffer messages given the protocol buffer descriptor.
66+
*
67+
* @param protoSchema
68+
* @param json
69+
* @param jsonScope Debugging purposes
70+
* @param allowUnknownFields Ignores unknown JSON fields.
71+
* @param topLevel checks if root level has any matching fields.
72+
* @throws IllegalArgumentException when JSON data is not compatible with proto descriptor.
73+
*/
74+
private static DynamicMessage convertJsonToProtoMessageImpl(
75+
Descriptor protoSchema,
76+
JSONObject json,
77+
String jsonScope,
78+
boolean topLevel,
79+
boolean allowUnknownFields)
80+
throws IllegalArgumentException {
81+
82+
DynamicMessage.Builder protoMsg = DynamicMessage.newBuilder(protoSchema);
83+
String[] jsonNames = JSONObject.getNames(json);
84+
if (jsonNames == null) {
85+
return protoMsg.build();
86+
}
87+
int matchedFields = 0;
88+
for (int i = 0; i < jsonNames.length; i++) {
89+
String jsonName = jsonNames[i];
90+
// We want lowercase here to support case-insensitive data writes.
91+
// The protobuf descriptor that is used is assumed to have all lowercased fields
92+
String jsonLowercaseName = jsonName.toLowerCase();
93+
String currentScope = jsonScope + "." + jsonName;
94+
FieldDescriptor field = protoSchema.findFieldByName(jsonLowercaseName);
95+
if (field == null) {
96+
if (!allowUnknownFields) {
97+
throw new IllegalArgumentException(
98+
String.format(
99+
"JSONObject has fields unknown to BigQuery: %s. Set allowUnknownFields to True to allow unknown fields.",
100+
currentScope));
101+
} else {
102+
continue;
103+
}
104+
}
105+
matchedFields++;
106+
if (!field.isRepeated()) {
107+
fillField(protoMsg, field, json, jsonName, currentScope, allowUnknownFields);
108+
} else {
109+
fillRepeatedField(protoMsg, field, json, jsonName, currentScope, allowUnknownFields);
110+
}
111+
}
112+
113+
if (matchedFields == 0 && topLevel) {
114+
throw new IllegalArgumentException(
115+
"There are no matching fields found for the JSONObject and the protocol buffer descriptor.");
116+
}
117+
DynamicMessage msg;
118+
try {
119+
msg = protoMsg.build();
120+
} catch (UninitializedMessageException e) {
121+
String errorMsg = e.getMessage();
122+
int idxOfColon = errorMsg.indexOf(":");
123+
String missingFieldName = errorMsg.substring(idxOfColon + 2);
124+
throw new IllegalArgumentException(
125+
String.format(
126+
"JSONObject does not have the required field %s.%s.", jsonScope, missingFieldName));
127+
}
128+
if (topLevel && msg.getSerializedSize() == 0) {
129+
throw new IllegalArgumentException("The created protobuf message is empty.");
130+
}
131+
return msg;
132+
}
133+
134+
/**
135+
* Fills a non-repetaed protoField with the json data.
136+
*
137+
* @param protoMsg The protocol buffer message being constructed
138+
* @param fieldDescriptor
139+
* @param json
140+
* @param exactJsonKeyName Exact key name in JSONObject instead of lowercased version
141+
* @param currentScope Debugging purposes
142+
* @param allowUnknownFields Ignores unknown JSON fields.
143+
* @throws IllegalArgumentException when JSON data is not compatible with proto descriptor.
144+
*/
145+
private static void fillField(
146+
DynamicMessage.Builder protoMsg,
147+
FieldDescriptor fieldDescriptor,
148+
JSONObject json,
149+
String exactJsonKeyName,
150+
String currentScope,
151+
boolean allowUnknownFields)
152+
throws IllegalArgumentException {
153+
154+
java.lang.Object val = json.get(exactJsonKeyName);
155+
switch (fieldDescriptor.getType()) {
156+
case BOOL:
157+
if (val instanceof Boolean) {
158+
protoMsg.setField(fieldDescriptor, (Boolean) val);
159+
return;
160+
}
161+
break;
162+
case BYTES:
163+
if (val instanceof String) {
164+
protoMsg.setField(fieldDescriptor, ((String) val).getBytes());
165+
return;
166+
}
167+
break;
168+
case INT64:
169+
if (val instanceof Integer) {
170+
protoMsg.setField(fieldDescriptor, new Long((Integer) val));
171+
return;
172+
} else if (val instanceof Long) {
173+
protoMsg.setField(fieldDescriptor, (Long) val);
174+
return;
175+
}
176+
break;
177+
case INT32:
178+
if (val instanceof Integer) {
179+
protoMsg.setField(fieldDescriptor, (Integer) val);
180+
return;
181+
}
182+
break;
183+
case STRING:
184+
if (val instanceof String) {
185+
protoMsg.setField(fieldDescriptor, (String) val);
186+
return;
187+
}
188+
break;
189+
case DOUBLE:
190+
if (val instanceof Double) {
191+
protoMsg.setField(fieldDescriptor, (Double) val);
192+
return;
193+
} else if (val instanceof Float) {
194+
protoMsg.setField(fieldDescriptor, new Double((Float) val));
195+
return;
196+
}
197+
break;
198+
case MESSAGE:
199+
if (val instanceof JSONObject) {
200+
Message.Builder message = protoMsg.newBuilderForField(fieldDescriptor);
201+
protoMsg.setField(
202+
fieldDescriptor,
203+
convertJsonToProtoMessageImpl(
204+
fieldDescriptor.getMessageType(),
205+
json.getJSONObject(exactJsonKeyName),
206+
currentScope,
207+
/*topLevel =*/ false,
208+
allowUnknownFields));
209+
return;
210+
}
211+
break;
212+
}
213+
throw new IllegalArgumentException(
214+
String.format(
215+
"JSONObject does not have a %s field at %s.",
216+
FieldTypeToDebugMessage.get(fieldDescriptor.getType()), currentScope));
217+
}
218+
219+
/**
220+
* Fills a repeated protoField with the json data.
221+
*
222+
* @param protoMsg The protocol buffer message being constructed
223+
* @param fieldDescriptor
224+
* @param json If root level has no matching fields, throws exception.
225+
* @param exactJsonKeyName Exact key name in JSONObject instead of lowercased version
226+
* @param currentScope Debugging purposes
227+
* @param allowUnknownFields Ignores unknown JSON fields.
228+
* @throws IllegalArgumentException when JSON data is not compatible with proto descriptor.
229+
*/
230+
private static void fillRepeatedField(
231+
DynamicMessage.Builder protoMsg,
232+
FieldDescriptor fieldDescriptor,
233+
JSONObject json,
234+
String exactJsonKeyName,
235+
String currentScope,
236+
boolean allowUnknownFields)
237+
throws IllegalArgumentException {
238+
239+
JSONArray jsonArray;
240+
try {
241+
jsonArray = json.getJSONArray(exactJsonKeyName);
242+
} catch (JSONException e) {
243+
throw new IllegalArgumentException(
244+
"JSONObject does not have a array field at " + currentScope + ".");
245+
}
246+
java.lang.Object val;
247+
int index;
248+
boolean fail = false;
249+
for (int i = 0; i < jsonArray.length(); i++) {
250+
val = jsonArray.get(i);
251+
index = i;
252+
switch (fieldDescriptor.getType()) {
253+
case BOOL:
254+
if (val instanceof Boolean) {
255+
protoMsg.addRepeatedField(fieldDescriptor, (Boolean) val);
256+
} else {
257+
fail = true;
258+
}
259+
break;
260+
case BYTES:
261+
if (val instanceof String) {
262+
protoMsg.addRepeatedField(fieldDescriptor, ((String) val).getBytes());
263+
} else {
264+
fail = true;
265+
}
266+
break;
267+
case INT64:
268+
if (val instanceof Integer) {
269+
protoMsg.addRepeatedField(fieldDescriptor, new Long((Integer) val));
270+
} else if (val instanceof Long) {
271+
protoMsg.addRepeatedField(fieldDescriptor, (Long) val);
272+
} else {
273+
fail = true;
274+
}
275+
break;
276+
case INT32:
277+
if (val instanceof Integer) {
278+
protoMsg.addRepeatedField(fieldDescriptor, (Integer) val);
279+
} else {
280+
fail = true;
281+
}
282+
break;
283+
case STRING:
284+
if (val instanceof String) {
285+
protoMsg.addRepeatedField(fieldDescriptor, (String) val);
286+
} else {
287+
fail = true;
288+
}
289+
break;
290+
case DOUBLE:
291+
if (val instanceof Double) {
292+
protoMsg.addRepeatedField(fieldDescriptor, (Double) val);
293+
} else if (val instanceof Float) {
294+
protoMsg.addRepeatedField(fieldDescriptor, new Double((float) val));
295+
} else {
296+
fail = true;
297+
}
298+
break;
299+
case MESSAGE:
300+
if (val instanceof JSONObject) {
301+
Message.Builder message = protoMsg.newBuilderForField(fieldDescriptor);
302+
protoMsg.addRepeatedField(
303+
fieldDescriptor,
304+
convertJsonToProtoMessageImpl(
305+
fieldDescriptor.getMessageType(),
306+
jsonArray.getJSONObject(i),
307+
currentScope,
308+
/*topLevel =*/ false,
309+
allowUnknownFields));
310+
} else {
311+
fail = true;
312+
}
313+
break;
314+
}
315+
if (fail) {
316+
throw new IllegalArgumentException(
317+
String.format(
318+
"JSONObject does not have a %s field at %s[%d].",
319+
FieldTypeToDebugMessage.get(fieldDescriptor.getType()), currentScope, index));
320+
}
321+
}
322+
}
323+
}

0 commit comments

Comments
 (0)