Skip to content

Commit 8fbb80e

Browse files
yirutangyirutang
authored
feat: proto converter library (#100)
* ProtoConverter library modified: google-cloud-bigquerystorage/pom.xml new file: google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverter.java new file: google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverterTest.java new file: google-cloud-bigquerystorage/src/test/proto/test.proto modified: pom.xml * Adjust dependency ordering
1 parent 555ce99 commit 8fbb80e

File tree

5 files changed

+346
-1
lines changed

5 files changed

+346
-1
lines changed

‎google-cloud-bigquerystorage/pom.xml‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,32 @@
1616
<properties>
1717
<site.installationModule>google-cloud-bigquerystorage</site.installationModule>
1818
</properties>
19+
<build>
20+
<extensions>
21+
<extension>
22+
<groupId>kr.motd.maven</groupId>
23+
<artifactId>os-maven-plugin</artifactId>
24+
<version>1.6.2</version>
25+
</extension>
26+
</extensions>
27+
<plugins>
28+
<plugin>
29+
<groupId>com.google.protobuf.tools</groupId>
30+
<artifactId>maven-protoc-plugin</artifactId>
31+
<version>0.4.2</version>
32+
<configuration>
33+
<protocArtifact>com.google.protobuf:protoc:${project.protobuf-java.version}:exe:${os.detected.classifier}</protocArtifact>
34+
</configuration>
35+
<executions>
36+
<execution>
37+
<goals>
38+
<goal>test-compile</goal>
39+
</goals>
40+
</execution>
41+
</executions>
42+
</plugin>
43+
</plugins>
44+
</build>
1945
<dependencies>
2046
<dependency>
2147
<groupId>io.grpc</groupId>
@@ -125,6 +151,10 @@
125151
<artifactId>google-cloud-core</artifactId>
126152
<version>${google.core.version}</version>
127153
</dependency>
154+
<dependency>
155+
<groupId>com.google.protobuf</groupId>
156+
<artifactId>protobuf-java</artifactId>
157+
</dependency>
128158

129159
<dependency>
130160
<groupId>com.google.api.grpc</groupId>
@@ -169,4 +199,4 @@
169199
</dependencies>
170200
</profile>
171201
</profiles>
172-
</project>
202+
</project>

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverter.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigquery.storage.v1alpha2;
17+
18+
import com.google.api.gax.grpc.GrpcStatusCode;
19+
import com.google.api.gax.rpc.InvalidArgumentException;
20+
import com.google.cloud.bigquery.storage.v1alpha2.ProtoBufProto.ProtoSchema;
21+
import com.google.protobuf.DescriptorProtos.DescriptorProto;
22+
import com.google.protobuf.DescriptorProtos.FieldDescriptorProto;
23+
import com.google.protobuf.Descriptors.Descriptor;
24+
import com.google.protobuf.Descriptors.FieldDescriptor;
25+
import io.grpc.Status;
26+
import java.util.*;
27+
28+
// A Converter class that turns a native protobuf::DescriptorProto to a self contained
29+
// protobuf::DescriptorProto
30+
// that can be reconstructed by the backend.
31+
public class ProtoSchemaConverter {
32+
private static class StructName {
33+
public String getName() {
34+
return "__S" + (count++);
35+
}
36+
37+
private int count = 0;
38+
}
39+
40+
private static ProtoSchema convertInternal(
41+
Descriptor input, List<String> visitedTypes, StructName structName) {
42+
DescriptorProto.Builder resultProto = DescriptorProto.newBuilder();
43+
resultProto.setName(structName.getName());
44+
visitedTypes.add(input.getFullName());
45+
for (int i = 0; i < input.getFields().size(); i++) {
46+
FieldDescriptor inputField = input.getFields().get(i);
47+
FieldDescriptorProto.Builder resultField = inputField.toProto().toBuilder();
48+
if (inputField.getType() == FieldDescriptor.Type.GROUP
49+
|| inputField.getType() == FieldDescriptor.Type.MESSAGE) {
50+
if (visitedTypes.contains(inputField.getMessageType().getFullName())) {
51+
throw new InvalidArgumentException(
52+
"Recursive type is not supported:" + inputField.getMessageType().getFullName(),
53+
null,
54+
GrpcStatusCode.of(Status.Code.INVALID_ARGUMENT),
55+
false);
56+
}
57+
resultProto.addNestedType(
58+
convertInternal(inputField.getMessageType(), visitedTypes, structName)
59+
.getProtoDescriptor());
60+
visitedTypes.remove(inputField.getMessageType().getFullName());
61+
resultField.setTypeName(
62+
resultProto.getNestedType(resultProto.getNestedTypeCount() - 1).getName());
63+
}
64+
if (inputField.getType() == FieldDescriptor.Type.ENUM) {
65+
resultProto.addEnumType(inputField.getEnumType().toProto());
66+
resultField.setTypeName(
67+
resultProto.getEnumType(resultProto.getEnumTypeCount() - 1).getName());
68+
}
69+
resultProto.addField(resultField);
70+
}
71+
return ProtoSchema.newBuilder().setProtoDescriptor(resultProto.build()).build();
72+
}
73+
74+
public static ProtoSchema convert(Descriptor descriptor) {
75+
ArrayList<String> visitedTypes = new ArrayList<String>();
76+
return convertInternal(descriptor, visitedTypes, new StructName());
77+
}
78+
}

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverterTest.java‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigquery.storage.v1alpha2;
17+
18+
import com.google.api.gax.rpc.InvalidArgumentException;
19+
import com.google.cloud.bigquery.storage.test.Test.*;
20+
import org.junit.*;
21+
22+
public class ProtoSchemaConverterTest {
23+
@Test
24+
public void convertSimple() {
25+
AllSupportedTypes testProto = AllSupportedTypes.newBuilder().setStringValue("abc").build();
26+
ProtoBufProto.ProtoSchema protoSchema =
27+
ProtoSchemaConverter.convert(testProto.getDescriptorForType());
28+
Assert.assertEquals(
29+
"name: \"__S0\"\n"
30+
+ "field {\n"
31+
+ " name: \"int32_value\"\n"
32+
+ " number: 1\n"
33+
+ " label: LABEL_OPTIONAL\n"
34+
+ " type: TYPE_INT32\n"
35+
+ "}\n"
36+
+ "field {\n"
37+
+ " name: \"int64_value\"\n"
38+
+ " number: 2\n"
39+
+ " label: LABEL_OPTIONAL\n"
40+
+ " type: TYPE_INT64\n"
41+
+ "}\n"
42+
+ "field {\n"
43+
+ " name: \"uint32_value\"\n"
44+
+ " number: 3\n"
45+
+ " label: LABEL_OPTIONAL\n"
46+
+ " type: TYPE_UINT32\n"
47+
+ "}\n"
48+
+ "field {\n"
49+
+ " name: \"uint64_value\"\n"
50+
+ " number: 4\n"
51+
+ " label: LABEL_OPTIONAL\n"
52+
+ " type: TYPE_UINT64\n"
53+
+ "}\n"
54+
+ "field {\n"
55+
+ " name: \"float_value\"\n"
56+
+ " number: 5\n"
57+
+ " label: LABEL_OPTIONAL\n"
58+
+ " type: TYPE_FLOAT\n"
59+
+ "}\n"
60+
+ "field {\n"
61+
+ " name: \"double_value\"\n"
62+
+ " number: 6\n"
63+
+ " label: LABEL_OPTIONAL\n"
64+
+ " type: TYPE_DOUBLE\n"
65+
+ "}\n"
66+
+ "field {\n"
67+
+ " name: \"bool_value\"\n"
68+
+ " number: 7\n"
69+
+ " label: LABEL_OPTIONAL\n"
70+
+ " type: TYPE_BOOL\n"
71+
+ "}\n"
72+
+ "field {\n"
73+
+ " name: \"enum_value\"\n"
74+
+ " number: 8\n"
75+
+ " label: LABEL_OPTIONAL\n"
76+
+ " type: TYPE_ENUM\n"
77+
+ " type_name: \"TestEnum\"\n"
78+
+ "}\n"
79+
+ "field {\n"
80+
+ " name: \"string_value\"\n"
81+
+ " number: 9\n"
82+
+ " label: LABEL_REQUIRED\n"
83+
+ " type: TYPE_STRING\n"
84+
+ "}\n"
85+
+ "enum_type {\n"
86+
+ " name: \"TestEnum\"\n"
87+
+ " value {\n"
88+
+ " name: \"TestEnum0\"\n"
89+
+ " number: 0\n"
90+
+ " }\n"
91+
+ " value {\n"
92+
+ " name: \"TestEnum1\"\n"
93+
+ " number: 1\n"
94+
+ " }\n"
95+
+ "}\n",
96+
protoSchema.getProtoDescriptor().toString());
97+
}
98+
99+
@Test
100+
public void convertNested() {
101+
ComplicateType testProto = ComplicateType.newBuilder().build();
102+
ProtoBufProto.ProtoSchema protoSchema =
103+
ProtoSchemaConverter.convert(testProto.getDescriptorForType());
104+
Assert.assertEquals(
105+
"name: \"__S0\"\n"
106+
+ "field {\n"
107+
+ " name: \"nested_repeated_type\"\n"
108+
+ " number: 1\n"
109+
+ " label: LABEL_REPEATED\n"
110+
+ " type: TYPE_MESSAGE\n"
111+
+ " type_name: \"__S1\"\n"
112+
+ "}\n"
113+
+ "field {\n"
114+
+ " name: \"inner_type\"\n"
115+
+ " number: 2\n"
116+
+ " label: LABEL_OPTIONAL\n"
117+
+ " type: TYPE_MESSAGE\n"
118+
+ " type_name: \"__S3\"\n"
119+
+ "}\n"
120+
+ "nested_type {\n"
121+
+ " name: \"__S1\"\n"
122+
+ " field {\n"
123+
+ " name: \"inner_type\"\n"
124+
+ " number: 1\n"
125+
+ " label: LABEL_REPEATED\n"
126+
+ " type: TYPE_MESSAGE\n"
127+
+ " type_name: \"__S2\"\n"
128+
+ " }\n"
129+
+ " nested_type {\n"
130+
+ " name: \"__S2\"\n"
131+
+ " field {\n"
132+
+ " name: \"value\"\n"
133+
+ " number: 1\n"
134+
+ " label: LABEL_REPEATED\n"
135+
+ " type: TYPE_STRING\n"
136+
+ " }\n"
137+
+ " }\n"
138+
+ "}\n"
139+
+ "nested_type {\n"
140+
+ " name: \"__S3\"\n"
141+
+ " field {\n"
142+
+ " name: \"value\"\n"
143+
+ " number: 1\n"
144+
+ " label: LABEL_REPEATED\n"
145+
+ " type: TYPE_STRING\n"
146+
+ " }\n"
147+
+ "}\n",
148+
protoSchema.getProtoDescriptor().toString());
149+
}
150+
151+
@Test
152+
public void convertRecursive() {
153+
try {
154+
RecursiveType testProto = RecursiveType.newBuilder().build();
155+
ProtoBufProto.ProtoSchema protoSchema =
156+
ProtoSchemaConverter.convert(testProto.getDescriptorForType());
157+
Assert.fail("No exception raised");
158+
} catch (InvalidArgumentException e) {
159+
// Expected exception
160+
}
161+
}
162+
}

‎google-cloud-bigquerystorage/src/test/proto/test.proto‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
syntax = "proto2";
17+
18+
package com.google.cloud.bigquery.storage.test;
19+
20+
import "google/protobuf/descriptor.proto";
21+
22+
enum TestEnum {
23+
TestEnum0 = 0;
24+
TestEnum1 = 1;
25+
}
26+
27+
message AllSupportedTypes {
28+
optional int32 int32_value = 1;
29+
optional int64 int64_value = 2;
30+
optional uint32 uint32_value = 3;
31+
optional uint64 uint64_value = 4;
32+
optional float float_value = 5;
33+
optional double double_value = 6;
34+
optional bool bool_value = 7;
35+
optional TestEnum enum_value = 8;
36+
required string string_value = 9;
37+
}
38+
39+
message InnerType {
40+
repeated string value = 1;
41+
}
42+
message NestedType {
43+
repeated InnerType inner_type = 1;
44+
}
45+
46+
message ComplicateType {
47+
repeated NestedType nested_repeated_type = 1;
48+
optional InnerType inner_type = 2;
49+
}
50+
51+
message ContainsRecursive {
52+
optional RecursiveType field = 1;
53+
}
54+
message RecursiveType {
55+
optional ContainsRecursive field = 2;
56+
}

‎pom.xml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
<properties>
6262
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
6363
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
64+
<project.protobuf-java.version>3.11.4</project.protobuf-java.version>
6465
<github.global.server>github</github.global.server>
6566
<site.installationModule>google-cloud-bigquerystorage-parent</site.installationModule>
6667
<google.core.version>1.93.2</google.core.version>
@@ -78,6 +79,24 @@
7879
<animal-sniffer.version>1.18</animal-sniffer.version>
7980
</properties>
8081

82+
<pluginRepositories>
83+
<pluginRepository>
84+
<releases>
85+
<updatePolicy>never</updatePolicy>
86+
</releases>
87+
<snapshots>
88+
<enabled>false</enabled>
89+
</snapshots>
90+
<id>central</id>
91+
<name>Central Repository</name>
92+
<url>https://repo.maven.apache.org/maven2</url>
93+
</pluginRepository>
94+
<pluginRepository>
95+
<id>protoc-plugin</id>
96+
<url>https://dl.bintray.com/sergei-ivanov/maven/</url>
97+
</pluginRepository>
98+
</pluginRepositories>
99+
81100
<dependencyManagement>
82101
<dependencies>
83102
<dependency>

0 commit comments

Comments
 (0)