Skip to content

Commit 6d3ff2e

Browse files
tseavertseaver
authored
Merge pull request #1925 from tseaver/bigtable-v2-generate_v2_protos_redux
Regenerate from final Bigtable v2 v2 protos
2 parents 90cd2e7 + 24989d6 commit 6d3ff2e

25 files changed

+1454
-750
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,3 +57,4 @@ scripts/pylintrc_reduced
5757
generated_python/
5858
cloud-bigtable-client/
5959
googleapis-pb/
60+
grpc_python_venv/

‎Makefile.bigtable_v1‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1+
GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv
12
GENERATED_DIR=$(shell pwd)/generated_python
23
GENERATED_SUBDIR=_generated
34
BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/$(GENERATED_SUBDIR)
4-
GRPC_PLUGIN=grpc_python_plugin
5-
PROTOC_CMD=protoc
6-
BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
7-
BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-protos/src/main/proto
5+
PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc
86
GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
7+
BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
8+
BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-client-core-parent/bigtable-protos/src/main/proto
99

1010
help:
1111
@echo 'Makefile for gcloud-python Bigtable protos '
@@ -15,19 +15,22 @@ help:
1515
@echo ' make clean Clean generated files '
1616

1717
generate:
18+
# Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools
19+
[ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV)
20+
$(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools
1821
# Retrieve git repos that have our *.proto files.
1922
[ -d $(BIGTABLE_CHECKOUT_DIR) ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
2023
cd $(BIGTABLE_CHECKOUT_DIR) && git pull origin master
21-
[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
22-
cd googleapis-pb && git pull origin master
24+
[ -d $(GOOGLEAPIS_PROTOS_DIR) ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
25+
cd $(GOOGLEAPIS_PROTOS_DIR) && git pull origin master
2326
# Make the directory where our *_pb2.py files will go.
2427
mkdir -p $(GENERATED_DIR)
2528
# Generate all *_pb2.py files that require gRPC.
2629
$(PROTOC_CMD) \
2730
--proto_path=$(BIGTABLE_PROTOS_DIR) \
31+
--proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
2832
--python_out=$(GENERATED_DIR) \
29-
--plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \
30-
--grpc_out=$(GENERATED_DIR) \
33+
--grpc_python_out=$(GENERATED_DIR) \
3134
$(BIGTABLE_PROTOS_DIR)/google/bigtable/v1/bigtable_service.proto \
3235
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto \
3336
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service.proto
@@ -52,7 +55,7 @@ generate:
5255
cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/v1/*.proto $(BIGTABLE_DIR)
5356
cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/*.proto $(BIGTABLE_DIR)
5457
cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/*.proto $(BIGTABLE_DIR)
55-
cp $(BIGTABLE_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
58+
cp $(GOOGLEAPIS_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
5659
# Rename all *.proto files in our library with an
5760
# underscore and remove executable bit.
5861
cd $(BIGTABLE_DIR) && \
@@ -63,8 +66,8 @@ generate:
6366
# Separate the gRPC parts of the operations service from the
6467
# non-gRPC parts so that the protos from `googleapis-common-protos`
6568
# can be used without gRPC.
66-
PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
67-
GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
69+
GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \
70+
GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
6871
python scripts/make_operations_grpc.py
6972
# Rewrite the imports in the generated *_pb2.py files.
7073
python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py
@@ -73,6 +76,6 @@ check_generate:
7376
python scripts/check_generate.py
7477

7578
clean:
76-
rm -fr $(BIGTABLE_CHECKOUT_DIR) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
79+
rm -fr $(GRPCIO_VIRTUALENV) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
7780

7881
.PHONY: generate check_generate clean

‎Makefile.bigtable_v2‎

Lines changed: 20 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
1+
GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv
12
GENERATED_DIR=$(shell pwd)/generated_python
23
GENERATED_SUBDIR=_generated_v2
34
BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/$(GENERATED_SUBDIR)
4-
GRPC_PLUGIN=grpc_python_plugin
5-
PROTOC_CMD=protoc
6-
BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
7-
BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-protos/src/main/proto
5+
PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc
86
GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
97

108
help:
@@ -15,41 +13,39 @@ help:
1513
@echo ' make clean Clean generated files '
1614

1715
generate:
16+
# Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools
17+
[ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV)
18+
$(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools
1819
# Retrieve git repos that have our *.proto files.
19-
[ -d $(BIGTABLE_CHECKOUT_DIR) ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
20-
cd $(BIGTABLE_CHECKOUT_DIR) && git pull origin master
2120
[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
2221
cd googleapis-pb && git pull origin master
2322
# Make the directory where our *_pb2.py files will go.
2423
mkdir -p $(GENERATED_DIR)
2524
# Generate all *_pb2.py files that require gRPC.
2625
$(PROTOC_CMD) \
27-
--proto_path=$(BIGTABLE_PROTOS_DIR) \
26+
--proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
2827
--python_out=$(GENERATED_DIR) \
29-
--plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \
30-
--grpc_out=$(GENERATED_DIR) \
31-
$(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/bigtable.proto \
32-
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_instance_admin.proto \
33-
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_table_admin.proto
28+
--grpc_python_out=$(GENERATED_DIR) \
29+
$(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/bigtable.proto \
30+
$(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_instance_admin.proto \
31+
$(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_table_admin.proto
3432
# Generate all *_pb2.py files that do not require gRPC.
3533
$(PROTOC_CMD) \
36-
--proto_path=$(BIGTABLE_PROTOS_DIR) \
3734
--proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
3835
--python_out=$(GENERATED_DIR) \
39-
$(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/data.proto \
40-
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/common.proto \
41-
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/instance.proto \
42-
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/table.proto \
36+
$(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/data.proto \
37+
$(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/common.proto \
38+
$(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/instance.proto \
39+
$(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/table.proto \
4340
# Move the newly generated *_pb2.py files into our library.
4441
cp $(GENERATED_DIR)/google/bigtable/v2/* $(BIGTABLE_DIR)
4542
cp $(GENERATED_DIR)/google/bigtable/admin/v2/* $(BIGTABLE_DIR)
46-
cp $(GENERATED_DIR)/google/bigtable/admin/v2/* $(BIGTABLE_DIR)
4743
# Remove all existing *.proto files before we replace
4844
rm -f $(BIGTABLE_DIR)/*.proto
4945
# Copy over the *.proto files into our library.
50-
cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/*.proto $(BIGTABLE_DIR)
51-
cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/*.proto $(BIGTABLE_DIR)
52-
cp $(BIGTABLE_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
46+
cp $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/*.proto $(BIGTABLE_DIR)
47+
cp $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/*.proto $(BIGTABLE_DIR)
48+
cp $(GOOGLEAPIS_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
5349
# Rename all *.proto files in our library with an
5450
# underscore and remove executable bit.
5551
cd $(BIGTABLE_DIR) && \
@@ -60,8 +56,8 @@ generate:
6056
# Separate the gRPC parts of the operations service from the
6157
# non-gRPC parts so that the protos from `googleapis-common-protos`
6258
# can be used without gRPC.
63-
PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
64-
GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
59+
GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \
60+
GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
6561
python scripts/make_operations_grpc.py
6662
# Rewrite the imports in the generated *_pb2.py files.
6763
python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py
@@ -70,6 +66,6 @@ check_generate:
7066
python scripts/check_generate.py
7167

7268
clean:
73-
rm -fr $(BIGTABLE_CHECKOUT_DIR) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
69+
rm -fr $(GRPCIO_VIRTUALENV) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
7470

7571
.PHONY: generate check_generate clean

‎Makefile.datastore‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1+
GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv
12
GENERATED_DIR=$(shell pwd)/generated_python
23
DATASTORE_DIR=$(shell pwd)/gcloud/datastore/_generated
3-
GRPC_PLUGIN=grpc_python_plugin
4-
PROTOC_CMD=protoc
4+
PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc
55
GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
66

77
help:
@@ -12,6 +12,9 @@ help:
1212
@echo ' make clean Clean generated files '
1313

1414
generate:
15+
# Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools
16+
[ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV)
17+
$(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools
1518
# Retrieve git repos that have our *.proto files.
1619
[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
1720
cd googleapis-pb && git pull origin master
@@ -39,7 +42,8 @@ generate:
3942
done
4043
# Separate the gRPC parts of the datastore service from the
4144
# non-gRPC parts so that the protos can be used without gRPC.
42-
PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
45+
GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \
46+
GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
4347
python scripts/make_datastore_grpc.py
4448
# Rewrite the imports in the generated *_pb2.py files.
4549
python scripts/rewrite_imports.py $(DATASTORE_DIR)/*pb2.py

0 commit comments

Comments
 (0)