Skip to content
This repository was archived by the owner on Nov 29, 2023. It is now read-only.

Commit 116077b

Browse files
gcf-owl-bot[bot]gcf-owl-bot[bot]
authored
feat!: add support for Virtual Dataproc cluster running on GKE cluster (#344)
- [ ] Regenerate this pull request now. Committer: @Padmaar PiperOrigin-RevId: 429111624 Source-Link: googleapis/googleapis@da999a2 Source-Link: https://github.com/googleapis/googleapis-gen/commit/99c5b3e98baa1de93776aa4b5cd4c7361353e4f6 Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiOTljNWIzZTk4YmFhMWRlOTM3NzZhYTRiNWNkNGM3MzYxMzUzZTRmNiJ9 fix: remove namespaced_gke_deployment_target fix: move GkeClusterConfig to google.cloud.dataproc_v1.types.shared
1 parent 4344dfa commit 116077b

File tree

11 files changed

+402
-162
lines changed

11 files changed

+402
-162
lines changed

‎google/cloud/dataproc/__init__.py‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
from google.cloud.dataproc_v1.types.batches import SparkSqlBatch
8484
from google.cloud.dataproc_v1.types.clusters import AcceleratorConfig
8585
from google.cloud.dataproc_v1.types.clusters import AutoscalingConfig
86+
from google.cloud.dataproc_v1.types.clusters import AuxiliaryServicesConfig
8687
from google.cloud.dataproc_v1.types.clusters import Cluster
8788
from google.cloud.dataproc_v1.types.clusters import ClusterConfig
8889
from google.cloud.dataproc_v1.types.clusters import ClusterMetrics
@@ -97,7 +98,6 @@
9798
from google.cloud.dataproc_v1.types.clusters import EndpointConfig
9899
from google.cloud.dataproc_v1.types.clusters import GceClusterConfig
99100
from google.cloud.dataproc_v1.types.clusters import GetClusterRequest
100-
from google.cloud.dataproc_v1.types.clusters import GkeClusterConfig
101101
from google.cloud.dataproc_v1.types.clusters import IdentityConfig
102102
from google.cloud.dataproc_v1.types.clusters import InstanceGroupConfig
103103
from google.cloud.dataproc_v1.types.clusters import KerberosConfig
@@ -115,6 +115,7 @@
115115
from google.cloud.dataproc_v1.types.clusters import StartClusterRequest
116116
from google.cloud.dataproc_v1.types.clusters import StopClusterRequest
117117
from google.cloud.dataproc_v1.types.clusters import UpdateClusterRequest
118+
from google.cloud.dataproc_v1.types.clusters import VirtualClusterConfig
118119
from google.cloud.dataproc_v1.types.jobs import CancelJobRequest
119120
from google.cloud.dataproc_v1.types.jobs import DeleteJobRequest
120121
from google.cloud.dataproc_v1.types.jobs import GetJobRequest
@@ -144,6 +145,11 @@
144145
from google.cloud.dataproc_v1.types.operations import ClusterOperationStatus
145146
from google.cloud.dataproc_v1.types.shared import EnvironmentConfig
146147
from google.cloud.dataproc_v1.types.shared import ExecutionConfig
148+
from google.cloud.dataproc_v1.types.shared import GkeClusterConfig
149+
from google.cloud.dataproc_v1.types.shared import GkeNodePoolConfig
150+
from google.cloud.dataproc_v1.types.shared import GkeNodePoolTarget
151+
from google.cloud.dataproc_v1.types.shared import KubernetesClusterConfig
152+
from google.cloud.dataproc_v1.types.shared import KubernetesSoftwareConfig
147153
from google.cloud.dataproc_v1.types.shared import PeripheralsConfig
148154
from google.cloud.dataproc_v1.types.shared import RuntimeConfig
149155
from google.cloud.dataproc_v1.types.shared import RuntimeInfo
@@ -219,6 +225,7 @@
219225
"SparkSqlBatch",
220226
"AcceleratorConfig",
221227
"AutoscalingConfig",
228+
"AuxiliaryServicesConfig",
222229
"Cluster",
223230
"ClusterConfig",
224231
"ClusterMetrics",
@@ -233,7 +240,6 @@
233240
"EndpointConfig",
234241
"GceClusterConfig",
235242
"GetClusterRequest",
236-
"GkeClusterConfig",
237243
"IdentityConfig",
238244
"InstanceGroupConfig",
239245
"KerberosConfig",
@@ -251,6 +257,7 @@
251257
"StartClusterRequest",
252258
"StopClusterRequest",
253259
"UpdateClusterRequest",
260+
"VirtualClusterConfig",
254261
"CancelJobRequest",
255262
"DeleteJobRequest",
256263
"GetJobRequest",
@@ -280,6 +287,11 @@
280287
"ClusterOperationStatus",
281288
"EnvironmentConfig",
282289
"ExecutionConfig",
290+
"GkeClusterConfig",
291+
"GkeNodePoolConfig",
292+
"GkeNodePoolTarget",
293+
"KubernetesClusterConfig",
294+
"KubernetesSoftwareConfig",
283295
"PeripheralsConfig",
284296
"RuntimeConfig",
285297
"RuntimeInfo",

‎google/cloud/dataproc_v1/__init__.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
from .types.batches import SparkSqlBatch
4848
from .types.clusters import AcceleratorConfig
4949
from .types.clusters import AutoscalingConfig
50+
from .types.clusters import AuxiliaryServicesConfig
5051
from .types.clusters import Cluster
5152
from .types.clusters import ClusterConfig
5253
from .types.clusters import ClusterMetrics
@@ -61,7 +62,6 @@
6162
from .types.clusters import EndpointConfig
6263
from .types.clusters import GceClusterConfig
6364
from .types.clusters import GetClusterRequest
64-
from .types.clusters import GkeClusterConfig
6565
from .types.clusters import IdentityConfig
6666
from .types.clusters import InstanceGroupConfig
6767
from .types.clusters import KerberosConfig
@@ -79,6 +79,7 @@
7979
from .types.clusters import StartClusterRequest
8080
from .types.clusters import StopClusterRequest
8181
from .types.clusters import UpdateClusterRequest
82+
from .types.clusters import VirtualClusterConfig
8283
from .types.jobs import CancelJobRequest
8384
from .types.jobs import DeleteJobRequest
8485
from .types.jobs import GetJobRequest
@@ -108,6 +109,11 @@
108109
from .types.operations import ClusterOperationStatus
109110
from .types.shared import EnvironmentConfig
110111
from .types.shared import ExecutionConfig
112+
from .types.shared import GkeClusterConfig
113+
from .types.shared import GkeNodePoolConfig
114+
from .types.shared import GkeNodePoolTarget
115+
from .types.shared import KubernetesClusterConfig
116+
from .types.shared import KubernetesSoftwareConfig
111117
from .types.shared import PeripheralsConfig
112118
from .types.shared import RuntimeConfig
113119
from .types.shared import RuntimeInfo
@@ -146,6 +152,7 @@
146152
"AutoscalingConfig",
147153
"AutoscalingPolicy",
148154
"AutoscalingPolicyServiceClient",
155+
"AuxiliaryServicesConfig",
149156
"BasicAutoscalingAlgorithm",
150157
"BasicYarnAutoscalingConfig",
151158
"Batch",
@@ -187,6 +194,8 @@
187194
"GetJobRequest",
188195
"GetWorkflowTemplateRequest",
189196
"GkeClusterConfig",
197+
"GkeNodePoolConfig",
198+
"GkeNodePoolTarget",
190199
"HadoopJob",
191200
"HiveJob",
192201
"IdentityConfig",
@@ -202,6 +211,8 @@
202211
"JobScheduling",
203212
"JobStatus",
204213
"KerberosConfig",
214+
"KubernetesClusterConfig",
215+
"KubernetesSoftwareConfig",
205216
"LifecycleConfig",
206217
"ListAutoscalingPoliciesRequest",
207218
"ListAutoscalingPoliciesResponse",
@@ -250,6 +261,7 @@
250261
"UpdateJobRequest",
251262
"UpdateWorkflowTemplateRequest",
252263
"ValueValidation",
264+
"VirtualClusterConfig",
253265
"WorkflowGraph",
254266
"WorkflowMetadata",
255267
"WorkflowNode",

‎google/cloud/dataproc_v1/services/cluster_controller/async_client.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,6 @@ class ClusterControllerAsyncClient:
5353
DEFAULT_ENDPOINT = ClusterControllerClient.DEFAULT_ENDPOINT
5454
DEFAULT_MTLS_ENDPOINT = ClusterControllerClient.DEFAULT_MTLS_ENDPOINT
5555

56-
cluster_path = staticmethod(ClusterControllerClient.cluster_path)
57-
parse_cluster_path = staticmethod(ClusterControllerClient.parse_cluster_path)
5856
service_path = staticmethod(ClusterControllerClient.service_path)
5957
parse_service_path = staticmethod(ClusterControllerClient.parse_service_path)
6058
common_billing_account_path = staticmethod(

‎google/cloud/dataproc_v1/services/cluster_controller/client.py‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -168,22 +168,6 @@ def transport(self) -> ClusterControllerTransport:
168168
"""
169169
return self._transport
170170

171-
@staticmethod
172-
def cluster_path(project: str, location: str, cluster: str,) -> str:
173-
"""Returns a fully-qualified cluster string."""
174-
return "projects/{project}/locations/{location}/clusters/{cluster}".format(
175-
project=project, location=location, cluster=cluster,
176-
)
177-
178-
@staticmethod
179-
def parse_cluster_path(path: str) -> Dict[str, str]:
180-
"""Parses a cluster path into its component segments."""
181-
m = re.match(
182-
r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)/clusters/(?P<cluster>.+?)$",
183-
path,
184-
)
185-
return m.groupdict() if m else {}
186-
187171
@staticmethod
188172
def service_path(project: str, location: str, service: str,) -> str:
189173
"""Returns a fully-qualified service string."""

‎google/cloud/dataproc_v1/services/workflow_template_service/async_client.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,6 @@ class WorkflowTemplateServiceAsyncClient:
5353
DEFAULT_ENDPOINT = WorkflowTemplateServiceClient.DEFAULT_ENDPOINT
5454
DEFAULT_MTLS_ENDPOINT = WorkflowTemplateServiceClient.DEFAULT_MTLS_ENDPOINT
5555

56-
cluster_path = staticmethod(WorkflowTemplateServiceClient.cluster_path)
57-
parse_cluster_path = staticmethod(WorkflowTemplateServiceClient.parse_cluster_path)
5856
service_path = staticmethod(WorkflowTemplateServiceClient.service_path)
5957
parse_service_path = staticmethod(WorkflowTemplateServiceClient.parse_service_path)
6058
workflow_template_path = staticmethod(

‎google/cloud/dataproc_v1/services/workflow_template_service/client.py‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -168,22 +168,6 @@ def transport(self) -> WorkflowTemplateServiceTransport:
168168
"""
169169
return self._transport
170170

171-
@staticmethod
172-
def cluster_path(project: str, location: str, cluster: str,) -> str:
173-
"""Returns a fully-qualified cluster string."""
174-
return "projects/{project}/locations/{location}/clusters/{cluster}".format(
175-
project=project, location=location, cluster=cluster,
176-
)
177-
178-
@staticmethod
179-
def parse_cluster_path(path: str) -> Dict[str, str]:
180-
"""Parses a cluster path into its component segments."""
181-
m = re.match(
182-
r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)/clusters/(?P<cluster>.+?)$",
183-
path,
184-
)
185-
return m.groupdict() if m else {}
186-
187171
@staticmethod
188172
def service_path(project: str, location: str, service: str,) -> str:
189173
"""Returns a fully-qualified service string."""

‎google/cloud/dataproc_v1/types/__init__.py‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
from .clusters import (
4141
AcceleratorConfig,
4242
AutoscalingConfig,
43+
AuxiliaryServicesConfig,
4344
Cluster,
4445
ClusterConfig,
4546
ClusterMetrics,
@@ -54,7 +55,6 @@
5455
EndpointConfig,
5556
GceClusterConfig,
5657
GetClusterRequest,
57-
GkeClusterConfig,
5858
IdentityConfig,
5959
InstanceGroupConfig,
6060
KerberosConfig,
@@ -72,6 +72,7 @@
7272
StartClusterRequest,
7373
StopClusterRequest,
7474
UpdateClusterRequest,
75+
VirtualClusterConfig,
7576
)
7677
from .jobs import (
7778
CancelJobRequest,
@@ -107,6 +108,11 @@
107108
from .shared import (
108109
EnvironmentConfig,
109110
ExecutionConfig,
111+
GkeClusterConfig,
112+
GkeNodePoolConfig,
113+
GkeNodePoolTarget,
114+
KubernetesClusterConfig,
115+
KubernetesSoftwareConfig,
110116
PeripheralsConfig,
111117
RuntimeConfig,
112118
RuntimeInfo,
@@ -161,6 +167,7 @@
161167
"SparkSqlBatch",
162168
"AcceleratorConfig",
163169
"AutoscalingConfig",
170+
"AuxiliaryServicesConfig",
164171
"Cluster",
165172
"ClusterConfig",
166173
"ClusterMetrics",
@@ -175,7 +182,6 @@
175182
"EndpointConfig",
176183
"GceClusterConfig",
177184
"GetClusterRequest",
178-
"GkeClusterConfig",
179185
"IdentityConfig",
180186
"InstanceGroupConfig",
181187
"KerberosConfig",
@@ -193,6 +199,7 @@
193199
"StartClusterRequest",
194200
"StopClusterRequest",
195201
"UpdateClusterRequest",
202+
"VirtualClusterConfig",
196203
"CancelJobRequest",
197204
"DeleteJobRequest",
198205
"GetJobRequest",
@@ -222,6 +229,11 @@
222229
"ClusterOperationStatus",
223230
"EnvironmentConfig",
224231
"ExecutionConfig",
232+
"GkeClusterConfig",
233+
"GkeNodePoolConfig",
234+
"GkeNodePoolTarget",
235+
"KubernetesClusterConfig",
236+
"KubernetesSoftwareConfig",
225237
"PeripheralsConfig",
226238
"RuntimeConfig",
227239
"RuntimeInfo",

0 commit comments

Comments
 (0)