feat: add Spark runtime versioning for Spark batches (#318)

- [ ] Regenerate this pull request now. Committer: @medb PiperOrigin-RevId: 420175701 Source-Link: googleapis/googleapis@1116092 Source-Link: https://github.com/googleapis/googleapis-gen/commit/72c0b7f4dcdf2cd4d4729769cbdeb2b228c504e1 Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiNzJjMGI3ZjRkY2RmMmNkNGQ0NzI5NzY5Y2JkZWIyYjIyOGM1MDRlMSJ9 feat: custom image containers for Spark batches feat: auto-diagnostic of failed Spark batches feat: local SSD NVME interface support for GCE clusters

Author: gcf-owl-bot[bot]

google/cloud/dataproc_v1/services/batch_controller/async_client.py

@@ -208,7 +208,7 @@ async def create_batch(
  the final component of the batch's resource name.
 
  This value must be 4-63 characters. Valid characters are
- ``/[a-z][0-9]-/``.
+ /[a-z][0-9]-/.
 
  This corresponds to the ``batch_id`` field
  on the ``request`` instance; if ``request`` is provided, this

google/cloud/dataproc_v1/services/batch_controller/client.py

@@ -400,7 +400,7 @@ def create_batch(
  the final component of the batch's resource name.
 
  This value must be 4-63 characters. Valid characters are
- ``/[a-z][0-9]-/``.
+ /[a-z][0-9]-/.
 
  This corresponds to the ``batch_id`` field
  on the ``request`` instance; if ``request`` is provided, this

google/cloud/dataproc_v1/services/workflow_template_service/async_client.py

@@ -207,7 +207,7 @@ async def create_workflow_template(
  as described in
  https://cloud.google.com/apis/design/resource_names.
 
- - For ``projects.regions.workflowTemplates,create``,
+ - For ``projects.regions.workflowTemplates.create``,
  the resource name of the region has the following
  format: ``projects/{project_id}/regions/{region}``
 

google/cloud/dataproc_v1/services/workflow_template_service/client.py

@@ -422,7 +422,7 @@ def create_workflow_template(
  as described in
  https://cloud.google.com/apis/design/resource_names.
 
- - For ``projects.regions.workflowTemplates,create``,
+ - For ``projects.regions.workflowTemplates.create``,
  the resource name of the region has the following
  format: ``projects/{project_id}/regions/{region}``
 

google/cloud/dataproc_v1/types/batches.py

@@ -50,7 +50,7 @@ class CreateBatchRequest(proto.Message):
  final component of the batch's resource name.
 
  This value must be 4-63 characters. Valid characters are
- ``/[a-z][0-9]-/``.
+ /[a-z][0-9]-/.
  request_id (str):
  Optional. A unique ID used to identify the request. If the
  service receives two

google/cloud/dataproc_v1/types/clusters.py

@@ -75,9 +75,10 @@ class Cluster(proto.Message):
  within a project must be unique. Names of
  deleted clusters can be reused.
  config (google.cloud.dataproc_v1.types.ClusterConfig):
- Required. The cluster config. Note that
- Dataproc may set default values, and values may
- change when clusters are updated.
+ Optional. The cluster config for a cluster of
+ Compute Engine Instances. Note that Dataproc may
+ set default values, and values may change when
+ clusters are updated.
  labels (Sequence[google.cloud.dataproc_v1.types.Cluster.LabelsEntry]):
  Optional. The labels to associate with this cluster. Label
  **keys** must contain 1 to 63 characters, and must conform
@@ -682,11 +683,17 @@ class DiskConfig(proto.Message):
  data. If one or more SSDs are attached, this runtime bulk
  data is spread across them, and the boot disk contains only
  basic config and installed binaries.
+ local_ssd_interface (str):
+ Optional. Interface type of local SSDs (default is "scsi").
+ Valid values: "scsi" (Small Computer System Interface),
+ "nvme" (Non-Volatile Memory Express). See `SSD Interface
+ types <https://cloud.google.com/compute/docs/disks/local-ssd#performance>`__.
  """
 
  boot_disk_type = proto.Field(proto.STRING, number=3,)
  boot_disk_size_gb = proto.Field(proto.INT32, number=1,)
  num_local_ssds = proto.Field(proto.INT32, number=2,)
+ local_ssd_interface = proto.Field(proto.STRING, number=4,)
 
 
 class NodeInitializationAction(proto.Message):

google/cloud/dataproc_v1/types/jobs.py

@@ -814,21 +814,28 @@ class JobScheduling(proto.Message):
 
  Attributes:
  max_failures_per_hour (int):
- Optional. Maximum number of times per hour a
- driver may be restarted as a result of driver
- exiting with non-zero code before job is
- reported failed.
+ Optional. Maximum number of times per hour a driver may be
+ restarted as a result of driver exiting with non-zero code
+ before job is reported failed.
 
- A job may be reported as thrashing if driver
- exits with non-zero code 4 times within 10
- minute window.
+ A job may be reported as thrashing if driver exits with
+ non-zero code 4 times within 10 minute window.
 
  Maximum value is 10.
+
+ **Note:** Currently, this restartable job option is not
+ supported in Dataproc `workflow
+ template <https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#adding_jobs_to_a_template>`__
+ jobs.
  max_failures_total (int):
- Optional. Maximum number of times in total a
- driver may be restarted as a result of driver
- exiting with non-zero code before job is
- reported failed. Maximum value is 240.
+ Optional. Maximum number of times in total a driver may be
+ restarted as a result of driver exiting with non-zero code
+ before job is reported failed. Maximum value is 240.
+
+ **Note:** Currently, this restartable job option is not
+ supported in Dataproc `workflow
+ template <https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#adding_jobs_to_a_template>`__
+ jobs.
  """
 
  max_failures_per_hour = proto.Field(proto.INT32, number=1,)

google/cloud/dataproc_v1/types/shared.py

@@ -61,12 +61,20 @@ class RuntimeConfig(proto.Message):
  r"""Runtime configuration for a workload.
 
  Attributes:
+ version (str):
+ Optional. Version of the batch runtime.
+ container_image (str):
+ Optional. Optional custom container image for
+ the job runtime environment. If not specified, a
+ default container image will be used.
  properties (Sequence[google.cloud.dataproc_v1.types.RuntimeConfig.PropertiesEntry]):
  Optional. A mapping of property names to
  values, which are used to configure workload
  execution.
  """
 
+ version = proto.Field(proto.STRING, number=1,)
+ container_image = proto.Field(proto.STRING, number=2,)
  properties = proto.MapField(proto.STRING, proto.STRING, number=3,)
 
 
@@ -174,10 +182,14 @@ class RuntimeInfo(proto.Message):
  output_uri (str):
  Output only. A URI pointing to the location
  of the stdout and stderr of the workload.
+ diagnostic_output_uri (str):
+ Output only. A URI pointing to the location
+ of the diagnostics tarball.
  """
 
  endpoints = proto.MapField(proto.STRING, proto.STRING, number=1,)
  output_uri = proto.Field(proto.STRING, number=2,)
+ diagnostic_output_uri = proto.Field(proto.STRING, number=3,)
 
 
 __all__ = tuple(sorted(__protobuf__.manifest))

google/cloud/dataproc_v1/types/workflow_templates.py

@@ -626,7 +626,7 @@ class CreateWorkflowTemplateRequest(proto.Message):
  described in
  https://cloud.google.com/apis/design/resource_names.
 
- - For ``projects.regions.workflowTemplates,create``, the
+ - For ``projects.regions.workflowTemplates.create``, the
  resource name of the region has the following format:
  ``projects/{project_id}/regions/{region}``