From f0ed26c6ccd2e9f438d1d5f31c5512761b0e20b9 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 12 Oct 2021 15:56:24 -0600 Subject: [PATCH] feat: add Dataproc Serverless for Spark Batches API (#290) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add Dataproc Serverless for Spark Batches API Committer: @medb PiperOrigin-RevId: 402631995 Source-Link: https://github.com/googleapis/googleapis/commit/95af2e435b5e5ae0a405e2b6a469d12e807d99d0 Source-Link: https://github.com/googleapis/googleapis-gen/commit/0ee7abd9ecd2951e958303681a4b251a948107b6 Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMGVlN2FiZDllY2QyOTUxZTk1ODMwMzY4MWE0YjI1MWE5NDgxMDdiNiJ9 * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- google/cloud/dataproc/__init__.py | 18 +++ google/cloud/dataproc_v1/__init__.py | 18 +++ .../cluster_controller/async_client.py | 13 +- .../services/cluster_controller/client.py | 13 +- .../cluster_controller/transports/grpc.py | 3 + .../transports/grpc_asyncio.py | 3 + google/cloud/dataproc_v1/types/__init__.py | 21 +++ .../dataproc_v1/types/autoscaling_policies.py | 10 ++ google/cloud/dataproc_v1/types/clusters.py | 75 +++++++--- google/cloud/dataproc_v1/types/jobs.py | 4 +- google/cloud/dataproc_v1/types/operations.py | 44 +++++- google/cloud/dataproc_v1/types/shared.py | 137 +++++++++++++++++- scripts/fixup_dataproc_v1_keywords.py | 2 +- 13 files changed, 321 insertions(+), 40 deletions(-) diff --git a/google/cloud/dataproc/__init__.py b/google/cloud/dataproc/__init__.py index 90d84a64..f6c49af9 100644 --- a/google/cloud/dataproc/__init__.py +++ b/google/cloud/dataproc/__init__.py @@ -71,6 +71,7 @@ from google.cloud.dataproc_v1.types.clusters import ClusterConfig from google.cloud.dataproc_v1.types.clusters import ClusterMetrics from google.cloud.dataproc_v1.types.clusters import ClusterStatus +from google.cloud.dataproc_v1.types.clusters import ConfidentialInstanceConfig from google.cloud.dataproc_v1.types.clusters import CreateClusterRequest from google.cloud.dataproc_v1.types.clusters import DeleteClusterRequest from google.cloud.dataproc_v1.types.clusters import DiagnoseClusterRequest @@ -122,9 +123,17 @@ from google.cloud.dataproc_v1.types.jobs import SubmitJobRequest from google.cloud.dataproc_v1.types.jobs import UpdateJobRequest from google.cloud.dataproc_v1.types.jobs import YarnApplication +from google.cloud.dataproc_v1.types.operations import BatchOperationMetadata from google.cloud.dataproc_v1.types.operations import ClusterOperationMetadata from google.cloud.dataproc_v1.types.operations import ClusterOperationStatus +from google.cloud.dataproc_v1.types.shared import EnvironmentConfig +from google.cloud.dataproc_v1.types.shared import ExecutionConfig +from google.cloud.dataproc_v1.types.shared import PeripheralsConfig +from google.cloud.dataproc_v1.types.shared import RuntimeConfig +from google.cloud.dataproc_v1.types.shared import RuntimeInfo +from google.cloud.dataproc_v1.types.shared import SparkHistoryServerConfig from google.cloud.dataproc_v1.types.shared import Component +from google.cloud.dataproc_v1.types.shared import FailureAction from google.cloud.dataproc_v1.types.workflow_templates import ClusterOperation from google.cloud.dataproc_v1.types.workflow_templates import ClusterSelector from google.cloud.dataproc_v1.types.workflow_templates import ( @@ -186,6 +195,7 @@ "ClusterConfig", "ClusterMetrics", "ClusterStatus", + "ConfidentialInstanceConfig", "CreateClusterRequest", "DeleteClusterRequest", "DiagnoseClusterRequest", @@ -237,9 +247,17 @@ "SubmitJobRequest", "UpdateJobRequest", "YarnApplication", + "BatchOperationMetadata", "ClusterOperationMetadata", "ClusterOperationStatus", + "EnvironmentConfig", + "ExecutionConfig", + "PeripheralsConfig", + "RuntimeConfig", + "RuntimeInfo", + "SparkHistoryServerConfig", "Component", + "FailureAction", "ClusterOperation", "ClusterSelector", "CreateWorkflowTemplateRequest", diff --git a/google/cloud/dataproc_v1/__init__.py b/google/cloud/dataproc_v1/__init__.py index 22af0899..b5dc3e25 100644 --- a/google/cloud/dataproc_v1/__init__.py +++ b/google/cloud/dataproc_v1/__init__.py @@ -39,6 +39,7 @@ from .types.clusters import ClusterConfig from .types.clusters import ClusterMetrics from .types.clusters import ClusterStatus +from .types.clusters import ConfidentialInstanceConfig from .types.clusters import CreateClusterRequest from .types.clusters import DeleteClusterRequest from .types.clusters import DiagnoseClusterRequest @@ -90,9 +91,17 @@ from .types.jobs import SubmitJobRequest from .types.jobs import UpdateJobRequest from .types.jobs import YarnApplication +from .types.operations import BatchOperationMetadata from .types.operations import ClusterOperationMetadata from .types.operations import ClusterOperationStatus +from .types.shared import EnvironmentConfig +from .types.shared import ExecutionConfig +from .types.shared import PeripheralsConfig +from .types.shared import RuntimeConfig +from .types.shared import RuntimeInfo +from .types.shared import SparkHistoryServerConfig from .types.shared import Component +from .types.shared import FailureAction from .types.workflow_templates import ClusterOperation from .types.workflow_templates import ClusterSelector from .types.workflow_templates import CreateWorkflowTemplateRequest @@ -126,6 +135,7 @@ "AutoscalingPolicyServiceClient", "BasicAutoscalingAlgorithm", "BasicYarnAutoscalingConfig", + "BatchOperationMetadata", "CancelJobRequest", "Cluster", "ClusterConfig", @@ -137,6 +147,7 @@ "ClusterSelector", "ClusterStatus", "Component", + "ConfidentialInstanceConfig", "CreateAutoscalingPolicyRequest", "CreateClusterRequest", "CreateWorkflowTemplateRequest", @@ -149,6 +160,9 @@ "DiskConfig", "EncryptionConfig", "EndpointConfig", + "EnvironmentConfig", + "ExecutionConfig", + "FailureAction", "GceClusterConfig", "GetAutoscalingPolicyRequest", "GetClusterRequest", @@ -187,15 +201,19 @@ "NodeInitializationAction", "OrderedJob", "ParameterValidation", + "PeripheralsConfig", "PigJob", "PrestoJob", "PySparkJob", "QueryList", "RegexValidation", "ReservationAffinity", + "RuntimeConfig", + "RuntimeInfo", "SecurityConfig", "ShieldedInstanceConfig", "SoftwareConfig", + "SparkHistoryServerConfig", "SparkJob", "SparkRJob", "SparkSqlJob", diff --git a/google/cloud/dataproc_v1/services/cluster_controller/async_client.py b/google/cloud/dataproc_v1/services/cluster_controller/async_client.py index 57ca5e8a..36ab734a 100644 --- a/google/cloud/dataproc_v1/services/cluster_controller/async_client.py +++ b/google/cloud/dataproc_v1/services/cluster_controller/async_client.py @@ -220,7 +220,7 @@ async def create_cluster( An object representing a long-running operation. The result type for the operation will be :class:`google.cloud.dataproc_v1.types.Cluster` Describes the identifying information, config, and status of - a cluster of Compute Engine instances. + a Dataproc cluster """ # Create or coerce a protobuf request object. @@ -292,6 +292,9 @@ async def update_cluster( [Operation.metadata][google.longrunning.Operation.metadata] will be `ClusterOperationMetadata `__. + The cluster must be in a + [``RUNNING``][google.cloud.dataproc.v1.ClusterStatus.State] + state or an error is returned. Args: request (:class:`google.cloud.dataproc_v1.types.UpdateClusterRequest`): @@ -398,7 +401,7 @@ async def update_cluster( An object representing a long-running operation. The result type for the operation will be :class:`google.cloud.dataproc_v1.types.Cluster` Describes the identifying information, config, and status of - a cluster of Compute Engine instances. + a Dataproc cluster """ # Create or coerce a protobuf request object. @@ -483,7 +486,7 @@ async def stop_cluster( An object representing a long-running operation. The result type for the operation will be :class:`google.cloud.dataproc_v1.types.Cluster` Describes the identifying information, config, and status of - a cluster of Compute Engine instances. + a Dataproc cluster """ # Create or coerce a protobuf request object. @@ -535,7 +538,7 @@ async def start_cluster( An object representing a long-running operation. The result type for the operation will be :class:`google.cloud.dataproc_v1.types.Cluster` Describes the identifying information, config, and status of - a cluster of Compute Engine instances. + a Dataproc cluster """ # Create or coerce a protobuf request object. @@ -727,7 +730,7 @@ async def get_cluster( google.cloud.dataproc_v1.types.Cluster: Describes the identifying information, config, and status of a - cluster of Compute Engine instances. + Dataproc cluster """ # Create or coerce a protobuf request object. diff --git a/google/cloud/dataproc_v1/services/cluster_controller/client.py b/google/cloud/dataproc_v1/services/cluster_controller/client.py index f6659382..adc835bb 100644 --- a/google/cloud/dataproc_v1/services/cluster_controller/client.py +++ b/google/cloud/dataproc_v1/services/cluster_controller/client.py @@ -422,7 +422,7 @@ def create_cluster( An object representing a long-running operation. The result type for the operation will be :class:`google.cloud.dataproc_v1.types.Cluster` Describes the identifying information, config, and status of - a cluster of Compute Engine instances. + a Dataproc cluster """ # Create or coerce a protobuf request object. @@ -485,6 +485,9 @@ def update_cluster( [Operation.metadata][google.longrunning.Operation.metadata] will be `ClusterOperationMetadata `__. + The cluster must be in a + [``RUNNING``][google.cloud.dataproc.v1.ClusterStatus.State] + state or an error is returned. Args: request (Union[google.cloud.dataproc_v1.types.UpdateClusterRequest, dict]): @@ -591,7 +594,7 @@ def update_cluster( An object representing a long-running operation. The result type for the operation will be :class:`google.cloud.dataproc_v1.types.Cluster` Describes the identifying information, config, and status of - a cluster of Compute Engine instances. + a Dataproc cluster """ # Create or coerce a protobuf request object. @@ -667,7 +670,7 @@ def stop_cluster( An object representing a long-running operation. The result type for the operation will be :class:`google.cloud.dataproc_v1.types.Cluster` Describes the identifying information, config, and status of - a cluster of Compute Engine instances. + a Dataproc cluster """ # Create or coerce a protobuf request object. @@ -720,7 +723,7 @@ def start_cluster( An object representing a long-running operation. The result type for the operation will be :class:`google.cloud.dataproc_v1.types.Cluster` Describes the identifying information, config, and status of - a cluster of Compute Engine instances. + a Dataproc cluster """ # Create or coerce a protobuf request object. @@ -904,7 +907,7 @@ def get_cluster( google.cloud.dataproc_v1.types.Cluster: Describes the identifying information, config, and status of a - cluster of Compute Engine instances. + Dataproc cluster """ # Create or coerce a protobuf request object. diff --git a/google/cloud/dataproc_v1/services/cluster_controller/transports/grpc.py b/google/cloud/dataproc_v1/services/cluster_controller/transports/grpc.py index 19283747..9c6f2b9a 100644 --- a/google/cloud/dataproc_v1/services/cluster_controller/transports/grpc.py +++ b/google/cloud/dataproc_v1/services/cluster_controller/transports/grpc.py @@ -282,6 +282,9 @@ def update_cluster( [Operation.metadata][google.longrunning.Operation.metadata] will be `ClusterOperationMetadata `__. + The cluster must be in a + [``RUNNING``][google.cloud.dataproc.v1.ClusterStatus.State] + state or an error is returned. Returns: Callable[[~.UpdateClusterRequest], diff --git a/google/cloud/dataproc_v1/services/cluster_controller/transports/grpc_asyncio.py b/google/cloud/dataproc_v1/services/cluster_controller/transports/grpc_asyncio.py index b3def8d7..555486fc 100644 --- a/google/cloud/dataproc_v1/services/cluster_controller/transports/grpc_asyncio.py +++ b/google/cloud/dataproc_v1/services/cluster_controller/transports/grpc_asyncio.py @@ -287,6 +287,9 @@ def update_cluster( [Operation.metadata][google.longrunning.Operation.metadata] will be `ClusterOperationMetadata `__. + The cluster must be in a + [``RUNNING``][google.cloud.dataproc.v1.ClusterStatus.State] + state or an error is returned. Returns: Callable[[~.UpdateClusterRequest], diff --git a/google/cloud/dataproc_v1/types/__init__.py b/google/cloud/dataproc_v1/types/__init__.py index 814a8c19..7d7a9820 100644 --- a/google/cloud/dataproc_v1/types/__init__.py +++ b/google/cloud/dataproc_v1/types/__init__.py @@ -32,6 +32,7 @@ ClusterConfig, ClusterMetrics, ClusterStatus, + ConfidentialInstanceConfig, CreateClusterRequest, DeleteClusterRequest, DiagnoseClusterRequest, @@ -87,9 +88,20 @@ YarnApplication, ) from .operations import ( + BatchOperationMetadata, ClusterOperationMetadata, ClusterOperationStatus, ) +from .shared import ( + EnvironmentConfig, + ExecutionConfig, + PeripheralsConfig, + RuntimeConfig, + RuntimeInfo, + SparkHistoryServerConfig, + Component, + FailureAction, +) from .workflow_templates import ( ClusterOperation, ClusterSelector, @@ -131,6 +143,7 @@ "ClusterConfig", "ClusterMetrics", "ClusterStatus", + "ConfidentialInstanceConfig", "CreateClusterRequest", "DeleteClusterRequest", "DiagnoseClusterRequest", @@ -182,9 +195,17 @@ "SubmitJobRequest", "UpdateJobRequest", "YarnApplication", + "BatchOperationMetadata", "ClusterOperationMetadata", "ClusterOperationStatus", + "EnvironmentConfig", + "ExecutionConfig", + "PeripheralsConfig", + "RuntimeConfig", + "RuntimeInfo", + "SparkHistoryServerConfig", "Component", + "FailureAction", "ClusterOperation", "ClusterSelector", "CreateWorkflowTemplateRequest", diff --git a/google/cloud/dataproc_v1/types/autoscaling_policies.py b/google/cloud/dataproc_v1/types/autoscaling_policies.py index 2b10f9cc..0b590fe1 100644 --- a/google/cloud/dataproc_v1/types/autoscaling_policies.py +++ b/google/cloud/dataproc_v1/types/autoscaling_policies.py @@ -67,6 +67,15 @@ class AutoscalingPolicy(proto.Message): secondary_worker_config (google.cloud.dataproc_v1.types.InstanceGroupAutoscalingPolicyConfig): Optional. Describes how the autoscaler will operate for secondary workers. + labels (Sequence[google.cloud.dataproc_v1.types.AutoscalingPolicy.LabelsEntry]): + Optional. The labels to associate with this autoscaling + policy. Label **keys** must contain 1 to 63 characters, and + must conform to `RFC + 1035 `__. Label + **values** may be empty, but, if present, must contain 1 to + 63 characters, and must conform to `RFC + 1035 `__. No more than + 32 labels can be associated with an autoscaling policy. """ id = proto.Field(proto.STRING, number=1,) @@ -80,6 +89,7 @@ class AutoscalingPolicy(proto.Message): secondary_worker_config = proto.Field( proto.MESSAGE, number=5, message="InstanceGroupAutoscalingPolicyConfig", ) + labels = proto.MapField(proto.STRING, proto.STRING, number=6,) class BasicAutoscalingAlgorithm(proto.Message): diff --git a/google/cloud/dataproc_v1/types/clusters.py b/google/cloud/dataproc_v1/types/clusters.py index 4e52ca9f..a1634912 100644 --- a/google/cloud/dataproc_v1/types/clusters.py +++ b/google/cloud/dataproc_v1/types/clusters.py @@ -33,6 +33,7 @@ "GceClusterConfig", "NodeGroupAffinity", "ShieldedInstanceConfig", + "ConfidentialInstanceConfig", "InstanceGroupConfig", "ManagedGroupConfig", "AcceleratorConfig", @@ -63,7 +64,7 @@ class Cluster(proto.Message): r"""Describes the identifying information, config, and status of - a cluster of Compute Engine instances. + a Dataproc cluster Attributes: project_id (str): @@ -125,10 +126,10 @@ class ClusterConfig(proto.Message): your cluster's staging bucket according to the Compute Engine zone where your cluster is deployed, and then create and manage this project-level, per-location bucket (see - `Dataproc staging - bucket `__). - **This field requires a Cloud Storage bucket name, not a URI - to a Cloud Storage bucket.** + `Dataproc staging and temp + buckets `__). + **This field requires a Cloud Storage bucket name, not a + ``gs://...`` URI to a Cloud Storage bucket.** temp_bucket (str): Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data, such as Spark and MapReduce history @@ -138,23 +139,26 @@ class ClusterConfig(proto.Message): zone where your cluster is deployed, and then create and manage this project-level, per-location bucket. The default bucket has a TTL of 90 days, but you can use any TTL (or - none) if you specify a bucket. **This field requires a Cloud - Storage bucket name, not a URI to a Cloud Storage bucket.** + none) if you specify a bucket (see `Dataproc staging and + temp + buckets `__). + **This field requires a Cloud Storage bucket name, not a + ``gs://...`` URI to a Cloud Storage bucket.** gce_cluster_config (google.cloud.dataproc_v1.types.GceClusterConfig): Optional. The shared Compute Engine config settings for all instances in a cluster. master_config (google.cloud.dataproc_v1.types.InstanceGroupConfig): Optional. The Compute Engine config settings - for the master instance in a cluster. + for the cluster's master instance. worker_config (google.cloud.dataproc_v1.types.InstanceGroupConfig): Optional. The Compute Engine config settings - for worker instances in a cluster. + for the cluster's worker instances. secondary_worker_config (google.cloud.dataproc_v1.types.InstanceGroupConfig): Optional. The Compute Engine config settings - for additional worker instances in a cluster. + for a cluster's secondary worker instances software_config (google.cloud.dataproc_v1.types.SoftwareConfig): - Optional. The config settings for software - inside the cluster. + Optional. The config settings for cluster + software. initialization_actions (Sequence[google.cloud.dataproc_v1.types.NodeInitializationAction]): Optional. Commands to execute on each node after config is completed. By default, executables are run on master and all @@ -404,6 +408,10 @@ class GceClusterConfig(proto.Message): Optional. Shielded Instance Config for clusters using `Compute Engine Shielded VMs `__. + confidential_instance_config (google.cloud.dataproc_v1.types.ConfidentialInstanceConfig): + Optional. Confidential Instance Config for clusters using + `Confidential + VMs `__. """ class PrivateIpv6GoogleAccess(proto.Enum): @@ -438,6 +446,9 @@ class PrivateIpv6GoogleAccess(proto.Enum): shielded_instance_config = proto.Field( proto.MESSAGE, number=14, message="ShieldedInstanceConfig", ) + confidential_instance_config = proto.Field( + proto.MESSAGE, number=15, message="ConfidentialInstanceConfig", + ) class NodeGroupAffinity(proto.Message): @@ -482,6 +493,19 @@ class ShieldedInstanceConfig(proto.Message): enable_integrity_monitoring = proto.Field(proto.BOOL, number=3,) +class ConfidentialInstanceConfig(proto.Message): + r"""Confidential Instance Config for clusters using `Confidential + VMs `__ + + Attributes: + enable_confidential_compute (bool): + Optional. Defines whether the instance should + have confidential compute enabled. + """ + + enable_confidential_compute = proto.Field(proto.BOOL, number=1,) + + class InstanceGroupConfig(proto.Message): r"""The config settings for Compute Engine resources in an instance group, such as a master or worker group. @@ -714,6 +738,7 @@ class State(proto.Enum): CREATING = 1 RUNNING = 2 ERROR = 3 + ERROR_DUE_TO_UPDATE = 9 DELETING = 4 UPDATING = 5 STOPPING = 6 @@ -987,7 +1012,7 @@ class CreateClusterRequest(proto.Message): cluster (google.cloud.dataproc_v1.types.Cluster): Required. The cluster to create. request_id (str): - Optional. A unique id used to identify the request. If the + Optional. A unique ID used to identify the request. If the server receives two `CreateClusterRequest `__\ s with the same id, then the second request will be ignored @@ -998,15 +1023,21 @@ class CreateClusterRequest(proto.Message): It is recommended to always set this value to a `UUID `__. - The id must contain only letters (a-z, A-Z), numbers (0-9), + The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). The maximum length is 40 characters. + action_on_failed_primary_workers (google.cloud.dataproc_v1.types.FailureAction): + Optional. Failure action when primary worker + creation fails. """ project_id = proto.Field(proto.STRING, number=1,) region = proto.Field(proto.STRING, number=3,) cluster = proto.Field(proto.MESSAGE, number=2, message="Cluster",) request_id = proto.Field(proto.STRING, number=4,) + action_on_failed_primary_workers = proto.Field( + proto.ENUM, number=5, enum=shared.FailureAction, + ) class UpdateClusterRequest(proto.Message): @@ -1097,7 +1128,7 @@ class UpdateClusterRequest(proto.Message): request_id (str): - Optional. A unique id used to identify the request. If the + Optional. A unique ID used to identify the request. If the server receives two `UpdateClusterRequest `__\ s with the same id, then the second request will be ignored @@ -1108,7 +1139,7 @@ class UpdateClusterRequest(proto.Message): It is recommended to always set this value to a `UUID `__. - The id must contain only letters (a-z, A-Z), numbers (0-9), + The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). The maximum length is 40 characters. """ @@ -1143,7 +1174,7 @@ class StopClusterRequest(proto.Message): fail (with error NOT_FOUND) if a cluster with the specified UUID does not exist. request_id (str): - Optional. A unique id used to identify the request. If the + Optional. A unique ID used to identify the request. If the server receives two `StopClusterRequest `__\ s with the same id, then the second request will be ignored @@ -1154,7 +1185,7 @@ class StopClusterRequest(proto.Message): Recommendation: Set this value to a `UUID `__. - The id must contain only letters (a-z, A-Z), numbers (0-9), + The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). The maximum length is 40 characters. """ @@ -1183,7 +1214,7 @@ class StartClusterRequest(proto.Message): fail (with error NOT_FOUND) if a cluster with the specified UUID does not exist. request_id (str): - Optional. A unique id used to identify the request. If the + Optional. A unique ID used to identify the request. If the server receives two `StartClusterRequest `__\ s with the same id, then the second request will be ignored @@ -1194,7 +1225,7 @@ class StartClusterRequest(proto.Message): Recommendation: Set this value to a `UUID `__. - The id must contain only letters (a-z, A-Z), numbers (0-9), + The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). The maximum length is 40 characters. """ @@ -1223,7 +1254,7 @@ class DeleteClusterRequest(proto.Message): should fail (with error NOT_FOUND) if cluster with specified UUID does not exist. request_id (str): - Optional. A unique id used to identify the request. If the + Optional. A unique ID used to identify the request. If the server receives two `DeleteClusterRequest `__\ s with the same id, then the second request will be ignored @@ -1234,7 +1265,7 @@ class DeleteClusterRequest(proto.Message): It is recommended to always set this value to a `UUID `__. - The id must contain only letters (a-z, A-Z), numbers (0-9), + The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). The maximum length is 40 characters. """ diff --git a/google/cloud/dataproc_v1/types/jobs.py b/google/cloud/dataproc_v1/types/jobs.py index 5eece260..0acc8554 100644 --- a/google/cloud/dataproc_v1/types/jobs.py +++ b/google/cloud/dataproc_v1/types/jobs.py @@ -64,8 +64,8 @@ class LoggingConfig(proto.Message): class Level(proto.Enum): r"""The Log4j level for job execution. When running an `Apache - Hive `__ job, Cloud Dataproc configures the - Hive client to an equivalent verbosity level. + Hive `__ job, Cloud Dataproc configures + the Hive client to an equivalent verbosity level. """ LEVEL_UNSPECIFIED = 0 ALL = 1 diff --git a/google/cloud/dataproc_v1/types/operations.py b/google/cloud/dataproc_v1/types/operations.py index bb04624d..03721bc1 100644 --- a/google/cloud/dataproc_v1/types/operations.py +++ b/google/cloud/dataproc_v1/types/operations.py @@ -20,10 +20,52 @@ __protobuf__ = proto.module( package="google.cloud.dataproc.v1", - manifest={"ClusterOperationStatus", "ClusterOperationMetadata",}, + manifest={ + "BatchOperationMetadata", + "ClusterOperationStatus", + "ClusterOperationMetadata", + }, ) +class BatchOperationMetadata(proto.Message): + r"""Metadata describing the Batch operation. + + Attributes: + batch (str): + Name of the batch for the operation. + batch_uuid (str): + Batch UUID for the operation. + create_time (google.protobuf.timestamp_pb2.Timestamp): + The time when the operation was created. + done_time (google.protobuf.timestamp_pb2.Timestamp): + The time when the operation finished. + operation_type (google.cloud.dataproc_v1.types.BatchOperationMetadata.BatchOperationType): + The operation type. + description (str): + Short description of the operation. + labels (Sequence[google.cloud.dataproc_v1.types.BatchOperationMetadata.LabelsEntry]): + Labels associated with the operation. + warnings (Sequence[str]): + Warnings encountered during operation + execution. + """ + + class BatchOperationType(proto.Enum): + r"""Operation type for Batch resources""" + BATCH_OPERATION_TYPE_UNSPECIFIED = 0 + BATCH = 1 + + batch = proto.Field(proto.STRING, number=1,) + batch_uuid = proto.Field(proto.STRING, number=2,) + create_time = proto.Field(proto.MESSAGE, number=3, message=timestamp_pb2.Timestamp,) + done_time = proto.Field(proto.MESSAGE, number=4, message=timestamp_pb2.Timestamp,) + operation_type = proto.Field(proto.ENUM, number=6, enum=BatchOperationType,) + description = proto.Field(proto.STRING, number=7,) + labels = proto.MapField(proto.STRING, proto.STRING, number=8,) + warnings = proto.RepeatedField(proto.STRING, number=9,) + + class ClusterOperationStatus(proto.Message): r"""The status of the operation. diff --git a/google/cloud/dataproc_v1/types/shared.py b/google/cloud/dataproc_v1/types/shared.py index 2e397004..d4d4b616 100644 --- a/google/cloud/dataproc_v1/types/shared.py +++ b/google/cloud/dataproc_v1/types/shared.py @@ -17,14 +17,22 @@ __protobuf__ = proto.module( - package="google.cloud.dataproc.v1", manifest={"Component",}, + package="google.cloud.dataproc.v1", + manifest={ + "Component", + "FailureAction", + "RuntimeConfig", + "EnvironmentConfig", + "ExecutionConfig", + "SparkHistoryServerConfig", + "PeripheralsConfig", + "RuntimeInfo", + }, ) class Component(proto.Enum): - r"""Cluster components that can be activated. - Next ID: 16. - """ + r"""Cluster components that can be activated.""" COMPONENT_UNSPECIFIED = 0 ANACONDA = 5 DOCKER = 13 @@ -40,4 +48,125 @@ class Component(proto.Enum): ZOOKEEPER = 8 +class FailureAction(proto.Enum): + r"""Actions in response to failure of a resource associated with + a cluster. + """ + FAILURE_ACTION_UNSPECIFIED = 0 + NO_ACTION = 1 + DELETE = 2 + + +class RuntimeConfig(proto.Message): + r"""Runtime configuration for a workload. + + Attributes: + properties (Sequence[google.cloud.dataproc_v1.types.RuntimeConfig.PropertiesEntry]): + Optional. A mapping of property names to + values, which are used to configure workload + execution. + """ + + properties = proto.MapField(proto.STRING, proto.STRING, number=3,) + + +class EnvironmentConfig(proto.Message): + r"""Environment configuration for a workload. + + Attributes: + execution_config (google.cloud.dataproc_v1.types.ExecutionConfig): + Optional. Execution configuration for a + workload. + peripherals_config (google.cloud.dataproc_v1.types.PeripheralsConfig): + Optional. Peripherals configuration that + workload has access to. + """ + + execution_config = proto.Field(proto.MESSAGE, number=1, message="ExecutionConfig",) + peripherals_config = proto.Field( + proto.MESSAGE, number=2, message="PeripheralsConfig", + ) + + +class ExecutionConfig(proto.Message): + r"""Execution configuration for a workload. + + Attributes: + service_account (str): + Optional. Service account that used to + execute workload. + network_uri (str): + Optional. Network URI to connect workload to. + subnetwork_uri (str): + Optional. Subnetwork URI to connect workload + to. + network_tags (Sequence[str]): + Optional. Tags used for network traffic + control. + kms_key (str): + Optional. The Cloud KMS key to use for + encryption. + """ + + service_account = proto.Field(proto.STRING, number=2,) + network_uri = proto.Field(proto.STRING, number=4, oneof="network",) + subnetwork_uri = proto.Field(proto.STRING, number=5, oneof="network",) + network_tags = proto.RepeatedField(proto.STRING, number=6,) + kms_key = proto.Field(proto.STRING, number=7,) + + +class SparkHistoryServerConfig(proto.Message): + r"""Spark History Server configuration for the workload. + + Attributes: + dataproc_cluster (str): + Optional. Resource name of an existing Dataproc Cluster to + act as a Spark History Server for the workload. + + Example: + + - ``projects/[project_id]/regions/[region]/clusters/[cluster_name]`` + """ + + dataproc_cluster = proto.Field(proto.STRING, number=1,) + + +class PeripheralsConfig(proto.Message): + r"""Auxiliary services configuration for a workload. + + Attributes: + metastore_service (str): + Optional. Resource name of an existing Dataproc Metastore + service. + + Example: + + - ``projects/[project_id]/locations/[region]/services/[service_id]`` + spark_history_server_config (google.cloud.dataproc_v1.types.SparkHistoryServerConfig): + Optional. The Spark History Server + configuration for the workload. + """ + + metastore_service = proto.Field(proto.STRING, number=1,) + spark_history_server_config = proto.Field( + proto.MESSAGE, number=2, message="SparkHistoryServerConfig", + ) + + +class RuntimeInfo(proto.Message): + r"""Runtime information about workload execution. + + Attributes: + endpoints (Sequence[google.cloud.dataproc_v1.types.RuntimeInfo.EndpointsEntry]): + Output only. Map of remote access endpoints + (such as web interfaces and APIs) to their URIs. + output_uri (str): + Output only. A URI pointing to the location + of the stdout and stderr of the workload. + """ + + endpoints = proto.MapField(proto.STRING, proto.STRING, number=1,) + output_uri = proto.Field(proto.STRING, number=2,) + + __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/scripts/fixup_dataproc_v1_keywords.py b/scripts/fixup_dataproc_v1_keywords.py index c5d79a7f..64c133a6 100644 --- a/scripts/fixup_dataproc_v1_keywords.py +++ b/scripts/fixup_dataproc_v1_keywords.py @@ -41,7 +41,7 @@ class dataprocCallTransformer(cst.CSTTransformer): METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { 'cancel_job': ('project_id', 'region', 'job_id', ), 'create_autoscaling_policy': ('parent', 'policy', ), - 'create_cluster': ('project_id', 'region', 'cluster', 'request_id', ), + 'create_cluster': ('project_id', 'region', 'cluster', 'request_id', 'action_on_failed_primary_workers', ), 'create_workflow_template': ('parent', 'template', ), 'delete_autoscaling_policy': ('name', ), 'delete_cluster': ('project_id', 'region', 'cluster_name', 'cluster_uuid', 'request_id', ),