docs: update cluster sample #218
Changes from all commits
11cf08a
205a4d0
7b62aa7
dac9663
632fd09
5be2ed9
cd25bf8
9a80e3d
b48cbb3
853cf21
0bba122
9ac4e88
2412e9c
47b7639
aa93d29
eaacde5
8c51ca8
dfb5f19
6e0c6db
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
# Copyright 2021 Google LLC | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Double check the license is correct here. You can take it from here and replace the year with 2021. |
||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
# This sample walks a user through updating the number of clusters using the Dataproc | ||
# client library. | ||
|
||
# Usage: | ||
# python update_cluster.py --project_id <PROJECT_ID> --region <REGION> --cluster_name <CLUSTER_NAME> | ||
|
||
import sys | ||
|
||
# [START dataproc_update_cluster] | ||
from google.cloud import dataproc_v1 as dataproc | ||
|
||
|
||
def update_cluster(project_id, region, cluster_name, new_num_instances): | ||
"""This sample walks a user through updating a Cloud Dataproc cluster | ||
using the Python client library. | ||
|
||
Args: | ||
loferris marked this conversation as resolved.
Show resolved
Hide resolved
|
||
project_id (str): Project to use for creating resources. | ||
region (str): Region where the resources should live. | ||
cluster_name (str): Name to use for creating a cluster. | ||
""" | ||
|
||
# Create a client with the endpoint set to the desired cluster region. | ||
client = dataproc.ClusterControllerClient( | ||
client_options={"api_endpoint": f"{region}-dataproc.googleapis.com:443"} | ||
) | ||
|
||
# Get cluster you wish to update. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This may be a stylistic thing but I would recommend a line break before each new comment. |
||
cluster = client.get_cluster( | ||
project_id=project_id, region=region, cluster_name=cluster_name | ||
) | ||
|
||
# Update number of clusters | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Line break, see above. |
||
mask = {"paths": {"config.worker_config.num_instances": str(new_num_instances)}} | ||
|
||
# Update cluster config | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Line break, see above. |
||
cluster.config.worker_config.num_instances = new_num_instances | ||
|
||
# Update cluster | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Line break, see above. |
||
operation = client.update_cluster( | ||
project_id=project_id, | ||
region=region, | ||
cluster=cluster, | ||
cluster_name=cluster_name, | ||
update_mask=mask, | ||
) | ||
|
||
# Output a success message. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Line break, see above. |
||
updated_cluster = operation.result() | ||
print(f"Cluster was updated successfully: {updated_cluster.cluster_name}") | ||
|
||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Think you're missing the END region tag here. |
||
# [END dataproc_update_cluster] | ||
|
||
|
||
if __name__ == "__main__": | ||
if len(sys.argv) < 5: | ||
sys.exit("python update_cluster.py project_id region cluster_name") | ||
|
||
project_id = sys.argv[1] | ||
region = sys.argv[2] | ||
cluster_name = sys.argv[3] | ||
new_num_instances = sys.argv[4] | ||
update_cluster(project_id, region, cluster_name) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,80 @@ | ||
# Copyright 2021 Google LLC | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
# This sample walks a user through updating the number of clusters using the Dataproc | ||
# client library. | ||
|
||
|
||
import os | ||
import uuid | ||
|
||
from google.cloud.dataproc_v1.services.cluster_controller.client import ( | ||
ClusterControllerClient, | ||
) | ||
import pytest | ||
|
||
import update_cluster | ||
|
||
|
||
PROJECT_ID = os.environ["GOOGLE_CLOUD_PROJECT"] | ||
REGION = "us-central1" | ||
CLUSTER_NAME = f"py-cc-test-{str(uuid.uuid4())}" | ||
NEW_NUM_INSTANCES = 5 | ||
CLUSTER = { | ||
"project_id": PROJECT_ID, | ||
"cluster_name": CLUSTER_NAME, | ||
"config": { | ||
"master_config": {"num_instances": 1, "machine_type_uri": "n1-standard-2"}, | ||
"worker_config": {"num_instances": 2, "machine_type_uri": "n1-standard-2"}, | ||
}, | ||
} | ||
|
||
|
||
@pytest.fixture(autouse=True) | ||
def setup_teardown(cluster_client): | ||
# Create the cluster. | ||
operation = cluster_client.create_cluster( | ||
request={"project_id": PROJECT_ID, "region": REGION, "cluster": CLUSTER} | ||
) | ||
operation.result() | ||
|
||
yield | ||
|
||
cluster_client.delete_cluster( | ||
request={ | ||
"project_id": PROJECT_ID, | ||
"region": REGION, | ||
"cluster_name": CLUSTER_NAME, | ||
} | ||
) | ||
|
||
|
||
@pytest.fixture | ||
def cluster_client(): | ||
cluster_client = ClusterControllerClient( | ||
client_options={"api_endpoint": "{}-dataproc.googleapis.com:443".format(REGION)} | ||
) | ||
return cluster_client | ||
|
||
|
||
def test_update_cluster(capsys, cluster_client: ClusterControllerClient): | ||
# Wrapper function for client library function | ||
update_cluster.update_cluster(PROJECT_ID, REGION, CLUSTER_NAME, NEW_NUM_INSTANCES) | ||
new_num_cluster = cluster_client.get_cluster( | ||
project_id=PROJECT_ID, region=REGION, cluster_name=CLUSTER_NAME | ||
) | ||
|
||
out, _ = capsys.readouterr() | ||
assert CLUSTER_NAME in out | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For the test, I would instead suggest a get_cluster call and confirm that the number of instances matches There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've been calling There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmmm, the following ended up working for me when pointed to a cluster that I have with two workers. Did you also provide the region / cluster_name to
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @bradmiro I haven't been able to get it working in the context of a test. My recent commit has three options I've tried: creating a fixture, creating a static cluster, and creating the cluster in the test. I still can't get in deeper than There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Actually does seem to be working in the current commit in CI! There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. final commit ready for review! |
||
assert new_num_cluster.config.worker_config.num_instances == NEW_NUM_INSTANCES |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No need to block this PR on this, but there's an easier way to set regional endpoints now through
google.api_core.client_options
. See this speech sample: https://cloud.google.com/speech-to-text/docs/endpoints