Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Allow newline and quotes for BQ dataset and table descriptions #103

Merged
merged 2 commits into from Jun 24, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion templates/terraform/google_bigquery_dataset.tf.jinja2
Expand Up @@ -22,7 +22,7 @@ resource "google_bigquery_dataset" "{{ dataset_id }}" {
friendly_name = "{{ friendly_name }}"
{% endif -%}
{% if description -%}
description = "{{ description }}"
description = {{ description|tojson }}
{% endif -%}
{% if location -%}
location = "{{ location }}"
Expand Down
2 changes: 1 addition & 1 deletion templates/terraform/google_bigquery_table.tf.jinja2
Expand Up @@ -21,7 +21,7 @@ resource "google_bigquery_table" "{{ tf_resource_name }}" {
table_id = "{{ table_id }}"

{% if description -%}
description = "{{ description }}"
description = {{ description|tojson }}
{%- endif %}
{% if schema -%}
schema = <<EOF
Expand Down
75 changes: 75 additions & 0 deletions tests/scripts/test_generate_terraform.py
Expand Up @@ -454,6 +454,44 @@ def test_dataset_tf_file_contains_description_when_specified(
assert re.search(r"description\s+\=", result.group(1))


def test_bq_dataset_can_have_a_description_with_newlines_and_quotes(
dataset_path,
pipeline_path,
project_id,
bucket_name_prefix,
region,
impersonating_acct,
env,
):
shutil.copyfile(SAMPLE_YAML_PATHS["dataset"], dataset_path / "dataset.yaml")
shutil.copyfile(SAMPLE_YAML_PATHS["pipeline"], pipeline_path / "pipeline.yaml")

config = yaml.load(open(dataset_path / "dataset.yaml"))

# Get a bigquery_dataset resource and modify the `description` field
bq_dataset = next(
(r for r in config["resources"] if r["type"] == "bigquery_dataset"), None
)
test_description = 'Multiline\nstring with\n"quotes"'
bq_dataset["description"] = test_description
with open(dataset_path / "dataset.yaml", "w") as file:
yaml.dump(config, file)

generate_terraform.main(
dataset_path.name,
project_id,
bucket_name_prefix,
region,
impersonating_acct,
env,
None,
None,
)

env_dataset_path = ENV_DATASETS_PATH / dataset_path.name
subprocess.check_call(["terraform", "fmt"], cwd=env_dataset_path / "_terraform")


def test_dataset_tf_has_no_bq_dataset_description_when_unspecified(
dataset_path,
pipeline_path,
Expand Down Expand Up @@ -601,6 +639,43 @@ def test_pipeline_tf_has_no_bq_table_description_when_unspecified(
assert not re.search(r"description\s+\=", result.group(1))


def test_bq_table_can_have_a_description_with_newlines_and_quotes(
dataset_path,
pipeline_path,
project_id,
bucket_name_prefix,
region,
impersonating_acct,
env,
):
shutil.copyfile(SAMPLE_YAML_PATHS["dataset"], dataset_path / "dataset.yaml")
shutil.copyfile(SAMPLE_YAML_PATHS["pipeline"], pipeline_path / "pipeline.yaml")

config = yaml.load(open(pipeline_path / "pipeline.yaml"))

# Get a bigquery_table resource and modify the `description` field
bq_table = next(
(r for r in config["resources"] if r["type"] == "bigquery_table"), None
)
bq_table["description"] = 'Multiline\nstring with\n"quotes"'
with open(pipeline_path / "pipeline.yaml", "w") as file:
yaml.dump(config, file)

generate_terraform.main(
dataset_path.name,
project_id,
bucket_name_prefix,
region,
impersonating_acct,
env,
None,
None,
)

env_dataset_path = ENV_DATASETS_PATH / dataset_path.name
subprocess.check_call(["terraform", "fmt"], cwd=env_dataset_path / "_terraform")


def test_bq_table_name_starts_with_digits_but_tf_resource_name_does_not(
dataset_path,
pipeline_path,
Expand Down