Skip to content

Commit

Permalink
feat(backup): Generate validation run cloudbuild.yaml (#59372)
Browse files Browse the repository at this point in the history
This file instruments the validation run we need to perform before we
can kick off the actual relocation. We build a bespoke YAML for each
such run, rather than using substitutions, as there is enough
variability that this ends up a bit easier to read, and some variables
could grow longer than the substitution API is willing to accept.

Issue: getsentry/team-ospo#203
  • Loading branch information
azaslavsky committed Nov 7, 2023
1 parent 90781a3 commit 1b853c3
Show file tree
Hide file tree
Showing 5 changed files with 753 additions and 7 deletions.
13 changes: 8 additions & 5 deletions src/sentry/models/relocation.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,18 +161,21 @@ class Kind(Enum):
def get_choices(cls) -> list[tuple[int, str]]:
return [(key.value, key.name) for key in cls]

def to_filename(self, suffix: str):
def __str__(self):
if self.name == "RAW_USER_DATA":
return f"raw-relocation-data.{suffix}"
return "raw-relocation-data"
elif self.name == "NORMALIZED_USER_DATA":
return f"normalized-relocation-data.{suffix}"
return "normalized-relocation-data"
elif self.name == "BASELINE_CONFIG_VALIDATION_DATA":
return f"baseline-config.{suffix}"
return "baseline-config"
elif self.name == "COLLIDING_USERS_VALIDATION_DATA":
return f"colliding-users.{suffix}"
return "colliding-users"
else:
raise ValueError("Cannot extract a filename from `RelocationFile.Kind.UNKNOWN`.")

def to_filename(self, ext: str):
return str(self) + "." + ext

relocation = FlexibleForeignKey("sentry.Relocation")
file = FlexibleForeignKey("sentry.File")
kind = models.SmallIntegerField(choices=Kind.get_choices())
Expand Down
12 changes: 12 additions & 0 deletions src/sentry/tasks/relocation.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
RELOCATION_BLOB_SIZE,
RELOCATION_FILE_TYPE,
OrderedTask,
create_cloudbuild_yaml,
fail_relocation,
retry_task_or_fail_relocation,
start_relocation_task,
Expand Down Expand Up @@ -418,6 +419,17 @@ def preprocessing_complete(uuid: str) -> None:
ERR_PREPROCESSING_INTERNAL,
):
storage = get_storage()

# Build the `cloudbuild.yaml` file we'll use for validation.
cloudbuild_yaml = create_cloudbuild_yaml(relocation)
storage.save(f"relocations/runs/{uuid}/conf/cloudbuild.yaml", BytesIO(cloudbuild_yaml))

# Upload the `key-config.json` file we'll use to identify the correct KMS resource use
# during validation.
kms_config_bytes = json.dumps(DEFAULT_CRYPTO_KEY_VERSION).encode("utf-8")
storage.save(f"relocations/runs/{uuid}/in/kms-config.json", BytesIO(kms_config_bytes))

# Upload the exports we'll be validating.
for kind in RELOCATION_FILES_TO_BE_VALIDATED:
raw_relocation_file = (
RelocationFile.objects.filter(
Expand Down

0 comments on commit 1b853c3

Please sign in to comment.