From e4762e9158abcc5acc2bfedec2b979e2c3766ef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20B=C3=A4ck?= Date: Mon, 15 Jan 2024 12:07:42 +0100 Subject: [PATCH] Add diff_definitions.py for diffing event definition files and schema files (#389) This new script makes it easy to compare the current definitions of events and other types against a chosen base, e.g. the current state of the master branch or a previous edition. It produces diff commands that can be run to produce the wanted comparison. Since figuring out the most recent version of each type is a pretty common operation, a new versions module is introduced for this purpose. The existing find-latest-schemas.py script has been adapted to use the new module. --- diff_definitions.py | 62 +++++++++++++++++++++++++++ find-latest-schemas.py | 50 ++++++++++++---------- test_versions.py | 95 ++++++++++++++++++++++++++++++++++++++++++ versions.py | 72 ++++++++++++++++++++++++++++++++ 4 files changed, 257 insertions(+), 22 deletions(-) create mode 100755 diff_definitions.py create mode 100644 test_versions.py create mode 100644 versions.py diff --git a/diff_definitions.py b/diff_definitions.py new file mode 100755 index 00000000..59079883 --- /dev/null +++ b/diff_definitions.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Axis Communications AB. +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Context-sensitive diffing of Eiffel type definitions. Compares the +current workspace's latest type versions with the ones from a specified +base, and prints diff commands. For example, if the current commit has +added v4.3.0 of ActT and the given base had v4.2.0 as its latest version, +you'll get the following output: + +diff -u definitions/EiffelActivityTriggeredEvent/4.2.0.yml definitions/EiffelActivityTriggeredEvent/4.3.0.yml +diff -u schemas/EiffelActivityTriggeredEvent/4.2.0.json schemas/EiffelActivityTriggeredEvent/4.3.0.json + +By default, the base of the comparison is origin/master, but any commit +reference can be given as an argument. +""" + +import sys +from pathlib import Path + +import versions + + +def _main(): + base = "origin/master" + if len(sys.argv) > 2: + print(f"Usage: python {sys.argv[0]} [ base ]") + sys.exit(-1) + elif len(sys.argv) == 2: + base = sys.argv[1] + + base_defs = versions.latest_in_gitref(base, ".", Path("definitions")) + workspace_defs = versions.latest_in_dir(Path("definitions")) + for type, workspace_version in sorted(workspace_defs.items()): + base_version = base_defs.get(type) + if not base_version: + print(f"diff -u /dev/null definitions/{type}/{workspace_version}.yml") + print(f"diff -u /dev/null schemas/{type}/{workspace_version}.json") + elif base_version != workspace_version: + print( + f"diff -u definitions/{type}/{base_version}.yml definitions/{type}/{workspace_version}.yml" + ) + print( + f"diff -u schemas/{type}/{base_version}.json schemas/{type}/{workspace_version}.json" + ) + + +if __name__ == "__main__": + _main() diff --git a/find-latest-schemas.py b/find-latest-schemas.py index a35b9ad0..3f969d14 100644 --- a/find-latest-schemas.py +++ b/find-latest-schemas.py @@ -1,8 +1,23 @@ -import os +# Copyright 2016-2024 Ericsson AB and others. +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sys +from pathlib import Path from shutil import copyfile -import semver +import versions """ Finds the latest versions of schemas found under , with @@ -16,28 +31,19 @@ def main(): if len(sys.argv) != 3: - print("Usage: python {} input_folder output_folder".format(sys.argv[0])) + print(f"Usage: python {sys.argv[0]} input_folder output_folder") sys.exit(-1) - input_folder = sys.argv[1] - output_folder = sys.argv[2] - - if not os.path.exists(output_folder): - os.makedirs(output_folder) - - latest_versions = [] - for base, _, files in os.walk(input_folder): - if len(files) != 0: - latest_version = "0.0.0" - for f in files: - latest_version = semver.max_ver(latest_version, os.path.splitext(f)[0]) - latest_versions.append(os.path.join(base, latest_version + ".json")) - - for f in latest_versions: - new_name = os.path.split(os.path.dirname(f))[1] + ".json" - output_file = os.path.join(output_folder, new_name) - copyfile(f, output_file) - print("{} => {}".format(f, output_file)) + input_folder = Path(sys.argv[1]) + output_folder = Path(sys.argv[2]) + + output_folder.mkdir(exist_ok=True) + + for type, version in versions.latest_in_dir(input_folder).items(): + input_file = input_folder / type / f"{version}.json" + output_file = output_folder / f"{type}.json" + copyfile(input_file, output_file) + print(f"{input_file} => {output_file}") if __name__ == "__main__": diff --git a/test_versions.py b/test_versions.py new file mode 100644 index 00000000..ec263f82 --- /dev/null +++ b/test_versions.py @@ -0,0 +1,95 @@ +# Copyright 2024 Axis Communications AB. +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +from pathlib import Path + +import pytest +import semver + +import versions + + +class Git: + """Simple class for running Git commands in a given directory.""" + + def __init__(self, path: Path): + self.path = path + self.command("init") + # "git commit" requires that we identify ourselves. + self.command("config", "user.name", "John Doe") + self.command("config", "user.email", "john@example.com") + + def command(self, *args: str) -> None: + subprocess.check_call(["git"] + list(args), cwd=self.path) + + +@pytest.fixture +def tmp_git(tmp_path): + """Injects a Git instance rooted in a temporary directory.""" + yield Git(tmp_path) + + +def create_files(base_path: Path, *args: str) -> None: + for p in args: + fullpath = base_path / p + fullpath.parent.mkdir(parents=True, exist_ok=True) + fullpath.touch() + + +def test_latest_in_gitref(tmp_git): + # Create a bunch of files in the git, commit them, and tag that commit. + create_files( + tmp_git.path, + "subdir_c/6.0.0.json", + "definitions/subdir_a/1.0.0.yml", + "definitions/subdir_a/2.0.0.yml", + "definitions/subdir_a/3.0.0.othersuffix", + "definitions/subdir_b/3.0.0.yml", + "definitions/subdir_b/4.0.0.yml", + ) + tmp_git.command("add", "-A") + tmp_git.command("commit", "-m", "Initial set of files") + tmp_git.command("tag", "v1.0.0") + + # Add an additional file and delete one of the original files. + (tmp_git.path / "definitions/subdir_b/5.0.0.yml").touch() + tmp_git.command("rm", "definitions/subdir_a/2.0.0.yml") + tmp_git.command("add", "-A") + tmp_git.command("commit", "-m", "Make changes") + + # Make sure the results we get are consistent with the original + # contents of the git. + assert versions.latest_in_gitref("v1.0.0", tmp_git.path, "definitions") == { + "subdir_a": semver.VersionInfo.parse("2.0.0"), + "subdir_b": semver.VersionInfo.parse("4.0.0"), + } + + +def test_latest_in_dir(tmp_path): + create_files( + tmp_path, + "subdir_c/6.0.0.yml", + "definitions/subdir_a/1.0.0.yml", + "definitions/subdir_a/2.0.0.yml", + "definitions/subdir_a/3.0.0.othersuffix", + "definitions/subdir_b/3.0.0.yml", + "definitions/subdir_b/4.0.0.yml", + ) + + assert versions.latest_in_dir(tmp_path / "definitions") == { + "subdir_a": semver.VersionInfo.parse("2.0.0"), + "subdir_b": semver.VersionInfo.parse("4.0.0"), + } diff --git a/versions.py b/versions.py new file mode 100644 index 00000000..f250e01d --- /dev/null +++ b/versions.py @@ -0,0 +1,72 @@ +# Copyright 2024 Axis Communications AB. +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The versions module contains functions for discovering definition files.""" + +import os +import subprocess +from pathlib import Path +from typing import Dict +from typing import Iterable + +import semver + + +def latest_in_gitref( + committish: str, gitdir: Path, subdir: Path +) -> Dict[str, semver.version.Version]: + """Lists the definition files found under a given subdirectory of a + git at a given point in time (described by a committish, e.g. a + SHA-1, tag, or branch reference) and returns a dict that maps each + typename (e.g. EiffelArtifactCreatedEvent) to the latest version found. + """ + return _latest_versions( + Path(line) + for line in ( + subprocess.check_output( + ["git", "ls-tree", "-r", "--name-only", committish, "--", subdir], + cwd=gitdir, + ) + .decode("utf-8") + .splitlines() + ) + if Path(line).suffix == ".yml" + ) + + +def latest_in_dir(path: Path) -> Dict[str, semver.version.Version]: + """Inspects the definition files found under a given path and returns + a dict that maps each typename (e.g. EiffelArtifactCreatedEvent) to + its latest version found. + """ + return _latest_versions( + Path(current) / f + for current, _, files in os.walk(path) + for f in files + if Path(f).suffix == ".yml" + ) + + +def _latest_versions(paths: Iterable[Path]) -> Dict[str, semver.version.Version]: + """Given a list of foo//. pathnames, returns + a dict mapping typenames to the most recent version of that type. + """ + result = {} + for path in paths: + type = path.parent.name + this_version = semver.VersionInfo.parse(Path(path.name).stem) + if type not in result or result[type] < this_version: + result[type] = this_version + return result