gen_config_vars.py

#!/usr/bin/env python3

# Copyright 2023 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import binascii
import collections
import ctypes
import datetime
import json
import math
import os
import re
import sys

import yaml

with open("src/core/lib/config/config_vars.yaml") as f:
    attrs = yaml.safe_load(f.read())

error = False
today = datetime.date.today()
two_quarters_from_now = today + datetime.timedelta(days=180)
for attr in attrs:
    if "name" not in attr:
        print("config has no name: %r" % attr)
        error = True
        continue
    if "experiment" in attr["name"] and attr["name"] != "experiments":
        print("use experiment system for experiments")
        error = True
    if "description" not in attr:
        print("no description for %s" % attr["name"])
        error = True
    if "default" not in attr:
        print("no default for %s" % attr["name"])
        error = True

if error:
    sys.exit(1)


def c_str(s, encoding="ascii"):
    if s is None:
        return '""'
    if isinstance(s, str):
        s = s.encode(encoding)
    result = ""
    for c in s:
        c = chr(c) if isinstance(c, int) else c
        if not (32 <= ord(c) < 127) or c in ("\\", '"'):
            result += "\\%03o" % ord(c)
        else:
            result += c
    return '"' + result + '"'


def snake_to_pascal(s):
    return "".join(x.capitalize() for x in s.split("_"))


# utility: print a big comment block into a set of files
def put_banner(files, banner):
    for f in files:
        for line in banner:
            print("// %s" % line, file=f)
        print(file=f)


def put_copyright(file):
    # copy-paste copyright notice from this file
    with open(sys.argv[0]) as my_source:
        copyright = []
        for line in my_source:
            if line[0] != "#":
                break
        for line in my_source:
            if line[0] == "#":
                copyright.append(line)
                break
        for line in my_source:
            if line[0] != "#":
                break
            copyright.append(line)
        put_banner([file], [line[2:].rstrip() for line in copyright])


RETURN_TYPE = {
    "int": "int32_t",
    "string": "absl::string_view",
    "comma_separated_string": "absl::string_view",
    "bool": "bool",
}

MEMBER_TYPE = {
    "int": "int32_t",
    "string": "std::string",
    "comma_separated_string": "std::string",
    "bool": "bool",
}

FLAG_TYPE = {
    "int": "absl::optional<int32_t>",
    "string": "absl::optional<std::string>",
    "comma_separated_string": "std::vector<std::string>",
    "bool": "absl::optional<bool>",
}

PROTO_TYPE = {
    "int": "int32",
    "string": "string",
    "comma_separated_string": "string",
    "bool": "bool",
}

SORT_ORDER_FOR_PACKING = {
    "int": 0,
    "bool": 1,
    "string": 2,
    "comma_separated_string": 3,
}


def bool_default_value(x, name):
    if x == True:
        return "true"
    if x == False:
        return "false"
    if isinstance(x, str) and x.startswith("$"):
        return x[1:]
    return x


def int_default_value(x, name):
    if isinstance(x, str) and x.startswith("$"):
        return x[1:]
    return x


def string_default_value(x, name):
    if x is None:
        return '""'
    if x.startswith("$"):
        return x[1:]
    return c_str(x)


DEFAULT_VALUE = {
    "int": int_default_value,
    "bool": bool_default_value,
    "string": string_default_value,
    "comma_separated_string": string_default_value,
}

TO_STRING = {
    "int": "$",
    "bool": '$?"true":"false"',
    "string": '"\\"", absl::CEscape($), "\\""',
    "comma_separated_string": '"\\"", absl::CEscape($), "\\""',
}

attrs_in_packing_order = sorted(
    attrs, key=lambda a: SORT_ORDER_FOR_PACKING[a["type"]]
)

with open("test/core/util/fuzz_config_vars.proto", "w") as P:
    put_copyright(P)

    put_banner(
        [P],
        [
            "",
            "Automatically generated by tools/codegen/core/gen_config_vars.py",
            "",
        ],
    )

    print('syntax = "proto3";', file=P)
    print(file=P)
    print("package grpc.testing;", file=P)
    print(file=P)
    print("message FuzzConfigVars {", file=P)
    for attr in attrs_in_packing_order:
        if attr.get("fuzz", False) == False:
            continue
        print(
            "  optional %s %s = %d;"
            % (
                attr.get("fuzz_type", PROTO_TYPE[attr["type"]]),
                attr["name"],
                binascii.crc32(attr["name"].encode("ascii")) & 0x1FFFFFFF,
            ),
            file=P,
        )
    print("};", file=P)

with open("test/core/util/fuzz_config_vars.h", "w") as H:
    put_copyright(H)

    put_banner(
        [H],
        [
            "",
            "Automatically generated by tools/codegen/core/gen_config_vars.py",
            "",
        ],
    )

    print("#ifndef GRPC_TEST_CORE_UTIL_FUZZ_CONFIG_VARS_H", file=H)
    print("#define GRPC_TEST_CORE_UTIL_FUZZ_CONFIG_VARS_H", file=H)
    print(file=H)
    print("#include <grpc/support/port_platform.h>", file=H)
    print(file=H)
    print('#include "test/core/util/fuzz_config_vars.pb.h"', file=H)
    print('#include "src/core/lib/config/config_vars.h"', file=H)
    print(file=H)
    print("namespace grpc_core {", file=H)
    print(file=H)
    print(
        (
            "ConfigVars::Overrides OverridesFromFuzzConfigVars(const"
            " grpc::testing::FuzzConfigVars& vars);"
        ),
        file=H,
    )
    print(
        "void ApplyFuzzConfigVars(const grpc::testing::FuzzConfigVars& vars);",
        file=H,
    )
    print(file=H)
    print("}  // namespace grpc_core", file=H)
    print(file=H)
    print("#endif  // GRPC_TEST_CORE_UTIL_FUZZ_CONFIG_VARS_H", file=H)

with open("test/core/util/fuzz_config_vars.cc", "w") as C:
    put_copyright(C)

    put_banner(
        [C],
        [
            "",
            "Automatically generated by tools/codegen/core/gen_config_vars.py",
            "",
        ],
    )

    print('#include "test/core/util/fuzz_config_vars.h"', file=C)
    print('#include "test/core/util/fuzz_config_vars_helpers.h"', file=C)
    print(file=C)
    print("namespace grpc_core {", file=C)
    print(file=C)
    print(
        (
            "ConfigVars::Overrides OverridesFromFuzzConfigVars(const"
            " grpc::testing::FuzzConfigVars& vars) {"
        ),
        file=C,
    )
    print("  ConfigVars::Overrides overrides;", file=C)
    for attr in attrs_in_packing_order:
        fuzz = attr.get("fuzz", False)
        if not fuzz:
            continue
        print("  if (vars.has_%s()) {" % attr["name"], file=C)
        if isinstance(fuzz, str):
            print(
                "    overrides.%s = %s(vars.%s());"
                % (attr["name"], fuzz, attr["name"]),
                file=C,
            )
        else:
            print(
                "    overrides.%s = vars.%s();" % (attr["name"], attr["name"]),
                file=C,
            )
        print("  }", file=C)
    print("  return overrides;", file=C)
    print("}", file=C)
    print(
        "void ApplyFuzzConfigVars(const grpc::testing::FuzzConfigVars& vars) {",
        file=C,
    )
    print(
        "  ConfigVars::SetOverrides(OverridesFromFuzzConfigVars(vars));", file=C
    )
    print("}", file=C)
    print(file=C)
    print("}  // namespace grpc_core", file=C)

with open("src/core/lib/config/config_vars.h", "w") as H:
    put_copyright(H)

    put_banner(
        [H],
        [
            "",
            "Automatically generated by tools/codegen/core/gen_config_vars.py",
            "",
        ],
    )

    print("#ifndef GRPC_SRC_CORE_LIB_CONFIG_CONFIG_VARS_H", file=H)
    print("#define GRPC_SRC_CORE_LIB_CONFIG_CONFIG_VARS_H", file=H)
    print(file=H)
    print("#include <grpc/support/port_platform.h>", file=H)
    print(file=H)
    print("#include <string>", file=H)
    print("#include <atomic>", file=H)
    print("#include <stdint.h>", file=H)
    print('#include "absl/strings/string_view.h"', file=H)
    print('#include "absl/types/optional.h"', file=H)
    print(file=H)
    print("namespace grpc_core {", file=H)
    print(file=H)
    print("class GPR_DLL ConfigVars {", file=H)
    print(" public:", file=H)
    print("  struct Overrides {", file=H)
    for attr in attrs_in_packing_order:
        print(
            "    absl::optional<%s> %s;"
            % (MEMBER_TYPE[attr["type"]], attr["name"]),
            file=H,
        )
    print("  };", file=H)
    print("  ConfigVars(const ConfigVars&) = delete;", file=H)
    print("  ConfigVars& operator=(const ConfigVars&) = delete;", file=H)
    print(
        "  // Get the core configuration; if it does not exist, create it.",
        file=H,
    )
    print("  static const ConfigVars& Get() {", file=H)
    print("    auto* p = config_vars_.load(std::memory_order_acquire);", file=H)
    print("    if (p != nullptr) return *p;", file=H)
    print("    return Load();", file=H)
    print("  }", file=H)
    print("  static void SetOverrides(const Overrides& overrides);", file=H)
    print(
        "  // Drop the config vars. Users must ensure no other threads are",
        file=H,
    )
    print("  // accessing the configuration.", file=H)
    print("  static void Reset();", file=H)
    print("  std::string ToString() const;", file=H)
    for attr in attrs:
        for line in attr["description"].splitlines():
            print("  // %s" % line, file=H)
        if attr.get("force-load-on-access", False):
            print(
                "  %s %s() const;"
                % (MEMBER_TYPE[attr["type"]], snake_to_pascal(attr["name"])),
                file=H,
            )
        else:
            print(
                "  %s %s() const { return %s_; }"
                % (
                    RETURN_TYPE[attr["type"]],
                    snake_to_pascal(attr["name"]),
                    attr["name"],
                ),
                file=H,
            )
    print(" private:", file=H)
    print("  explicit ConfigVars(const Overrides& overrides);", file=H)
    print("  static const ConfigVars& Load();", file=H)
    print("  static std::atomic<ConfigVars*> config_vars_;", file=H)
    for attr in attrs_in_packing_order:
        if attr.get("force-load-on-access", False):
            continue
        print("  %s %s_;" % (MEMBER_TYPE[attr["type"]], attr["name"]), file=H)
    for attr in attrs_in_packing_order:
        if attr.get("force-load-on-access", False) == False:
            continue
        print(
            "  absl::optional<%s> override_%s_;"
            % (MEMBER_TYPE[attr["type"]], attr["name"]),
            file=H,
        )
    print("};", file=H)
    print(file=H)
    print("}  // namespace grpc_core", file=H)
    print(file=H)
    print("#endif  // GRPC_SRC_CORE_LIB_CONFIG_CONFIG_VARS_H", file=H)

with open("src/core/lib/config/config_vars.cc", "w") as C:
    put_copyright(C)

    put_banner(
        [C],
        [
            "",
            "Automatically generated by tools/codegen/core/gen_config_vars.py",
            "",
        ],
    )

    print("#include <grpc/support/port_platform.h>", file=C)
    print('#include "src/core/lib/config/config_vars.h"', file=C)
    print('#include "src/core/lib/config/load_config.h"', file=C)
    print('#include "absl/strings/escaping.h"', file=C)
    print('#include "absl/flags/flag.h"', file=C)
    print(file=C)

    for attr in attrs:
        if "prelude" in attr:
            print(attr["prelude"], file=C)

    for attr in attrs:
        print(
            "ABSL_FLAG(%s, %s, {}, %s);"
            % (
                FLAG_TYPE[attr["type"]],
                "grpc_" + attr["name"],
                c_str(attr["description"]),
            ),
            file=C,
        )
    print(file=C)
    print("namespace grpc_core {", file=C)
    print(file=C)
    print("ConfigVars::ConfigVars(const Overrides& overrides) :", file=C)
    initializers = [
        '%s_(LoadConfig(FLAGS_grpc_%s, "GRPC_%s", overrides.%s, %s))'
        % (
            attr["name"],
            attr["name"],
            attr["name"].upper(),
            attr["name"],
            DEFAULT_VALUE[attr["type"]](attr["default"], attr["name"]),
        )
        for attr in attrs_in_packing_order
        if attr.get("force-load-on-access", False) == False
    ]
    initializers += [
        "override_%s_(overrides.%s)" % (attr["name"], attr["name"])
        for attr in attrs_in_packing_order
        if attr.get("force-load-on-access", False)
    ]
    print(",".join(initializers), file=C)
    print("{}", file=C)
    print(file=C)
    for attr in attrs:
        if attr.get("force-load-on-access", False):
            print(
                "%s ConfigVars::%s() const { return LoadConfig(FLAGS_grpc_%s,"
                ' "GRPC_%s", override_%s_, %s); }'
                % (
                    MEMBER_TYPE[attr["type"]],
                    snake_to_pascal(attr["name"]),
                    attr["name"],
                    attr["name"].upper(),
                    attr["name"],
                    DEFAULT_VALUE[attr["type"]](attr["default"], attr["name"]),
                ),
                file=C,
            )
            print(file=C)
    print("std::string ConfigVars::ToString() const {", file=C)
    print("  return absl::StrCat(", file=C)
    for i, attr in enumerate(attrs):
        if i:
            print(",", file=C)
            print(c_str(", " + attr["name"] + ": "), file=C)
        else:
            print(c_str(attr["name"] + ": "), file=C)
        print(
            ",",
            TO_STRING[attr["type"]].replace(
                "$", snake_to_pascal(attr["name"]) + "()"
            ),
            file=C,
        )
    print(");}", file=C)
    print(file=C)
    print("}", file=C)