Pass parent config to UEP templates

Introduces the `parent_config` variable to UEP templates, which is
always populated with the `Config` of the parent MEP. Also validates
that users don't attempt to override it.

Additionally introduces an autoclass doc for the `Config` object, to
show admins exactly what is available.

changelog.d/20240517_150132_chris_pass_parent_config_to_template.rst

@@ -0,0 +1,6 @@
+New Functionality
+^^^^^^^^^^^^^^^^^
+
+- MEPs now pass their configuration to UEP config templates via the ``parent_config``
+  variable. `See the docs <https://globus-compute.readthedocs.io/en/latest/endpoints/multi_user.html#user-config-template-yaml-j2>`
+  for more information.

compute_endpoint/globus_compute_endpoint/endpoint/config/utils.py

@@ -145,10 +145,18 @@ def load_user_config_schema(endpoint_dir: pathlib.Path) -> dict | None:
         raise
 
 
-def _validate_user_opts(user_opts: dict, schema: dict) -> None:
-    """Validates user config options against a JSON schema."""
+def _validate_user_opts(user_opts: dict, schema: dict | None) -> None:
+    """Validates user config options, optionally against a JSON schema."""
     import jsonschema  # Only load package when called by EP manager
 
+    if "parent_config" in user_opts:
+        raise ValueError(
+            "'parent_config' is a reserved word and cannot be passed in via user config"
+        )
+
+    if not schema:
+        return
+
     try:
         jsonschema.validate(instance=user_opts, schema=schema)
     except jsonschema.SchemaError:
@@ -212,6 +220,7 @@ def load_user_config_template(endpoint_dir: pathlib.Path) -> tuple[str, dict | N
 
 
 def render_config_user_template(
+    parent_config: Config,
     user_config_template: str,
     user_config_schema: dict | None = None,
     user_opts: dict | None = None,
@@ -223,16 +232,15 @@ def render_config_user_template(
     from jinja2.sandbox import SandboxedEnvironment
 
     _user_opts = user_opts or {}
-    if user_config_schema:
-        _validate_user_opts(_user_opts, user_config_schema)
+    _validate_user_opts(_user_opts, user_config_schema)
     _user_opts = _sanitize_user_opts(_user_opts)
 
     environment = SandboxedEnvironment(undefined=jinja2.StrictUndefined)
     environment.filters["shell_escape"] = _shell_escape_filter
     template = environment.from_string(user_config_template)
 
     try:
-        return template.render(**_user_opts)
+        return template.render(**_user_opts, parent_config=parent_config)
     except jinja2.exceptions.UndefinedError as e:
         log.debug("Missing required user option: %s", e)
         raise

compute_endpoint/globus_compute_endpoint/endpoint/endpoint_manager.py

@@ -963,7 +963,7 @@ def cmd_start_endpoint(
 
             user_opts = kwargs.get("user_opts", {})
             user_config = render_config_user_template(
-                template_str, user_config_schema, user_opts
+                self._config, template_str, user_config_schema, user_opts
             )
             stdin_data_dict = {
                 "amqp_creds": kwargs.get("amqp_creds"),

compute_endpoint/tests/unit/test_endpoint_unit.py

@@ -21,6 +21,7 @@
     config as default_config,
 )
 from globus_compute_endpoint.endpoint.config.utils import (
+    get_config,
     load_user_config_template,
     render_config_user_template,
     serialize_config,
@@ -645,10 +646,11 @@ def test_mu_endpoint_user_ep_yamls_world_readable(tmp_path):
 def test_mu_endpoint_user_ep_sensible_default(tmp_path):
     ep_dir = tmp_path / "new_endpoint_dir"
     Endpoint.init_endpoint_dir(ep_dir, multi_user=True)
+    parent_cfg = get_config(ep_dir)
 
     tmpl_str, schema = load_user_config_template(ep_dir)
     # Doesn't crash; loads yaml, jinja template has defaults
-    render_config_user_template(tmpl_str, schema, {})
+    render_config_user_template(parent_cfg, tmpl_str, schema, {})
 
 
 def test_always_prints_endpoint_id_to_terminal(mocker, mock_ep_data, mock_reg_info):

compute_endpoint/tests/unit/test_endpointmanager_unit.py

@@ -59,6 +59,7 @@
 
 
 _MOCK_BASE = "globus_compute_endpoint.endpoint.endpoint_manager."
+_CFG_UTILS_MOCK_BASE = "globus_compute_endpoint.endpoint.config.utils."
 
 _mock_rootuser_rec = pwd.struct_passwd(
     ("root", "", 0, 0, "Mock Root User", "/mock_root", "/bin/false")
@@ -1942,15 +1943,16 @@ def test_load_user_config_template_prefer_j2(conf_dir: pathlib.Path):
 def test_render_user_config(mocker, data):
     is_valid, user_opts = data
     template = "heartbeat_period: {{ heartbeat }}"
+    parent_cfg = Config()
 
     if is_valid:
-        rendered = render_config_user_template(template, {}, user_opts)
+        rendered = render_config_user_template(parent_cfg, template, {}, user_opts)
         rendered_dict = yaml.safe_load(rendered)
         assert rendered_dict["heartbeat_period"] == user_opts["heartbeat"]
     else:
-        mock_log = mocker.patch("globus_compute_endpoint.endpoint.config.utils.log")
+        mock_log = mocker.patch(f"{_CFG_UTILS_MOCK_BASE}log")
         with pytest.raises(jinja2.exceptions.UndefinedError):
-            render_config_user_template(template, {}, user_opts)
+            render_config_user_template(parent_cfg, template, {}, user_opts)
         a, _k = mock_log.debug.call_args
         assert "Missing required" in a[0]
 
@@ -1986,7 +1988,7 @@ def test_render_user_config_escape_strings():
             "accelerators": [f"{uuid.uuid4()}\n    mem_per_worker: 100"],
         },
     }
-    rendered = render_config_user_template(template, {}, user_opts)
+    rendered = render_config_user_template(Config(), template, {}, user_opts)
     rendered_dict = yaml.safe_load(rendered)
 
     assert len(rendered_dict) == 2
@@ -2017,12 +2019,13 @@ def test_render_user_config_option_types(data):
     is_valid, val = data
     template = "foo: {{ foo }}"
     user_opts = {"foo": val}
+    parent_cfg = Config()
 
     if is_valid:
-        render_config_user_template(template, {}, user_opts)
+        render_config_user_template(parent_cfg, template, {}, user_opts)
     else:
         with pytest.raises(ValueError) as pyt_exc:
-            render_config_user_template(template, {}, user_opts)
+            render_config_user_template(parent_cfg, template, {}, user_opts)
         assert "not a valid user config option type" in pyt_exc.exconly()
 
 
@@ -2039,12 +2042,12 @@ def test_render_user_config_sandbox(mocker: MockFixture, data: t.Tuple[str, t.An
     template = f"foo: {jinja_op}"
     user_opts = {"foo": val}
     mocker.patch(
-        "globus_compute_endpoint.endpoint.config.utils._sanitize_user_opts",
+        f"{_CFG_UTILS_MOCK_BASE}_sanitize_user_opts",
         return_value=user_opts,
     )
 
     with pytest.raises(jinja2.exceptions.SecurityError):
-        render_config_user_template(template, {}, user_opts)
+        render_config_user_template(Config(), template, {}, user_opts)
 
 
 @pytest.mark.parametrize(
@@ -2070,7 +2073,7 @@ def test_render_user_config_shell_escape(data: t.Tuple[bool, t.Any]):
     is_valid, option = data
     template = "option: {{ option|shell_escape }}"
     user_opts = {"option": option}
-    rendered = render_config_user_template(template, {}, user_opts)
+    rendered = render_config_user_template(Config(), template, {}, user_opts)
     rendered_dict = yaml.safe_load(rendered)
 
     assert len(rendered_dict) == 1
@@ -2098,7 +2101,7 @@ def test_render_user_config_apply_schema(mocker: MockFixture, schema_exists: boo
     mock_validate = mocker.patch.object(jsonschema, "validate")
 
     user_opts = {"foo": "bar"}
-    render_config_user_template(template, schema, user_opts)
+    render_config_user_template(Config(), template, schema, user_opts)
 
     if schema_exists:
         assert mock_validate.called
@@ -2109,6 +2112,16 @@ def test_render_user_config_apply_schema(mocker: MockFixture, schema_exists: boo
         assert not mock_validate.called
 
 
+def test_render_config_passes_parent_config():
+    template = "parent_heartbeat: {{ parent_config.heartbeat_period }}"
+    parent_cfg = Config()
+
+    rendered = render_config_user_template(parent_cfg, template)
+
+    rendered_dict = yaml.safe_load(rendered)
+    assert rendered_dict["parent_heartbeat"] == parent_cfg.heartbeat_period
+
+
 @pytest.mark.parametrize(
     "data",
     [
@@ -2123,7 +2136,7 @@ def test_render_user_config_apply_schema(mocker: MockFixture, schema_exists: boo
 def test_validate_user_config_options(mocker: MockFixture, data: t.Tuple[bool, dict]):
     is_valid, user_opts = data
 
-    mock_log = mocker.patch("globus_compute_endpoint.endpoint.config.utils.log")
+    mock_log = mocker.patch(f"{_CFG_UTILS_MOCK_BASE}log")
 
     schema = {
         "$schema": "https://json-schema.org/draft/2020-12/schema",
@@ -2151,7 +2164,7 @@ def test_validate_user_config_options(mocker: MockFixture, data: t.Tuple[bool, d
 def test_validate_user_config_options_invalid_schema(
     mocker: MockFixture, schema: t.Any
 ):
-    mock_log = mocker.patch("globus_compute_endpoint.endpoint.config.utils.log")
+    mock_log = mocker.patch(f"{_CFG_UTILS_MOCK_BASE}log")
     user_opts = {"foo": "bar"}
     with pytest.raises(jsonschema.SchemaError):
         _validate_user_opts(user_opts, schema)
@@ -2160,6 +2173,14 @@ def test_validate_user_config_options_invalid_schema(
     assert "user config schema is invalid" in str(a)
 
 
+def test_validate_parent_config_reserved():
+    with pytest.raises(ValueError) as pyt_exc:
+        render_config_user_template(Config(), {}, user_opts={"parent_config": "foo"})
+
+    assert "parent_config" in str(pyt_exc)
+    assert "reserved" in str(pyt_exc)
+
+
 @pytest.mark.parametrize(
     "data", [(True, '{"foo": "bar"}'), (False, '{"foo": "bar", }')]
 )
@@ -2168,7 +2189,7 @@ def test_load_user_config_schema(
 ):
     is_valid, schema_json = data
 
-    mock_log = mocker.patch("globus_compute_endpoint.endpoint.config.utils.log")
+    mock_log = mocker.patch(f"{_CFG_UTILS_MOCK_BASE}log")
 
     template = Endpoint.user_config_schema_path(conf_dir)
     template.write_text(schema_json)

docs/endpoints/multi_user.rst

@@ -311,6 +311,11 @@ Given the above template, users submitting to this MEP would be able to specify
 ``endpoint_setup`` and ``worker_init`` values.  All other values will remain unchanged
 when the UEP starts up.
 
+All templates additionally have access to the configuration of the parent MEP via the
+``parent_config`` variable. This is a reserved word in the Compute template
+specification which always takes the shape of a `Config object`_, and users are unable
+to specify a value for it.
+
 As linked on the left, :doc:`there are a number of example configurations
 <endpoint_examples>` to showcase the available options, but ``idle_heartbeats_soft`` and
 ``idle_heartbeats_hard`` bear describing.
@@ -894,6 +899,7 @@ Administrator Quickstart
 .. _globus-identity-mapping: https://pypi.org/project/globus-identity-mapping/
 .. _getpwnam(3): https://www.man7.org/linux/man-pages/man3/getpwnam.3.html
 .. _Jinja template: https://jinja.palletsprojects.com/en/3.1.x/
+.. _Config object: https://globus-compute.readthedocs.io/en/latest/reference/config.html
 .. _Globus Connect Server Identity Mapping Guide: https://docs.globus.org/globus-connect-server/v5.4/identity-mapping-guide/#mapping_recipes
 .. _#help on the Globus Compute Slack: https://funcx.slack.com/archives/C017637NZFA
 .. _the documentation for a number of known working examples: https://globus-compute.readthedocs.io/en/latest/endpoints.html#example-configurations

docs/reference/config.rst

@@ -0,0 +1,7 @@
+The Globus Compute Endpoint Configuration
+=========================================
+
+.. autoclass:: globus_compute_endpoint.endpoint.config.Config
+    :members:
+    :member-order: bysource
+

docs/reference/index.rst

@@ -9,3 +9,4 @@ Globus Compute SDK
     executor
     client
     engine
+    config