Further improve function initialization (#3547)

* Further improve function initialization

The primary purpose of this change is to introduce the "discovery"
subpackage of "runtimes". This package is a common utility that all
runtimes will use to discover their backend spec based on the "container
contract" being finalized currently.

The container contract says that we must interact with customer code
using tools we'd have in a docker container: a fully-contained directory
of code, a "run" command, and environment variables. Backend specs can
be discovered as:

1. The file /backend.yaml at the root of the container
2. The response to http://localhost:${ADMIN_PORT}/backend.yaml when
   running the container's RUN command with ADMIN_PORT set to a free
   port.

The golang runtime fulfills #2 through two steps:

1. The Functions SDK includes the "codegen" package that reads the
   customer code and creates an "autogen" main package that runs
   the customer code as the Firebase runtime (skeleton implementation
   currently).
2. We then start up the Firebase runtime with ADMIN_PORT set and
   fetch /backend.yaml

After this + a minor fix to the template code, we have working Go
deploys!

I also moved gomod parsing into its own file to keep index.ts to a
minimal set of things that aren't easily unit tested. Unfortunately
the diff is largr than it needs to be now.

* PR feedback

src/deploy/functions/prepareFunctionsUpload.ts

@@ -51,6 +51,7 @@ export async function getFunctionsConfig(context: args.Context): Promise<{ [key:
 export async function getEnvs(context: args.Context): Promise<{ [key: string]: string }> {
   const envs = {
     FIREBASE_CONFIG: JSON.stringify(context.firebaseConfig),
+    GCLOUD_PROJECT: context.projectId,
   };
   return Promise.resolve(envs);
 }

src/deploy/functions/runtimes/discovery/index.ts

@@ -0,0 +1,97 @@
+import fetch from "node-fetch";
+import * as fs from "fs";
+import * as path from "path";
+import * as yaml from "js-yaml";
+import { promisify } from "util";
+
+import { logger } from "../../../../logger";
+import * as api from "../../.../../../../api";
+import * as backend from "../../backend";
+import * as runtimes from "..";
+import * as v1alpha1 from "./v1alpha1";
+import { FirebaseError } from "../../../../error";
+
+export const readFileAsync = promisify(fs.readFile);
+
+export function yamlToBackend(
+  yaml: any,
+  project: string,
+  region: string,
+  runtime: runtimes.Runtime
+): backend.Backend {
+  try {
+    if (!yaml.specVersion) {
+      throw new FirebaseError("Expect backend yaml to specify a version number");
+    }
+    if (yaml.specVersion === "v1alpha1") {
+      return v1alpha1.backendFromV1Alpha1(yaml, project, region, runtime);
+    }
+    throw new FirebaseError(
+      "It seems you are using a newer SDK than this version of the CLI can handle. Please update your CLI with `npm install -g firebase-tools`"
+    );
+  } catch (err) {
+    throw new FirebaseError("Failed to parse backend specification", { children: [err] });
+  }
+}
+
+export async function detectFromYaml(
+  directory: string,
+  project: string,
+  runtime: runtimes.Runtime
+): Promise<backend.Backend | undefined> {
+  let text: string;
+  try {
+    text = await exports.readFileAsync(path.join(directory, "backend.yaml"), "utf8");
+  } catch (err) {
+    if (err.code === "ENOENT") {
+      logger.debug("Could not find backend.yaml. Must use http discovery");
+    } else {
+      logger.debug("Unexpected error looking for backend.yaml file:", err);
+    }
+    return;
+  }
+
+  logger.debug("Found backend.yaml. Got spec:", text);
+  const parsed = yaml.load(text);
+  return yamlToBackend(parsed, project, api.functionsDefaultRegion, runtime);
+}
+
+export async function detectFromPort(
+  port: number,
+  project: string,
+  runtime: runtimes.Runtime,
+  timeout: number = 30_000 /* 30s to boot up */
+): Promise<backend.Backend> {
+  // The result type of fetch isn't exported
+  let res: { text(): Promise<string> };
+  const timedOut = new Promise<never>((resolve, reject) => {
+    setTimeout(() => {
+      reject(new FirebaseError("User code failed to load. Cannot determine backend specification"));
+    }, timeout);
+  });
+
+  while (true) {
+    try {
+      res = await Promise.race([fetch(`http://localhost:${port}/backend.yaml`), timedOut]);
+      break;
+    } catch (err) {
+      // Allow us to wait until the server is listening.
+      if (err?.code === "ECONNREFUSED") {
+        continue;
+      }
+      throw err;
+    }
+  }
+
+  const text = await res.text();
+  logger.debug("Got response from /backend.yaml", text);
+
+  let parsed: any;
+  try {
+    parsed = yaml.load(text);
+  } catch (err) {
+    throw new FirebaseError("Failed to parse backend specification", { children: [err] });
+  }
+
+  return yamlToBackend(parsed, project, api.functionsDefaultRegion, runtime);
+}

src/deploy/functions/runtimes/discovery/mockDiscoveryServer.ts

@@ -0,0 +1,14 @@
+import * as express from "express";
+
+const app = express();
+app.get("/backend.yaml", (req, res) => {
+  res.setHeader("content-type", "text/yaml");
+  res.send(process.env.BACKEND);
+});
+
+let port = 8080;
+if (process.env.ADMIN_PORT) {
+  port = Number.parseInt(process.env.ADMIN_PORT);
+}
+console.error("Serving at port", port);
+app.listen(port);

src/deploy/functions/runtimes/discovery/parsing.ts

@@ -0,0 +1,58 @@
+import { FirebaseError } from "../../../../error";
+
+// Use "omit" for output only fields. This allows us to fully exhaust keyof T
+// while still recognizing output-only fields
+export type KeyType = "string" | "number" | "boolean" | "object" | "array" | "omit";
+export function requireKeys<T extends object>(prefix: string, yaml: T, ...keys: (keyof T)[]) {
+  if (prefix) {
+    prefix = prefix + ".";
+  }
+  for (const key of keys) {
+    if (!yaml[key]) {
+      throw new FirebaseError(`Expected key ${prefix + key}`);
+    }
+  }
+}
+
+export function assertKeyTypes<T extends Object>(
+  prefix: string,
+  yaml: T | undefined,
+  schema: Record<keyof T, KeyType>
+) {
+  if (!yaml) {
+    return;
+  }
+  for (const [keyAsString, value] of Object.entries(yaml)) {
+    // I don't know why Object.entries(foo)[0] isn't type of keyof foo...
+    const key = keyAsString as keyof T;
+    const fullKey = prefix ? prefix + "." + key : key;
+    if (!schema[key] || schema[key] === "omit") {
+      throw new FirebaseError(
+        `Unexpected key ${fullKey}. You may need to install a newer version of the Firebase CLI`
+      );
+    }
+    if (schema[key] === "string") {
+      if (typeof value !== "string") {
+        throw new FirebaseError(`Expected ${fullKey} to be string; was ${typeof value}`);
+      }
+    } else if (schema[key] === "number") {
+      if (typeof value !== "number") {
+        throw new FirebaseError(`Expected ${fullKey} to be a number; was ${typeof value}`);
+      }
+    } else if (schema[key] === "boolean") {
+      if (typeof value !== "boolean") {
+        throw new FirebaseError(`Expected ${fullKey} to be a boolean; was ${typeof value}`);
+      }
+    } else if (schema[key] === "array") {
+      if (!Array.isArray(value)) {
+        throw new FirebaseError(`Expected ${fullKey} to be an array; was ${typeof value}`);
+      }
+    } else if (schema[key] === "object") {
+      if (value === null || typeof value !== "object" || Array.isArray(value)) {
+        throw new FirebaseError(`Expected ${fullKey} to be an object; was ${typeof value}`);
+      }
+    } else {
+      throw new FirebaseError("YAML validation is missing a handled type " + schema[key]);
+    }
+  }
+}

src/deploy/functions/runtimes/discovery/v1alpha1.ts

@@ -0,0 +1,167 @@
+import * as backend from "../../backend";
+import * as runtimes from "..";
+import { assertKeyTypes, requireKeys } from "./parsing";
+
+export function backendFromV1Alpha1(
+  yaml: any,
+  project: string,
+  region: string,
+  runtime: runtimes.Runtime
+): backend.Backend {
+  const bkend: backend.Backend = JSON.parse(JSON.stringify(yaml));
+  delete (bkend as any).specVersion;
+  tryValidate(bkend);
+  fillDefaults(bkend, project, region, runtime);
+  return bkend;
+}
+
+function tryValidate(typed: backend.Backend) {
+  // Use a helper type to help guide code complete when writing this function
+  assertKeyTypes("", typed, {
+    requiredAPIs: "object",
+    cloudFunctions: "array",
+    topics: "array",
+    schedules: "array",
+    environmentVariables: "object",
+  });
+  requireKeys("", typed, "cloudFunctions");
+
+  for (let ndx = 0; ndx < typed.cloudFunctions.length; ndx++) {
+    const prefix = `cloudFunctions[${ndx}]`;
+    const func = typed.cloudFunctions[ndx];
+    requireKeys(prefix, func, "apiVersion", "id", "entryPoint", "trigger");
+    assertKeyTypes(prefix, func, {
+      apiVersion: "number",
+      id: "string",
+      region: "string",
+      project: "string",
+      runtime: "string",
+      entryPoint: "string",
+      availableMemoryMb: "number",
+      maxInstances: "number",
+      minInstances: "number",
+      serviceAccountEmail: "string",
+      timeout: "string",
+      trigger: "object",
+      vpcConnector: "string",
+      vpcConnectorEgressSettings: "string",
+      labels: "object",
+      ingressSettings: "string",
+      environmentVariables: "omit",
+      uri: "omit",
+      sourceUploadUrl: "omit",
+    });
+    if (backend.isEventTrigger(func.trigger)) {
+      requireKeys(prefix + ".trigger", func.trigger, "eventType", "eventFilters");
+      assertKeyTypes(prefix + ".trigger", func.trigger, {
+        eventFilters: "object",
+        eventType: "string",
+        retry: "boolean",
+        region: "string",
+        serviceAccountEmail: "string",
+      });
+    } else {
+      assertKeyTypes(prefix + ".trigger", func.trigger, {
+        allowInsecure: "boolean",
+      });
+    }
+  }
+
+  for (let ndx = 0; ndx < typed.topics?.length; ndx++) {
+    let prefix = `topics[${ndx}]`;
+    const topic = typed.topics[ndx];
+    requireKeys(prefix, topic, "id", "targetService");
+    assertKeyTypes(prefix, topic, {
+      id: "string",
+      labels: "object",
+      project: "string",
+      targetService: "object",
+    });
+
+    prefix += ".targetService";
+    requireKeys(prefix, topic.targetService, "id");
+    assertKeyTypes(prefix, topic.targetService, {
+      id: "string",
+      project: "string",
+      region: "string",
+    });
+  }
+
+  for (let ndx = 0; ndx < typed.schedules?.length; ndx++) {
+    let prefix = `schedules[${ndx}]`;
+    const schedule = typed.schedules[ndx];
+    requireKeys(prefix, schedule, "id", "schedule", "transport", "targetService");
+    assertKeyTypes(prefix, schedule, {
+      id: "string",
+      project: "string",
+      retryConfig: "object",
+      schedule: "string",
+      timeZone: "string",
+      transport: "string",
+      targetService: "object",
+    });
+
+    assertKeyTypes(prefix + ".retryConfig", schedule.retryConfig, {
+      maxBackoffDuration: "string",
+      minBackoffDuration: "string",
+      maxDoublings: "number",
+      maxRetryDuration: "string",
+      retryCount: "number",
+    });
+
+    requireKeys((prefix = ".targetService"), schedule.targetService, "id");
+    assertKeyTypes(prefix + ".targetService", schedule.targetService, {
+      id: "string",
+      project: "string",
+      region: "string",
+    });
+  }
+}
+
+function fillDefaults(
+  want: backend.Backend,
+  project: string,
+  region: string,
+  runtime: runtimes.Runtime
+) {
+  want.requiredAPIs = want.requiredAPIs || {};
+  want.environmentVariables = want.environmentVariables || {};
+  want.schedules = want.schedules || [];
+  want.topics = want.topics || [];
+
+  for (const cloudFunction of want.cloudFunctions) {
+    if (!cloudFunction.project) {
+      cloudFunction.project = project;
+    }
+    if (!cloudFunction.region) {
+      cloudFunction.region = region;
+    }
+    if (!cloudFunction.runtime) {
+      cloudFunction.runtime = runtime;
+    }
+  }
+
+  for (const topic of want.topics) {
+    if (!topic.project) {
+      topic.project = project;
+    }
+    if (!topic.targetService.project) {
+      topic.targetService.project = project;
+    }
+    if (!topic.targetService.region) {
+      topic.targetService.region = region;
+    }
+  }
+
+  for (const schedule of want.schedules) {
+    if (!schedule.project) {
+      schedule.project = project;
+    }
+    if (!schedule.targetService.project) {
+      schedule.targetService.project = project;
+    }
+    if (!schedule.targetService.region) {
+      schedule.targetService.region = region;
+    }
+  }
+}