Skip to content

Commit 7a1c5bc

Browse files
inlinedinlined
committed
Further improve function initialization
The primary purpose of this change is to introduce the "discovery" subpackage of "runtimes". This package is a common utility that all runtimes will use to discover their backend spec based on the "container contract" being finalized currently. The container contract says that we must interact with customer code using tools we'd have in a docker container: a fully-contained directory of code, a "run" command, and environment variables. Backend specs can be discovered as: 1. The file /backend.yaml at the root of the container 2. The response to http://localhost:${ADMIN_PORT}/backend.yaml when running the container's RUN command with ADMIN_PORT set to a free port. The golang runtime fulfills #2 through two steps: 1. The Functions SDK includes the "codegen" package that reads the customer code and creates an "autogen" main package that runs the customer code as the Firebase runtime (skeleton implementation currently). 2. We then start up the Firebase runtime with ADMIN_PORT set and fetch /backend.yaml After this + a minor fix to the template code, we have working Go deploys! I also moved gomod parsing into its own file to keep index.ts to a minimal set of things that aren't easily unit tested. Unfortunately the diff is largr than it needs to be now.
1 parent b81da44 commit 7a1c5bc

File tree

12 files changed

+1081
-95
lines changed

12 files changed

+1081
-95
lines changed

‎src/deploy/functions/prepareFunctionsUpload.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ export async function getFunctionsConfig(context: args.Context): Promise<{ [key:
5151
export async function getEnvs(context: args.Context): Promise<{ [key: string]: string }> {
5252
const envs = {
5353
FIREBASE_CONFIG: JSON.stringify(context.firebaseConfig),
54+
GCLOUD_PROJECT: context.projectId,
5455
};
5556
return Promise.resolve(envs);
5657
}

‎src/deploy/functions/runtimes/discovery/index.ts‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import fetch from "node-fetch";
2+
import * as fs from "fs";
3+
import * as path from "path";
4+
import * as yaml from "js-yaml";
5+
import { promisify } from "util";
6+
7+
import { logger } from "../../../../logger";
8+
import * as api from "../../.../../../../api";
9+
import * as backend from "../../backend";
10+
import * as runtimes from "..";
11+
import * as v1alpha1 from "./v1alpha1";
12+
import { FirebaseError } from "../../../../error";
13+
14+
export const readFileAsync = promisify(fs.readFile);
15+
16+
export function yamlToBackend(
17+
yaml: any,
18+
project: string,
19+
region: string,
20+
runtime: runtimes.Runtime
21+
): backend.Backend {
22+
try {
23+
if (!yaml.specVersion) {
24+
throw new FirebaseError("Expect backend yaml to specify a version number");
25+
}
26+
if (yaml.specVersion === "v1alpha1") {
27+
return v1alpha1.backendFromV1Alpha1(yaml, project, region, runtime);
28+
}
29+
throw new FirebaseError(
30+
"It seems you are using a newer SDK than this version of the CLI can handle. Please update your CLI with `npm install -g firebase-tools`"
31+
);
32+
} catch (err) {
33+
throw new FirebaseError("Failed to parse backend specification", { children: [err] });
34+
}
35+
}
36+
37+
export async function detectFromYaml(
38+
directory: string,
39+
project: string,
40+
runtime: runtimes.Runtime
41+
): Promise<backend.Backend | undefined> {
42+
let text: string;
43+
try {
44+
text = await exports.readFileAsync(path.join(directory, "backend.yaml"), "utf8");
45+
} catch (err) {
46+
if (err.code === "ENOENT") {
47+
logger.debug("Could not find backend.yaml. Must use http discovery");
48+
} else {
49+
logger.debug("Unexpected error looking for backend.yaml file:", err);
50+
}
51+
return;
52+
}
53+
54+
logger.debug("Found backend.yaml. Got spec:", text);
55+
// TOODO(inlined): use a schema instead of manually checking everything or blindly trusting input.
56+
const parsed = yaml.load(text);
57+
return yamlToBackend(parsed, project, api.functionsDefaultRegion, runtime);
58+
}
59+
60+
export async function detectFromPort(
61+
port: number,
62+
project: string,
63+
runtime: runtimes.Runtime,
64+
timeout: number = 30_000 /* 30s to boot up */
65+
): Promise<backend.Backend> {
66+
// The result type of fetch isn't exported
67+
let res: { text(): Promise<string> };
68+
const timedOut = new Promise<never>((resolve, reject) => {
69+
setTimeout(() => {
70+
reject(new FirebaseError("User code failed to load. Cannot determine backend specification"));
71+
}, timeout);
72+
});
73+
74+
while (true) {
75+
try {
76+
res = await Promise.race([fetch(`http://localhost:${port}/backend.yaml`), timedOut]);
77+
break;
78+
} catch (err) {
79+
// Allow us to wait until the server is listening.
80+
if (/ECONNREFUSED/.exec(err?.message)) {
81+
continue;
82+
}
83+
throw err;
84+
}
85+
}
86+
87+
const text = await res.text();
88+
logger.debug("Got response from /backend.yaml", text);
89+
90+
let parsed: any;
91+
try {
92+
parsed = yaml.load(text);
93+
} catch (err) {
94+
throw new FirebaseError("Failed to parse backend specification", { children: [err] });
95+
}
96+
97+
return yamlToBackend(parsed, project, api.functionsDefaultRegion, runtime);
98+
}

‎src/deploy/functions/runtimes/discovery/mockDiscoveryServer.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import * as express from "express";
2+
3+
const app = express();
4+
app.get("/backend.yaml", (req, res) => {
5+
res.setHeader("content-type", "text/yaml");
6+
res.send(process.env.BACKEND);
7+
});
8+
9+
let port = 8080;
10+
if (process.env.ADMIN_PORT) {
11+
port = Number.parseInt(process.env.ADMIN_PORT);
12+
}
13+
console.error("Serving at port", port);
14+
app.listen(port);

‎src/deploy/functions/runtimes/discovery/parsing.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import { FirebaseError } from "../../../../error";
2+
3+
// Use "omit" for output only fields. This allows us to fully exhaust keyof T
4+
// while still recognizing output-only fields
5+
export type Type = "string" | "number" | "boolean" | "object" | "array" | "omit";
6+
export function requireKeys<T extends object>(prefix: string, yaml: T, ...keys: (keyof T)[]) {
7+
if (prefix) {
8+
prefix = prefix + ".";
9+
}
10+
for (const key of keys) {
11+
if (!yaml[key]) {
12+
throw new FirebaseError(`Expected key ${prefix + key}`);
13+
}
14+
}
15+
}
16+
17+
export function assertKeyTypes<T extends Object>(
18+
prefix: string,
19+
yaml: T | undefined,
20+
schema: Record<keyof T, Type>
21+
) {
22+
if (!yaml) {
23+
return;
24+
}
25+
for (const [keyAsString, value] of Object.entries(yaml)) {
26+
// I don't know why Object.entries(foo)[0] isn't type of keyof foo...
27+
const key = keyAsString as keyof T;
28+
const fullKey = prefix ? prefix + "." + key : key;
29+
if (!schema[key] || schema[key] === "omit") {
30+
throw new FirebaseError(
31+
`Unexpected key ${fullKey}. You may need to install a newer version of the Firebase CLI`
32+
);
33+
}
34+
if (schema[key] === "string") {
35+
if (typeof value !== "string") {
36+
throw new FirebaseError(`Expected ${fullKey} to be string; was ${typeof value}`);
37+
}
38+
} else if (schema[key] === "number") {
39+
if (typeof value !== "number") {
40+
throw new FirebaseError(`Expected ${fullKey} to be a number; was ${typeof value}`);
41+
}
42+
} else if (schema[key] === "boolean") {
43+
if (typeof value !== "boolean") {
44+
throw new FirebaseError(`Expected ${fullKey} to be a boolean; was ${typeof value}`);
45+
}
46+
} else if (schema[key] === "array") {
47+
if (!Array.isArray(value)) {
48+
throw new FirebaseError(`Expected ${fullKey} to be an array; was ${typeof value}`);
49+
}
50+
} else if (schema[key] === "object") {
51+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
52+
throw new FirebaseError(`Expected ${fullKey} to be an object; was ${typeof value}`);
53+
}
54+
} else {
55+
throw new FirebaseError("YAML validation is missing a handled type " + schema[key]);
56+
}
57+
}
58+
}

‎src/deploy/functions/runtimes/discovery/v1alpha1.ts‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
import * as _ from "lodash";
2+
3+
import * as backend from "../../backend";
4+
import * as runtimes from "..";
5+
import { assertKeyTypes, requireKeys } from "./parsing";
6+
7+
export function backendFromV1Alpha1(
8+
yaml: any,
9+
project: string,
10+
region: string,
11+
runtime: runtimes.Runtime
12+
): backend.Backend {
13+
const bkend: backend.Backend = _.cloneDeep(yaml);
14+
delete (bkend as any).specVersion;
15+
tryValidate(bkend);
16+
fillDefaults(bkend, project, region, runtime);
17+
return bkend;
18+
}
19+
20+
function tryValidate(typed: backend.Backend) {
21+
// Use a helper type to help guide code complete when writing this function
22+
assertKeyTypes("", typed, {
23+
requiredAPIs: "object",
24+
cloudFunctions: "array",
25+
topics: "array",
26+
schedules: "array",
27+
environmentVariables: "object",
28+
});
29+
requireKeys("", typed, "cloudFunctions");
30+
31+
for (let ndx = 0; ndx < typed.cloudFunctions.length; ndx++) {
32+
const prefix = `cloudFunctions[${ndx}]`;
33+
const func = typed.cloudFunctions[ndx];
34+
requireKeys(prefix, func, "apiVersion", "id", "entryPoint", "trigger");
35+
assertKeyTypes(prefix, func, {
36+
apiVersion: "number",
37+
id: "string",
38+
region: "string",
39+
project: "string",
40+
runtime: "string",
41+
entryPoint: "string",
42+
availableMemoryMb: "number",
43+
maxInstances: "number",
44+
minInstances: "number",
45+
serviceAccountEmail: "string",
46+
timeout: "string",
47+
trigger: "object",
48+
vpcConnector: "string",
49+
vpcConnectorEgressSettings: "string",
50+
labels: "object",
51+
ingressSettings: "string",
52+
environmentVariables: "omit",
53+
uri: "omit",
54+
sourceUploadUrl: "omit",
55+
});
56+
if (backend.isEventTrigger(func.trigger)) {
57+
requireKeys(prefix + ".trigger", func.trigger, "eventType", "eventFilters");
58+
assertKeyTypes(prefix + ".trigger", func.trigger, {
59+
eventFilters: "object",
60+
eventType: "string",
61+
retry: "boolean",
62+
region: "string",
63+
serviceAccountEmail: "string",
64+
});
65+
} else {
66+
assertKeyTypes(prefix + ".trigger", func.trigger, {
67+
allowInsecure: "boolean",
68+
});
69+
}
70+
}
71+
72+
for (let ndx = 0; ndx < typed.topics?.length; ndx++) {
73+
let prefix = `topics[${ndx}]`;
74+
const topic = typed.topics[ndx];
75+
requireKeys(prefix, topic, "id", "targetService");
76+
assertKeyTypes(prefix, topic, {
77+
id: "string",
78+
labels: "object",
79+
project: "string",
80+
targetService: "object",
81+
});
82+
83+
prefix += ".targetService";
84+
requireKeys(prefix, topic.targetService, "id");
85+
assertKeyTypes(prefix, topic.targetService, {
86+
id: "string",
87+
project: "string",
88+
region: "string",
89+
});
90+
}
91+
92+
for (let ndx = 0; ndx < typed.schedules?.length; ndx++) {
93+
let prefix = `schedules[${ndx}]`;
94+
const schedule = typed.schedules[ndx];
95+
requireKeys(prefix, schedule, "id", "schedule", "transport", "targetService");
96+
assertKeyTypes(prefix, schedule, {
97+
id: "string",
98+
project: "string",
99+
retryConfig: "object",
100+
schedule: "string",
101+
timeZone: "string",
102+
transport: "string",
103+
targetService: "object",
104+
});
105+
106+
assertKeyTypes(prefix + ".retryConfig", schedule.retryConfig, {
107+
maxBackoffDuration: "string",
108+
minBackoffDuration: "string",
109+
maxDoublings: "number",
110+
maxRetryDuration: "string",
111+
retryCount: "number",
112+
});
113+
114+
requireKeys((prefix = ".targetService"), schedule.targetService, "id");
115+
assertKeyTypes(prefix + ".targetService", schedule.targetService, {
116+
id: "string",
117+
project: "string",
118+
region: "string",
119+
});
120+
}
121+
}
122+
123+
function fillDefaults(
124+
want: backend.Backend,
125+
project: string,
126+
region: string,
127+
runtime: runtimes.Runtime
128+
) {
129+
want.requiredAPIs = want.requiredAPIs || {};
130+
want.environmentVariables = want.environmentVariables || {};
131+
want.schedules = want.schedules || [];
132+
want.topics = want.topics || [];
133+
134+
for (const cloudFunction of want.cloudFunctions) {
135+
if (!cloudFunction.project) {
136+
cloudFunction.project = project;
137+
}
138+
if (!cloudFunction.region) {
139+
cloudFunction.region = region;
140+
}
141+
if (!cloudFunction.runtime) {
142+
cloudFunction.runtime = runtime;
143+
}
144+
}
145+
146+
for (const topic of want.topics) {
147+
if (!topic.project) {
148+
topic.project = project;
149+
}
150+
if (!topic.targetService.project) {
151+
topic.targetService.project = project;
152+
}
153+
if (!topic.targetService.region) {
154+
topic.targetService.region = region;
155+
}
156+
}
157+
158+
for (const schedule of want.schedules) {
159+
if (!schedule.project) {
160+
schedule.project = project;
161+
}
162+
if (!schedule.targetService.project) {
163+
schedule.targetService.project = project;
164+
}
165+
if (!schedule.targetService.region) {
166+
schedule.targetService.region = region;
167+
}
168+
}
169+
}

0 commit comments

Comments
 (0)