Skip to content

Commit 785536e

Browse files
fredzqmjoehan
fredzqm
and
joehan
authored
[FDC] Deploy should provision Cloud SQL async (#9004)
* save * save * working * changelog * tweaks * tweaks * error handling * Update CHANGELOG.md Co-authored-by: Joe Hanley <[email protected]> * Update CHANGELOG.md Co-authored-by: Joe Hanley <[email protected]> --------- Co-authored-by: Joe Hanley <[email protected]>
1 parent 00ae2e4 commit 785536e

File tree

16 files changed

+266
-260
lines changed

16 files changed

+266
-260
lines changed

‎CHANGELOG.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
- Fixed a bug when deploying firestore indexes failed due to broken index comparison logic (#8859)
22
- Added prefix support for multi-instance Cloud Functions extension parameters. (#8911)
3-
- Fixed a bug when `firebase deploy --only dataconnect` doesn't include GQL in nested folders (#8981)
43
- Make it possible to init a dataconnect project in non interactive mode (#8993)
54
- Added 2 new MCP tools for crashlytics `get_sample_crash_for_issue` and `get_issue_details` (#8995)
65
- Use Gemini to generate schema and seed_data.gql in `firebase init dataconnect` (#8988)
6+
- Fixed a bug when `firebase deploy --only dataconnect` didn't include GQL files in nested folders (#8981)
7+
- Changed `firebase deploy` create Cloud SQL instances asynchronously (#9004)

‎src/commands/dataconnect-sql-grant.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { grantRoleToUserInSchema } from "../dataconnect/schemaMigration";
88
import { requireAuth } from "../requireAuth";
99
import { FirebaseError } from "../error";
1010
import { fdcSqlRoleMap } from "../gcp/cloudsql/permissionsSetup";
11+
import { iamUserIsCSQLAdmin } from "../gcp/cloudsql/cloudsqladmin";
1112

1213
const allowedRoles = Object.keys(fdcSqlRoleMap);
1314

@@ -38,6 +39,14 @@ export const command = new Command("dataconnect:sql:grant [serviceId]")
3839
throw new FirebaseError(`Role should be one of ${allowedRoles.join(" | ")}.`);
3940
}
4041

42+
// Make sure current user can perform this action.
43+
const userIsCSQLAdmin = await iamUserIsCSQLAdmin(options);
44+
if (!userIsCSQLAdmin) {
45+
throw new FirebaseError(
46+
`Only users with 'roles/cloudsql.admin' can grant SQL roles. If you do not have this role, ask your database administrator to run this command or manually grant ${role} to ${email}`,
47+
);
48+
}
49+
4150
const projectId = needProjectId(options);
4251
await ensureApis(projectId);
4352
const serviceInfo = await pickService(projectId, options.config, serviceId);

‎src/commands/dataconnect-sql-setup.ts

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,7 @@ import { ensureApis } from "../dataconnect/ensureApis";
99
import { setupSQLPermissions, getSchemaMetadata } from "../gcp/cloudsql/permissionsSetup";
1010
import { DEFAULT_SCHEMA } from "../gcp/cloudsql/permissions";
1111
import { getIdentifiers, ensureServiceIsConnectedToCloudSql } from "../dataconnect/schemaMigration";
12-
import { getIAMUser } from "../gcp/cloudsql/connect";
13-
import * as cloudSqlAdminClient from "../gcp/cloudsql/cloudsqladmin";
12+
import { setupIAMUsers } from "../gcp/cloudsql/connect";
1413

1514
export const command = new Command("dataconnect:sql:setup [serviceId]")
1615
.description("set up your CloudSQL database")
@@ -41,9 +40,8 @@ export const command = new Command("dataconnect:sql:setup [serviceId]")
4140
/* linkIfNotConnected=*/ true,
4241
);
4342

44-
// Create an IAM user for the current identity.
45-
const { user, mode } = await getIAMUser(options);
46-
await cloudSqlAdminClient.createUser(projectId, instanceId, mode, user);
43+
// Setup the IAM user for the current identity.
44+
await setupIAMUsers(instanceId, options);
4745

4846
const schemaInfo = await getSchemaMetadata(instanceId, databaseId, DEFAULT_SCHEMA, options);
4947
await setupSQLPermissions(instanceId, databaseId, schemaInfo, options);

‎src/dataconnect/client.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,14 +125,15 @@ export async function listSchemas(
125125
export async function upsertSchema(
126126
schema: types.Schema,
127127
validateOnly: boolean = false,
128+
async: boolean = false,
128129
): Promise<types.Schema | undefined> {
129130
const op = await dataconnectClient().patch<types.Schema, types.Schema>(`${schema.name}`, schema, {
130131
queryParams: {
131132
allowMissing: "true",
132133
validateOnly: validateOnly ? "true" : "false",
133134
},
134135
});
135-
if (validateOnly) {
136+
if (validateOnly || async) {
136137
return;
137138
}
138139
return operationPoller.pollOperation<types.Schema>({

‎src/dataconnect/freeTrial.ts

Lines changed: 0 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import * as clc from "colorette";
22

33
import { queryTimeSeries, CmQuery } from "../gcp/cloudmonitoring";
4-
import { listInstances } from "../gcp/cloudsql/cloudsqladmin";
54
import * as utils from "../utils";
65

76
export function freeTrialTermsLink(): string {
@@ -40,49 +39,6 @@ export async function checkFreeTrialInstanceUsed(projectId: string): Promise<boo
4039
return used;
4140
}
4241

43-
export async function getFreeTrialInstanceId(projectId: string): Promise<string | undefined> {
44-
const instances = await listInstances(projectId);
45-
return instances.find((i) => i.settings.userLabels?.["firebase-data-connect"] === "ft")?.name;
46-
}
47-
48-
export async function isFreeTrialError(err: any, projectId: string): Promise<boolean> {
49-
// checkFreeTrialInstanceUsed is also called to ensure the request didn't fail due to an unrelated quota issue.
50-
return err.message.includes("Quota Exhausted") && (await checkFreeTrialInstanceUsed(projectId))
51-
? true
52-
: false;
53-
}
54-
55-
export function printFreeTrialUnavailable(
56-
projectId: string,
57-
configYamlPath: string,
58-
instanceId?: string,
59-
): void {
60-
if (!instanceId) {
61-
utils.logLabeledError(
62-
"dataconnect",
63-
"The CloudSQL free trial has already been used on this project.",
64-
);
65-
utils.logLabeledError(
66-
"dataconnect",
67-
`You may create or use a paid CloudSQL instance by visiting https://console.cloud.google.com/sql/instances`,
68-
);
69-
return;
70-
}
71-
utils.logLabeledError(
72-
"dataconnect",
73-
`Project '${projectId} already has a CloudSQL instance '${instanceId}' on the Firebase Data Connect no-cost trial.`,
74-
);
75-
const reuseHint =
76-
`To use a different database in the same instance, ${clc.bold(`change the ${clc.blue("instanceId")} to "${instanceId}"`)} and update ${clc.blue("location")} in ` +
77-
`${clc.green(configYamlPath)}.`;
78-
79-
utils.logLabeledError("dataconnect", reuseHint);
80-
utils.logLabeledError(
81-
"dataconnect",
82-
`Alternatively, you may create a new (paid) CloudSQL instance at https://console.cloud.google.com/sql/instances`,
83-
);
84-
}
85-
8642
export function upgradeInstructions(projectId: string): string {
8743
return `To provision a CloudSQL Postgres instance on the Firebase Data Connect no-cost trial:
8844

‎src/dataconnect/provisionCloudSql.ts

Lines changed: 111 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -4,139 +4,160 @@ import { grantRolesToCloudSqlServiceAccount } from "./checkIam";
44
import { Instance } from "../gcp/cloudsql/types";
55
import { promiseWithSpinner } from "../utils";
66
import { logger } from "../logger";
7+
import { freeTrialTermsLink, checkFreeTrialInstanceUsed } from "./freeTrial";
78

89
const GOOGLE_ML_INTEGRATION_ROLE = "roles/aiplatform.user";
910

10-
import { freeTrialTermsLink, checkFreeTrialInstanceUsed } from "./freeTrial";
11+
/** Sets up a Cloud SQL instance, database and its permissions. */
12+
export async function setupCloudSql(args: {
13+
projectId: string;
14+
location: string;
15+
instanceId: string;
16+
databaseId: string;
17+
requireGoogleMlIntegration: boolean;
18+
dryRun?: boolean;
19+
}): Promise<void> {
20+
await upsertInstance({ ...args });
21+
const { projectId, instanceId, requireGoogleMlIntegration, dryRun } = args;
22+
if (requireGoogleMlIntegration && !dryRun) {
23+
await grantRolesToCloudSqlServiceAccount(projectId, instanceId, [GOOGLE_ML_INTEGRATION_ROLE]);
24+
}
25+
}
1126

12-
export async function provisionCloudSql(args: {
27+
async function upsertInstance(args: {
1328
projectId: string;
1429
location: string;
1530
instanceId: string;
1631
databaseId: string;
17-
enableGoogleMlIntegration: boolean;
18-
waitForCreation: boolean;
19-
silent?: boolean;
32+
requireGoogleMlIntegration: boolean;
2033
dryRun?: boolean;
21-
}): Promise<string> {
22-
let connectionName = ""; // Not used yet, will be used for schema migration
23-
const {
24-
projectId,
25-
location,
26-
instanceId,
27-
databaseId,
28-
enableGoogleMlIntegration,
29-
waitForCreation,
30-
silent,
31-
dryRun,
32-
} = args;
34+
}): Promise<void> {
35+
const { projectId, instanceId, requireGoogleMlIntegration, dryRun } = args;
3336
try {
3437
const existingInstance = await cloudSqlAdminClient.getInstance(projectId, instanceId);
35-
silent ||
36-
utils.logLabeledBullet("dataconnect", `Found existing Cloud SQL instance ${instanceId}.`);
37-
connectionName = existingInstance?.connectionName || "";
38-
const why = getUpdateReason(existingInstance, enableGoogleMlIntegration);
38+
utils.logLabeledBullet("dataconnect", `Found existing Cloud SQL instance ${instanceId}.`);
39+
const why = getUpdateReason(existingInstance, requireGoogleMlIntegration);
3940
if (why) {
40-
const cta = dryRun
41-
? `It will be updated on your next deploy.`
42-
: `Updating instance. This may take a few minutes...`;
43-
silent ||
41+
if (dryRun) {
4442
utils.logLabeledBullet(
4543
"dataconnect",
46-
`Instance ${instanceId} settings not compatible with Firebase Data Connect. ` + cta + why,
44+
`Cloud SQL instance ${instanceId} settings not compatible with Firebase Data Connect. ` +
45+
`It will be updated on your next deploy.` +
46+
why,
47+
);
48+
} else {
49+
utils.logLabeledBullet(
50+
"dataconnect",
51+
`Cloud SQL instance ${instanceId} settings not compatible with Firebase Data Connect. ` +
52+
why,
4753
);
48-
if (!dryRun) {
4954
await promiseWithSpinner(
5055
() =>
5156
cloudSqlAdminClient.updateInstanceForDataConnect(
5257
existingInstance,
53-
enableGoogleMlIntegration,
58+
requireGoogleMlIntegration,
5459
),
55-
"Updating your instance...",
60+
"Updating your Cloud SQL instance...",
5661
);
57-
silent || utils.logLabeledBullet("dataconnect", "Instance updated");
5862
}
5963
}
64+
await upsertDatabase({ ...args });
6065
} catch (err: any) {
61-
// We only should catch NOT FOUND errors
6266
if (err.status !== 404) {
6367
throw err;
6468
}
65-
const cta = dryRun ? "It will be created on your next deploy" : "Creating it now.";
66-
const freeTrialUsed = await checkFreeTrialInstanceUsed(projectId);
67-
silent ||
68-
utils.logLabeledBullet(
69-
"dataconnect",
70-
`CloudSQL instance '${instanceId}' not found.` + cta + freeTrialUsed
71-
? ""
72-
: `\nThis instance is provided under the terms of the Data Connect no-cost trial ${freeTrialTermsLink()}` +
73-
dryRun
74-
? `\nMonitor the progress at ${cloudSqlAdminClient.instanceConsoleLink(projectId, instanceId)}`
75-
: "",
76-
);
69+
// Cloud SQL instance is not found, start its creation.
70+
await createInstance({ ...args });
71+
}
72+
}
7773

78-
if (!dryRun) {
79-
const newInstance = await promiseWithSpinner(
80-
() =>
81-
cloudSqlAdminClient.createInstance({
82-
projectId,
83-
location,
84-
instanceId,
85-
enableGoogleMlIntegration,
86-
waitForCreation,
87-
freeTrial: !freeTrialUsed,
88-
}),
89-
"Creating your instance...",
90-
);
91-
if (newInstance) {
92-
silent || utils.logLabeledBullet("dataconnect", "Instance created");
93-
connectionName = newInstance?.connectionName || "";
94-
} else {
95-
silent ||
96-
utils.logLabeledBullet(
97-
"dataconnect",
98-
"Cloud SQL instance creation started. While it is being set up, your data will be saved in a temporary database. When it is ready, your data will be migrated.",
99-
);
100-
return connectionName;
101-
}
102-
}
74+
async function createInstance(args: {
75+
projectId: string;
76+
location: string;
77+
instanceId: string;
78+
requireGoogleMlIntegration: boolean;
79+
dryRun?: boolean;
80+
}): Promise<void> {
81+
const { projectId, location, instanceId, requireGoogleMlIntegration, dryRun } = args;
82+
const freeTrialUsed = await checkFreeTrialInstanceUsed(projectId);
83+
if (dryRun) {
84+
utils.logLabeledBullet(
85+
"dataconnect",
86+
`Cloud SQL Instance ${instanceId} not found. It will be created on your next deploy.`,
87+
);
88+
} else {
89+
await cloudSqlAdminClient.createInstance({
90+
projectId,
91+
location,
92+
instanceId,
93+
enableGoogleMlIntegration: requireGoogleMlIntegration,
94+
freeTrial: !freeTrialUsed,
95+
});
96+
utils.logLabeledBullet(
97+
"dataconnect",
98+
cloudSQLBeingCreated(projectId, instanceId, !freeTrialUsed),
99+
);
103100
}
101+
}
102+
103+
/**
104+
* Returns a message indicating that a Cloud SQL instance is being created.
105+
*/
106+
export function cloudSQLBeingCreated(
107+
projectId: string,
108+
instanceId: string,
109+
includeFreeTrialToS?: boolean,
110+
): string {
111+
return (
112+
`Cloud SQL Instance ${instanceId} is being created.` +
113+
(includeFreeTrialToS
114+
? `\nThis instance is provided under the terms of the Data Connect no-cost trial ${freeTrialTermsLink()}`
115+
: "") +
116+
`
117+
Meanwhile, your data are saved in a temporary database and will be migrated once complete. Monitor its progress at
104118
119+
${cloudSqlAdminClient.instanceConsoleLink(projectId, instanceId)}
120+
`
121+
);
122+
}
123+
124+
async function upsertDatabase(args: {
125+
projectId: string;
126+
instanceId: string;
127+
databaseId: string;
128+
dryRun?: boolean;
129+
}): Promise<void> {
130+
const { projectId, instanceId, databaseId, dryRun } = args;
105131
try {
106132
await cloudSqlAdminClient.getDatabase(projectId, instanceId, databaseId);
107-
silent || utils.logLabeledBullet("dataconnect", `Found existing database ${databaseId}.`);
133+
utils.logLabeledBullet("dataconnect", `Found existing Postgres Database ${databaseId}.`);
108134
} catch (err: any) {
109-
if (err.status === 404) {
110-
if (dryRun) {
111-
silent ||
112-
utils.logLabeledBullet(
113-
"dataconnect",
114-
`Postgres database ${databaseId} not found. It will be created on your next deploy.`,
115-
);
116-
} else {
117-
await cloudSqlAdminClient.createDatabase(projectId, instanceId, databaseId);
118-
silent || utils.logLabeledBullet("dataconnect", `Postgres database ${databaseId} created.`);
119-
}
120-
} else {
135+
if (err.status !== 404) {
121136
// Skip it if the database is not accessible.
122137
// Possible that the CSQL instance is in the middle of something.
123-
logger.debug(`Unexpected error from CloudSQL: ${err}`);
124-
silent || utils.logLabeledWarning("dataconnect", `Database ${databaseId} is not accessible.`);
138+
logger.debug(`Unexpected error from Cloud SQL: ${err}`);
139+
utils.logLabeledWarning("dataconnect", `Postgres Database ${databaseId} is not accessible.`);
140+
return;
141+
}
142+
if (dryRun) {
143+
utils.logLabeledBullet(
144+
"dataconnect",
145+
`Postgres Database ${databaseId} not found. It will be created on your next deploy.`,
146+
);
147+
} else {
148+
await cloudSqlAdminClient.createDatabase(projectId, instanceId, databaseId);
149+
utils.logLabeledBullet("dataconnect", `Postgres Database ${databaseId} created.`);
125150
}
126151
}
127-
if (enableGoogleMlIntegration && !dryRun) {
128-
await grantRolesToCloudSqlServiceAccount(projectId, instanceId, [GOOGLE_ML_INTEGRATION_ROLE]);
129-
}
130-
return connectionName;
131152
}
132153

133154
/**
134-
* Validate that existing CloudSQL instances have the necessary settings.
155+
* Validate that existing Cloud SQL instances have the necessary settings.
135156
*/
136157
export function getUpdateReason(instance: Instance, requireGoogleMlIntegration: boolean): string {
137158
let reason = "";
138159
const settings = instance.settings;
139-
// CloudSQL instances must have public IP enabled to be used with Firebase Data Connect.
160+
// Cloud SQL instances must have public IP enabled to be used with Firebase Data Connect.
140161
if (!settings.ipConfiguration?.ipv4Enabled) {
141162
reason += "\n - to enable public IP.";
142163
}
@@ -154,7 +175,7 @@ export function getUpdateReason(instance: Instance, requireGoogleMlIntegration:
154175
}
155176
}
156177

157-
// CloudSQL instances must have IAM authentication enabled to be used with Firebase Data Connect.
178+
// Cloud SQL instances must have IAM authentication enabled to be used with Firebase Data Connect.
158179
const isIamEnabled =
159180
settings.databaseFlags?.some(
160181
(f) => f.name === "cloudsql.iam_authentication" && f.value === "on",

0 commit comments

Comments
 (0)