Fdc brownfield setup

CHANGELOG.md

@@ -1 +1,2 @@
 - Fix webframeworks deployments when using `site` in `firebase.json`. (#8295)
+- Add support for brownfield project onboard `dataconnect:sql:setup` (#8150)

src/commands/dataconnect-sql-grant.ts

@@ -7,7 +7,7 @@ import { pickService } from "../dataconnect/fileUtils";
 import { grantRoleToUserInSchema } from "../dataconnect/schemaMigration";
 import { requireAuth } from "../requireAuth";
 import { FirebaseError } from "../error";
-import { fdcSqlRoleMap } from "../gcp/cloudsql/permissions";
+import { fdcSqlRoleMap } from "../gcp/cloudsql/permissions_setup";
 
 const allowedRoles = Object.keys(fdcSqlRoleMap);
 

src/commands/dataconnect-sql-setup.ts

@@ -0,0 +1,38 @@
+import { Command } from "../command";
+import { Options } from "../options";
+import { needProjectId } from "../projectUtils";
+import { pickService } from "../dataconnect/fileUtils";
+import { FirebaseError } from "../error";
+import { requireAuth } from "../requireAuth";
+import { requirePermissions } from "../requirePermissions";
+import { ensureApis } from "../dataconnect/ensureApis";
+import { setupSQLPermissions, getSchemaMetadata } from "../gcp/cloudsql/permissions_setup";
+import { DEFAULT_SCHEMA } from "../gcp/cloudsql/permissions";
+import { getIdentifiers } from "../dataconnect/schemaMigration";
+
+export const command = new Command("dataconnect:sql:setup [serviceId]")
+  .description("Setup your CloudSQL database")
+  .before(requirePermissions, [
+    "firebasedataconnect.services.list",
+    "firebasedataconnect.schemas.list",
+    "firebasedataconnect.schemas.update",
+    "cloudsql.instances.connect",
+  ])
+  .before(requireAuth)
+  .action(async (serviceId: string, options: Options) => {
+    const projectId = needProjectId(options);
+    await ensureApis(projectId);
+    const serviceInfo = await pickService(projectId, options.config, serviceId);
+    const instanceId =
+      serviceInfo.dataConnectYaml.schema.datasource.postgresql?.cloudSql.instanceId;
+    if (!instanceId) {
+      throw new FirebaseError(
+        "dataconnect.yaml is missing field schema.datasource.postgresql.cloudsql.instanceId",
+      );
+    }
+
+    const { databaseId } = getIdentifiers(serviceInfo.schema);
+
+    const schemaInfo = await getSchemaMetadata(instanceId, databaseId, DEFAULT_SCHEMA, options);
+    await setupSQLPermissions(instanceId, databaseId, schemaInfo, options);
+  });

src/commands/index.ts

@@ -2,11 +2,11 @@
 /**
  * Loads all commands for our parser.
  */
 export function load(client: any): any {
   function loadCommand(name: string) {
     const t0 = process.hrtime.bigint();
     const { command: cmd } = require(`./${name}`);
     cmd.register(client);
     const t1 = process.hrtime.bigint();
     const diffMS = (t1 - t0) / BigInt(1e6);
     if (diffMS > 75) {
@@ -14,7 +14,7 @@
       // console.error(`Loading ${name} took ${diffMS}ms`);
     }
 
     return cmd.runner();
   }
 
   const t0 = process.hrtime.bigint();
@@ -221,6 +221,7 @@
   client.dataconnect.services.list = loadCommand("dataconnect-services-list");
   client.dataconnect.sql = {};
   client.dataconnect.sql.diff = loadCommand("dataconnect-sql-diff");
+  client.dataconnect.sql.setup = loadCommand("dataconnect-sql-setup");
   client.dataconnect.sql.migrate = loadCommand("dataconnect-sql-migrate");
   client.dataconnect.sql.grant = loadCommand("dataconnect-sql-grant");
   client.dataconnect.sql.shell = loadCommand("dataconnect-sql-shell");

src/dataconnect/schemaMigration.ts

@@ -4,26 +4,29 @@ import { format } from "sql-formatter";
 import { IncompatibleSqlSchemaError, Diff, SCHEMA_ID, SchemaValidation } from "./types";
 import { getSchema, upsertSchema, deleteConnector } from "./client";
 import {
-  setupIAMUsers,
   getIAMUser,
   executeSqlCmdsAsIamUser,
   executeSqlCmdsAsSuperUser,
   toDatabaseUser,
 } from "../gcp/cloudsql/connect";
+import { needProjectId } from "../projectUtils";
 import {
-  firebaseowner,
-  iamUserIsCSQLAdmin,
   checkSQLRoleIsGranted,
   fdcSqlRoleMap,
-} from "../gcp/cloudsql/permissions";
-import * as cloudSqlAdminClient from "../gcp/cloudsql/cloudsqladmin";
-import { needProjectId } from "../projectUtils";
+  setupSQLPermissions,
+  getSchemaMetadata,
+  SchemaSetupStatus,
+} from "../gcp/cloudsql/permissions_setup";
+import { DEFAULT_SCHEMA, firebaseowner } from "../gcp/cloudsql/permissions";
 import { promptOnce, confirm } from "../prompt";
 import { logger } from "../logger";
 import { Schema } from "./types";
 import { Options } from "../options";
 import { FirebaseError } from "../error";
 import { logLabeledBullet, logLabeledWarning, logLabeledSuccess } from "../utils";
+import { iamUserIsCSQLAdmin } from "../gcp/cloudsql/cloudsqladmin";
+import * as cloudSqlAdminClient from "../gcp/cloudsql/cloudsqladmin";
+
 import * as errors from "./errors";
 
 export async function diffSchema(
@@ -236,10 +239,34 @@ export async function grantRoleToUserInSchema(options: Options, schema: Schema)
     );
   }
 
-  // Run the database roles setup. This should be idempotent.
-  await setupIAMUsers(instanceId, databaseId, options);
+  // Make sure we have the right setup for the requested role grant.
+  const schemaInfo = await getSchemaMetadata(instanceId, databaseId, DEFAULT_SCHEMA, options);
+  let isGreenfieldSetup = schemaInfo.setupStatus === SchemaSetupStatus.GreenField;
+  switch (schemaInfo.setupStatus) {
+    case SchemaSetupStatus.NotSetup:
+    case SchemaSetupStatus.NotFound:
+      const newSetupStatus = await setupSQLPermissions(instanceId, databaseId, schemaInfo, options);
+      isGreenfieldSetup = newSetupStatus === SchemaSetupStatus.GreenField;
+      break;
+    default:
+      logger.info(
+        `Detected schema "${schemaInfo.name}" is setup in ${schemaInfo.setupStatus} mode. Skipping Setup.`,
+      );
+      break;
+  }
+
+  // Edge case: we can't grant firebase owner unless database is greenfield.
+  if (!isGreenfieldSetup && fdcSqlRole === firebaseowner(databaseId, DEFAULT_SCHEMA)) {
+    const newSetupStatus = await setupSQLPermissions(instanceId, databaseId, schemaInfo, options);
 
-  // Upsert user account into the database.
+    if (newSetupStatus !== SchemaSetupStatus.GreenField) {
+      throw new FirebaseError(
+        `Can't grant owner rule for brownfield databases. Consider fully migrating your database to FDC using 'firebase dataconnect:sql:setup'`,
+      );
+    }
+  }
+
+  // Upsert new user account into the database.
   await cloudSqlAdminClient.createUser(projectId, instanceId, mode, user);
 
   // Grant the role to the user.
@@ -351,10 +378,21 @@ async function handleIncompatibleSchemaError(args: {
         ${commandsToExecuteBySuperUser.join("\n")}`);
     }
 
-    // TODO (tammam-g): at some point we would want to only run this after notifying the admin but
-    // until we confirm stability it's ok to run it on every migration by admin user.
-    if (userIsCSQLAdmin) {
-      await setupIAMUsers(instanceId, databaseId, options);
+    const schemaInfo = await getSchemaMetadata(instanceId, databaseId, DEFAULT_SCHEMA, options);
+    if (schemaInfo.setupStatus !== SchemaSetupStatus.GreenField) {
+      const newSetupStatus = await setupSQLPermissions(
+        instanceId,
+        databaseId,
+        schemaInfo,
+        options,
+        /* silent=*/ true,
+      );
+
+      if (newSetupStatus !== SchemaSetupStatus.GreenField) {
+        throw new FirebaseError(
+          `Can't migrate brownfield databases. Consider fully migrating your database to FDC using 'firebase dataconnect:sql:setup'`,
+        );
+      }
     }
 
     // Test if iam user has access to the roles required for this migration

src/gcp/cloudsql/cloudsqladmin.ts

@@ -2,6 +2,10 @@ import { Client, ClientResponse } from "../../apiv2";
 import { cloudSQLAdminOrigin } from "../../api";
 import * as operationPoller from "../../operation-poller";
 import { Instance, Database, User, UserType, DatabaseFlag } from "./types";
+import { needProjectId } from "../../projectUtils";
+import { Options } from "../../options";
+import { logger } from "../../logger";
+import { testIamPermissions } from "../iam";
 import { FirebaseError } from "../../error";
 const API_VERSION = "v1";
 
@@ -16,6 +20,24 @@ interface Operation {
   name: string;
 }
 
+export async function iamUserIsCSQLAdmin(options: Options): Promise<boolean> {
+  const projectId = needProjectId(options);
+  const requiredPermissions = [
+    "cloudsql.instances.connect",
+    "cloudsql.instances.get",
+    "cloudsql.users.create",
+    "cloudsql.users.update",
+  ];
+
+  try {
+    const iamResult = await testIamPermissions(projectId, requiredPermissions);
+    return iamResult.passed;
+  } catch (err: any) {
+    logger.debug(`[iam] error while checking permissions, command may fail: ${err}`);
+    return false;
+  }
+}
+
 export async function listInstances(projectId: string): Promise<Instance[]> {
   const res = await client.get<{ items: Instance[] }>(`projects/${projectId}/instances`);
   return res.body.items ?? [];

src/gcp/cloudsql/connect.ts

@@ -11,7 +11,6 @@ import { logger } from "../../logger";
 import { FirebaseError } from "../../error";
 import { Options } from "../../options";
 import { FBToolsAuthClient } from "./fbToolsAuthClient";
-import { setupSQLPermissions, firebaseowner, firebasewriter } from "./permissions";
 
 export async function execute(
   sqlStatements: string[],
@@ -23,7 +22,7 @@ export async function execute(
     password?: string;
     silent?: boolean;
   },
-) {
+): Promise<pg.QueryResult[]> {
   const logFn = opts.silent ? logger.debug : logger.info;
   const instance = await cloudSqlAdminClient.getInstance(opts.projectId, opts.instanceId);
   const user = await cloudSqlAdminClient.getUser(opts.projectId, opts.instanceId, opts.username);
@@ -92,11 +91,12 @@ export async function execute(
   }
 
   const conn = await pool.connect();
+  const results: pg.QueryResult[] = [];
   logFn(`Logged in as ${opts.username}`);
   for (const s of sqlStatements) {
     logFn(`Executing: '${s}'`);
     try {
-      await conn.query(s);
+      results.push(await conn.query(s));
     } catch (err) {
       throw new FirebaseError(`Error executing ${err}`);
     }
@@ -105,6 +105,7 @@ export async function execute(
   conn.release();
   await pool.end();
   connector.close();
+  return results;
 }
 
 export async function executeSqlCmdsAsIamUser(
@@ -113,7 +114,7 @@ export async function executeSqlCmdsAsIamUser(
   databaseId: string,
   cmds: string[],
   silent = false,
-): Promise<void> {
+): Promise<pg.QueryResult[]> {
   const projectId = needProjectId(options);
   const { user: iamUser } = await getIAMUser(options);
 
@@ -135,7 +136,7 @@ export async function executeSqlCmdsAsSuperUser(
   databaseId: string,
   cmds: string[],
   silent = false,
-) {
+): Promise<pg.QueryResult[]> {
   const projectId = needProjectId(options);
   // 1. Create a temporary builtin user
   const superuser = "firebasesuperuser";
@@ -148,7 +149,7 @@ export async function executeSqlCmdsAsSuperUser(
     temporaryPassword,
   );
 
-  return await execute([`SET ROLE = cloudsqlsuperuser`, ...cmds], {
+  return await execute([`SET ROLE = '${superuser}'`, ...cmds], {
     projectId,
     instanceId,
     databaseId,
@@ -177,8 +178,6 @@ export async function getIAMUser(options: Options): Promise<{ user: string; mode
 // Steps:
 // 1. Create an IAM user for the current identity
 // 2. Create an IAM user for FDC P4SA
-// 3. Run setupSQLPermissions to setup the SQL database roles and permissions.
-// 4. Connect to the DB as the temporary user and run the necessary grants
 export async function setupIAMUsers(
   instanceId: string,
   databaseId: string,
@@ -200,18 +199,6 @@ export async function setupIAMUsers(
   );
   await cloudSqlAdminClient.createUser(projectId, instanceId, fdcP4SAmode, fdcP4SAUser);
 
-  // 3. Setup FDC required SQL roles and permissions.
-  await setupSQLPermissions(instanceId, databaseId, options, true);
-
-  // 4. Apply necessary grants.
-  const grants = [
-    // Grant firebaseowner role to the current IAM user.
-    `GRANT "${firebaseowner(databaseId)}" TO "${user}"`,
-    // Grant firebaswriter to the FDC P4SA user
-    `GRANT "${firebasewriter(databaseId)}" TO "${fdcP4SAUser}"`,
-  ];
-
-  await executeSqlCmdsAsSuperUser(options, instanceId, databaseId, grants, /** silent=*/ true);
   return user;
 }