Add approve functions and use typescript.

tinayuangao · tinayuangao · commit 1f67986af80c · 2017-03-24T18:21:12.000-07:00
diff --git a/functions/data.ts b/functions/data.ts
@@ -0,0 +1,22 @@
+import * as firebaseAdmin from 'firebase-admin';
+import {verifySecureTokenAndExecute} from './jwt_util';
+
+/**
+ * Handle data written to temporary folder. Validate the JWT and move the data out of
+ * temporary folder if the token is valid.
+ * Move the data to 'screenshot/reports/$prNumber/$path
+ */
+export function verifyJWTAndUpdateData(event: any, path: string) {
+  // Only edit data when it is first created. Exit when the data is deleted.
+  if (event.data.previous.exists() || !event.data.exists()) {
+    return;
+  }
+
+  let prNumber = event.params.prNumber;
+  let data = event.data.val();
+
+  return verifySecureTokenAndExecute(event).then(() => {
+    return firebaseAdmin.database().ref().child('screenshot/reports')
+      .child(prNumber).child(path).set(data);
+  });
+};
diff --git a/functions/data_image.ts b/functions/data_image.ts
@@ -0,0 +1,36 @@
+import * as firebaseFunctions from 'firebase-functions';
+import {writeFileSync} from 'fs';
+import {verifySecureTokenAndExecute} from './jwt_util';
+
+const gcs = require('@google-cloud/storage')();
+
+/** The storage bucket to store the images. The bucket is also used by Firebase Storage. */
+const bucket = gcs.bucket(firebaseFunctions.config().firebase.storageBucket);
+
+/**
+ * Convert data to images. Image data posted to database will be saved as png files
+ * and upload to screenshot/$prNumber/dataType/$filename
+ */
+export function convertTestImageDataToFiles(event: any) {
+  // Only edit data when it is first created. Exit when the data is deleted.
+  if (event.data.previous.exists() || !event.data.exists()) {
+    return;
+  }
+
+  let dataType = event.params.dataType;
+  let prNumber = event.params.prNumber;
+  let data = event.data.val();
+  let saveFilename = `${event.params.filename}.screenshot.png`;
+
+  if (dataType != 'diff' && dataType != 'test') {
+    return;
+  }
+
+  return verifySecureTokenAndExecute(event).then(() => {
+    let tempPath = `/tmp/${dataType}-${saveFilename}`;
+    let filePath = `screenshots/${prNumber}/${dataType}/${saveFilename}`;
+    let binaryData = new Buffer(data, 'base64').toString('binary');
+    writeFileSync(tempPath, binaryData, 'binary');
+    return bucket.upload(tempPath, {destination: filePath});
+  });
+};
diff --git a/functions/github.ts b/functions/github.ts
@@ -0,0 +1,32 @@
+import * as firebaseAdmin from 'firebase-admin';
+import * as firebaseFunctions from 'firebase-functions';
+import {setGithubStatus} from './util/github';
+
+/** Github status update token */
+const token = firebaseFunctions.config().secret.github;
+
+/** The repo slug. This is used to validate the JWT is sent from correct repo. */
+const repoSlug = firebaseFunctions.config().repo.slug;
+
+/** Domain to view the screenshots */
+const authDomain = firebaseFunctions.config().firebase.authDomain;
+
+/** The same of this screenshot testing tool */
+const toolName = firebaseFunctions.config().tool.name;
+
+export function updateGithubStatus(event: firebaseFunctions.Event<any>) {
+  if (!event.data.exists() || typeof event.data.val() != 'boolean') {
+    return;
+  }
+  let result = event.data.val() == true;
+  let prNumber = event.params.prNumber;
+  return event.data.ref.parent.child('sha').once('value').then((sha: firebaseAdmin.database.DataSnapshot) => {
+    return setGithubStatus(sha.val(),
+      result,
+      toolName,
+      `${toolName} ${result ? 'passed' : 'failed'}`,
+      `http://${authDomain}/${prNumber}`,
+      repoSlug,
+      token);
+  });
+}
diff --git a/functions/image_data.ts b/functions/image_data.ts
@@ -0,0 +1,37 @@
+import * as firebaseAdmin from 'firebase-admin';
+import * as path from 'path';
+import {readFileSync} from 'fs';
+
+const gcs = require('@google-cloud/storage')();
+
+const FIREBASE_DATA_GOLDENS = 'screenshot/goldens';
+
+/**
+ * Read golden files under /goldens/ and store the image data to
+ * database /screenshot/goldens/$filename
+ */
+export function convertGoldenImagesToData(name: string, resourceState: string, fileBucket: any) {
+  // The name should always look like "goldens/xxx.png"
+  let parsedPath = path.parse(name);
+  // Get the file name.
+  if (parsedPath.root != '' || parsedPath.dir != 'goldens' || parsedPath.ext != '.png') {
+    return;
+  }
+
+  let filenameKey = path.basename(parsedPath.name, '.screenshot');
+  let databaseRef = firebaseAdmin.database().ref(FIREBASE_DATA_GOLDENS).child(filenameKey);
+
+  // When a gold image is deleted, also delete the corresponding record in the firebase database.
+  if (resourceState === 'not_exists') {
+    return databaseRef.set(null);
+  }
+
+  let tempFilePath = `/tmp/${parsedPath.base}`;
+  let bucket = gcs.bucket(fileBucket);
+  // Download file from bucket.
+  return bucket.file(name).download({destination: tempFilePath})
+    .then(() => {
+      let data = readFileSync(tempFilePath);
+      return databaseRef.set(data);
+    }).catch((error: any) => console.error(`${filenameKey} ${error}`));
+};
diff --git a/functions/index.ts b/functions/index.ts
@@ -0,0 +1,120 @@
+'use strict';
+
+import * as firebaseFunctions from 'firebase-functions';
+import * as firebaseAdmin from 'firebase-admin';
+
+import {verifyJWTAndUpdateData} from './data';
+import {convertGoldenImagesToData} from './image_data';
+import {convertTestImageDataToFiles} from './data_image';
+import {copyTestImagesToGoldens} from './test_goldens';
+import {updateGithubStatus} from './github';
+
+/**
+ * Usage: Firebase functions only accept javascript file index.js
+ *   tsc
+ *   firebase deploy --only functions
+ *
+ *
+ * Data and images handling for Screenshot test.
+ *
+ * All users can post data to temporary folder. These Functions will check the data with JsonWebToken and
+ * move the valid data out of temporary folder.
+ *
+ * For valid data posted to database /$temp/screenshot/reports/$prNumber/$secureToken, move it to
+ * /screenshot/reports/$prNumber.
+ * These are data for screenshot results (success or failure), GitHub PR/commit and TravisCI job information
+ *
+ * For valid image results written to database /$temp/screenshot/images/$prNumber/$secureToken/, save the image
+ * data to image files and upload to google cloud storage under location /screenshots/$prNumber
+ * These are screenshot test result images, and difference images generated from screenshot comparison.
+ *
+ * For golden images uploaded to /goldens, read the data from images files and write the data to Firebase database
+ * under location /screenshot/goldens
+ * Screenshot tests can only read restricted database data with no credentials, and they cannot access
+ * Google Cloud Storage. Therefore we copy the image data to database to make it available to screenshot tests.
+ *
+ * The JWT is stored in the data path, so every write to database needs a valid JWT to be copied to database/storage.
+ * All invalid data will be removed.
+ * The JWT has 3 parts: header, payload and signature. These three parts are joint by '/' in path.
+ */
+
+// Initailize the admin app
+firebaseAdmin.initializeApp(firebaseFunctions.config().firebase);
+
+/** The valid data types database accepts */
+const dataTypes = ['result', 'sha', 'travis'];
+
+/** The Json Web Token format. The token is stored in data path. */
+const jwtFormat = '{jwtHeader}/{jwtPayload}/{jwtSignature}';
+
+/** The temporary folder name for screenshot data that needs to be validated via JWT.  */
+const tempFolder = '/untrustedInbox';
+
+
+/** Untrusted report data for a PR */
+const reportPath = `${tempFolder}/screenshot/reports/{prNumber}/${jwtFormat}/`;
+/** Untrusted image data for a PR */
+const imagePath =  `${tempFolder}/screenshot/images/{prNumber}/${jwtFormat}/`;
+/** Trusted report data for a PR */
+const trustedReportPath = `screenshot/reports/{prNumber}`;
+
+/**
+ * Copy valid data from /$temp/screenshot/reports/$prNumber/$secureToken/
+ *   to /screenshot/reports/$prNumber
+ * Data copied: filenames(image results names), commit(github PR info),
+ *     sha (github PR info), result (true or false for all the tests), travis job number
+ */
+const testDataPath = `${reportPath}/{dataType}`;
+exports.testData = firebaseFunctions.database.ref(testDataPath)
+    .onWrite((event: any) => {
+  const dataType = event.params.dataType;
+  if (dataTypes.includes(dataType)) {
+    return verifyJWTAndUpdateData(event, dataType);
+  }
+});
+
+/**
+ * Copy valid data from /$temp/screenshot/reports/$prNumber/$secureToken/
+ *   to /screenshot/reports/$prNumber
+ * Data copied: test result for each file/test with ${filename}. The value should be true or false.
+ */
+const testResultsPath = `${reportPath}/results/{filename}`;
+exports.testResults = firebaseFunctions.database.ref(testResultsPath)
+    .onWrite((event: any) => {
+  return verifyJWTAndUpdateData(event, `results/${event.params.filename}`);
+});
+
+/**
+ * Copy valid data from database /$temp/screenshot/images/$prNumber/$secureToken/
+ *   to storage /screenshots/$prNumber
+ * Data copied: test result images. Convert from data to image files in storage.
+ */
+const imageDataToFilePath = `${imagePath}/{dataType}/{filename}`;
+exports.imageDataToFile = firebaseFunctions.database.ref(imageDataToFilePath)
+  .onWrite(convertTestImageDataToFiles);
+
+/**
+ * Copy valid goldens from storage /goldens/ to database /screenshot/goldens/
+ * so we can read the goldens without credentials.
+ */
+exports.goldenImageToData = firebaseFunctions.storage.bucket(
+    firebaseFunctions.config().firebase.storageBucket).object().onChange((event: any) => {
+  return convertGoldenImagesToData(event.data.name, event.data.resourceState, event.data.bucket);
+});
+
+/**
+ * Copy test result images for PR to Goldens.
+ * Copy images from /screenshot/$prNumber/test/ to /goldens/
+ */
+const approveImagesPath = `${trustedReportPath}/approved`;
+exports.approveImages = firebaseFunctions.database.ref(approveImagesPath).onWrite((event: any) => {
+  return copyTestImagesToGoldens(event.params.prNumber);
+});
+
+/**
+ * Update github status. When the result is true, update github commit status to `success`,
+ * otherwise update github status to `failure`.
+ * The Github Status Token is set in config.secret.github
+ */
+const githubStatusPath = `${trustedReportPath}/result`;
+exports.githubStatus = firebaseFunctions.database.ref(githubStatusPath).onWrite(updateGithubStatus);
diff --git a/functions/jwt_util.ts b/functions/jwt_util.ts
@@ -0,0 +1,37 @@
+import * as firebaseFunctions from 'firebase-functions';
+import {verifySecureToken} from './util/jwt';
+
+/** The repo slug. This is used to validate the JWT is sent from correct repo. */
+const repoSlug = firebaseFunctions.config().repo.slug;
+
+/** The JWT secret. This is used to validate JWT. */
+const secret = firebaseFunctions.config().secret.key;
+
+/**
+ * Extract the Json Web Token from event params.
+ * In screenshot gulp task the path we use is {jwtHeader}/{jwtPayload}/{jwtSignature}.
+ * Replace '/' with '.' to get the token.
+ */
+function getSecureToken(event: firebaseFunctions.Event<any>) {
+  return `${event.params.jwtHeader}.${event.params.jwtPayload}.${event.params.jwtSignature}`;
+};
+
+/**
+ * Verify event params have correct JsonWebToken, and execute callback when the JWT is verified.
+ * Delete the data if there's an error or the callback is done
+ */
+export function verifySecureTokenAndExecute(event: firebaseFunctions.Event<any>) {
+  return new Promise((resolve, reject) => {
+    const prNumber = event.params.prNumber;
+    const secureToken = getSecureToken(event);
+
+    return verifySecureToken(secureToken, prNumber, secret, repoSlug).then(() => {
+      resolve();
+      event.data.ref.parent.set(null);
+    }).catch((error: any) => {
+      console.error(`Invalid secure token ${secureToken} ${error}`);
+      event.data.ref.parent.set(null);
+      reject();
+    });
+  });
+};
diff --git a/functions/package.json b/functions/package.json
@@ -5,6 +5,8 @@
     "@google-cloud/storage": "^0.8.0",
     "firebase-admin": "^4.1.3",
     "firebase-functions": "^0.5.2",
-    "jsonwebtoken": "^7.3.0"
+    "jsonwebtoken": "^7.3.0",
+    "request": "^2.81.0",
+    "typescript": "^2.2.1"
   }
 }
diff --git a/functions/test_goldens.ts b/functions/test_goldens.ts
@@ -0,0 +1,36 @@
+import * as firebaseFunctions from 'firebase-functions';
+import * as firebaseAdmin from 'firebase-admin';
+import * as path from 'path';
+
+const gcs = require('@google-cloud/storage')();
+
+/** The storage bucket to store the images. The bucket is also used by Firebase Storage. */
+const bucket = gcs.bucket(firebaseFunctions.config().firebase.storageBucket);
+
+/**
+ * Copy files from /screenshot/$prNumber/test/ to goldens/
+ * Only copy the files that test result is failure. Passed test images should be the same as
+ * goldens.
+ */
+export function copyTestImagesToGoldens(prNumber: string) {
+  return firebaseAdmin.database().ref(`screenshot/reports/${prNumber}/results`).once('value')
+    .then((snapshot: firebaseAdmin.database.DataSnapshot) => {
+      let keys: string[] = [];
+      let counter = 0;
+      snapshot.forEach((childSnapshot: firebaseAdmin.database.DataSnapshot) => {
+        if (childSnapshot.val() == false) {
+          keys.push(childSnapshot.key);
+        }
+        counter ++;
+        if (counter == snapshot.numChildren()) return true;
+      });
+      return keys;
+    }).then((keys: string[]) => {
+      return bucket.getFiles({prefix: `screenshots/${prNumber}/test`}).then(function (data: any) {
+        return Promise.all(data[0]
+          .filter((file: any) => keys.includes(path.basename(file.name, '.screenshot.png')))
+          .map((file: any) => file.copy(`goldens/${path.basename(file.name)}`)));
+      });
+    })
+
+};
diff --git a/functions/tsconfig.json b/functions/tsconfig.json
@@ -0,0 +1,31 @@
+{
+  "compilerOptions": {
+    "declaration": false,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "lib": ["es6", "es2016", "dom"],
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "noEmitOnError": true,
+    "noImplicitAny": true,
+    "outDir": "./",
+    "sourceMap": true,
+    "target": "es5",
+    "stripInternal": false,
+    "baseUrl": "",
+    "typeRoots": [
+      "../node_modules/@types/!(node)"
+    ]
+  },
+  "files": [
+    "data.ts",
+    "data_image.ts",
+    "github.ts",
+    "image_data.ts",
+    "index.ts",
+    "jwt_util.ts",
+    "test_goldens.ts",
+    "util/github.ts",
+    "util/jwt.ts"
+  ]
+}
diff --git a/functions/util/github.ts b/functions/util/github.ts
diff --git a/functions/util/jwt.ts b/functions/util/jwt.ts
diff --git a/tools/gulp/tasks/screenshots.ts b/tools/gulp/tasks/screenshots.ts
diff --git a/tools/gulp/util/firebase.ts b/tools/gulp/util/firebase.ts

Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,8 @@`
`5`	`5`	`"@google-cloud/storage": "^0.8.0",`
`6`	`6`	`"firebase-admin": "^4.1.3",`
`7`	`7`	`"firebase-functions": "^0.5.2",`
`8`		`- "jsonwebtoken": "^7.3.0"`
	`8`	`+ "jsonwebtoken": "^7.3.0",`
	`9`	`+ "request": "^2.81.0",`
	`10`	`+ "typescript": "^2.2.1"`
`9`	`11`	`}`
`10`	`12`	`}`