Merge pull request #34 from netlify/zip_it_and_ship_it

Zip it and ship it

Author: bcomnes

package.json

@@ -35,7 +35,6 @@
   ],
   "dependencies": {
     "@netlify/open-api": "^0.5.0",
-    "archiver": "^3.0.0",
     "backoff": "^2.5.0",
     "clean-deep": "^3.0.2",
     "debug": "^4.1.1",
@@ -59,7 +58,7 @@
     "through2-filter": "^3.0.0",
     "through2-map": "^3.0.0",
     "util.promisify": "^1.0.0",
-    "util.promisify-all": "^1.0.2"
+    "@netlify/zip-it-and-ship-it": "^0.2.0"
   },
   "devDependencies": {
     "@babel/core": "^7.1.2",

src/deploy/hash-fns.js

@@ -1,10 +1,10 @@
-const promisifyAll = require('util.promisify-all')
 const promisify = require('util.promisify')
 const pump = promisify(require('pump'))
-const fs = promisifyAll(require('fs'))
 const fromArray = require('from2-array')
+const zipIt = require('@netlify/zip-it-and-ship-it')
+const path = require('path')
 
-const { hasherCtor, manifestCollectorCtor, fnStatCtor, fnFilterCtor } = require('./hasher-segments')
+const { hasherCtor, manifestCollectorCtor } = require('./hasher-segments')
 
 module.exports = hashFns
 async function hashFns(dir, opts) {
@@ -14,32 +14,38 @@ async function hashFns(dir, opts) {
       assetType: 'function',
       hashAlgorithm: 'sha256',
       // tmpDir,
-      statusCb: () => { }
+      statusCb: () => {}
     },
     opts
   )
   // early out if the functions dir is omitted
   if (!dir) return { functions: {}, shaMap: {} }
-  if (!opts.filter) throw new Error('Missing required filter function')
   if (!opts.tmpDir) throw new Error('Missing tmpDir directory for zipping files')
 
-  const fileList = await fs.readdir(dir).then(files => files.filter(opts.filter))
-  const fileStream = fromArray.obj(fileList)
+  const functionZips = await zipIt.zipFunctions(dir, opts.tmpDir)
+
+  const fileObjs = functionZips.map(({ path: functionPath, runtime }) => ({
+    filepath: functionPath,
+    root: opts.tmpDir,
+    relname: path.relative(opts.tmpDir, functionPath),
+    basename: path.basename(functionPath),
+    extname: path.extname(functionPath),
+    type: 'file',
+    assetType: 'function',
+    normalizedPath: path.basename(functionPath, path.extname(functionPath)),
+    runtime
+  }))
+
+  const functionStream = fromArray.obj(fileObjs)
 
-  const fnStat = fnStatCtor({ root: dir, concurrentStat: opts.concurrentHash, tmpDir: opts.tmpDir })
-  const fnFilter = fnFilterCtor()
   const hasher = hasherCtor(opts)
 
   // Written to by manifestCollector
   const functions = {} // normalizedPath: hash (wanted by deploy API)
   const fnShaMap = {} //hash: [fileObj, fileObj, fileObj]
   const manifestCollector = manifestCollectorCtor(functions, fnShaMap, opts)
 
-  // TODO: Zip up functions, hash then send.
-  // This is totally wrong right now.
-  // See https://github.com/netlify/open-api/blob/master/go/porcelain/deploy.go#L544
-
-  await pump(fileStream, fnStat, fnFilter, hasher, manifestCollector)
+  await pump(functionStream, hasher, manifestCollector)
 
   return { functions, fnShaMap }
 }

src/deploy/hasher-segments.js

@@ -1,20 +1,17 @@
 const objFilterCtor = require('through2-filter').objCtor
 const objWriter = require('flush-write-stream').obj
-const { isExe, normalizePath } = require('./util')
+const { normalizePath } = require('./util')
 const transform = require('parallel-transform')
 const hasha = require('hasha')
-const path = require('path')
-const fs = require('fs')
 const map = require('through2-map').obj
-const pump = require('pump')
-const archiver = require('archiver')
 
 // a parallel transform stream segment ctor that hashes fileObj's created by folder-walker
 exports.hasherCtor = ({ concurrentHash, hashAlgorithm = 'sha1' }) => {
+  const hashaOpts = { algorithm: hashAlgorithm }
   if (!concurrentHash) throw new Error('Missing required opts')
   return transform(concurrentHash, { objectMode: true }, (fileObj, cb) => {
     hasha
-      .fromFile(fileObj.filepath, { algorithm: hashAlgorithm })
+      .fromFile(fileObj.filepath, hashaOpts)
       // insert hash and asset type to file obj
       .then(hash => cb(null, Object.assign({}, fileObj, { hash })))
       .catch(err => cb(err))
@@ -56,67 +53,3 @@ exports.manifestCollectorCtor = (filesObj, shaMap, { statusCb, assetType }) => {
 exports.fileFilterCtor = objFilterCtor(fileObj => {
   return fileObj.type === 'file'
 })
-
-exports.fnFilterCtor = objFilterCtor(fileObj => {
-  // filter additional files out of our fn pipeline
-  return fileObj.type === 'file' && !!fileObj.runtime
-})
-
-// Zip a file into a temporary directory
-function zipFunction(item, tmpDir, cb) {
-  const zipPath = path.join(tmpDir, item.normalizedPath + '.zip')
-  const output = fs.createWriteStream(zipPath)
-  const archive = archiver('zip')
-
-  archive.file(item.filepath, { name: item.basename })
-  archive.finalize()
-
-  pump(archive, output, err => {
-    if (err) return cb(err)
-
-    item.filepath = zipPath
-    cb(null, item)
-  })
-}
-
-// parallel stream ctor similar to folder-walker but specialized for netlify functions
-// Stream in names of files that may be functions, and this will stat the file and return a fileObj
-exports.fnStatCtor = ({ root, concurrentStat, tmpDir }) => {
-  if (!concurrentStat || !root || !tmpDir) throw new Error('Missing required opts')
-  return transform(concurrentStat, { objectMode: true }, (name, cb) => {
-    const filepath = path.join(root, name)
-    fs.stat(filepath, (err, stat) => {
-      if (err) return cb(err)
-
-      const item = {
-        root,
-        filepath,
-        stat,
-        relname: path.relative(root, filepath),
-        basename: path.basename(name),
-        extname: path.extname(name),
-        type: stat.isFile() ? 'file' : stat.isDirectory() ? 'directory' : null,
-        assetType: 'function',
-        normalizedPath: path.basename(name, path.extname(name))
-      }
-
-      if (['.zip'].some(ext => item.extname === ext)) {
-        item.runtime = 'js'
-        return cb(null, item)
-      }
-
-      if (['.js'].some(ext => item.extname === ext)) {
-        item.runtime = 'js'
-
-        return zipFunction(item, tmpDir, cb)
-      }
-
-      if (isExe(item.stat)) {
-        item.runtime = 'go'
-        return zipFunction(item, tmpDir, cb)
-      }
-
-      return cb(null, item)
-    })
-  })
-}

src/deploy/util.js

@@ -115,18 +115,4 @@ function getUploadList(required, shaMap) {
   return flatten(required.map(sha => shaMap[sha]))
 }
 
-// given a Stat object, return if its executable or not
-// https://github.com/jokeyrhyme/is-executable.js without the io
-exports.isExe = stat => {
-  const { mode, gid, uid } = stat
-  if (process.platform === 'win32') {
-    return true
-  }
-
-  const isGroup = gid ? process.getgid && gid === process.getgid() : true
-  const isUser = uid ? process.getuid && uid === process.getuid() : true
-
-  return Boolean(mode & 0o0001 || (mode & 0o0010 && isGroup) || (mode & 0o0100 && isUser))
-}
-
 exports.retry = async (fn, errHandler, opts) => {}