Add utilities for cacheable layers in docker builds

Dockerfile

@@ -1,7 +1,7 @@
-FROM yolean/node@sha256:ebdf2658467fb8408c242bdde9ec6714c838ff3612041f46e57b4717acdc0a84
+FROM yolean/node@sha256:f033123ae2292d60769e5b8eff94c4b7b9d299648d0d23917319c0743029c5ef
 
-ENV docker_version=17.06.2~ce-0~debian
-ENV compose_version=1.16.1
+ENV docker_version=17.09.1~ce-0~debian
+ENV compose_version=1.21.0 compose_sha256=af639f5e9ca229442c8738135b5015450d56e2c1ae07c0aaa93b7da9fe09c2b0
 
 RUN apt-get update \
   && apt-get install -y apt-transport-https curl ca-certificates gnupg2 \
@@ -16,12 +16,15 @@ RUN apt-get update \
 RUN update-rc.d -f docker remove
 
 RUN curl -L https://github.com/docker/compose/releases/download/$compose_version/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose \
+  && sha256sum /usr/local/bin/docker-compose \
+  && echo "${compose_sha256} /usr/local/bin/docker-compose" | sha256sum -c - \
   && chmod +x /usr/local/bin/docker-compose
 
 VOLUME /source
 WORKDIR /source
 
 COPY package.json build-contract parsetargets /usr/src/app/
-RUN cd /usr/src/app/ && npm install && ln -s /usr/src/app/build-contract /usr/local/bin/build-contract
+COPY nodejs /usr/src/app/nodejs
+RUN cd /usr/src/app/ && npm install && npm link
 ENTRYPOINT ["build-contract"]
 CMD ["push"]

README.md

@@ -6,15 +6,26 @@ Defines a successful build and test run for a microservice, from source to docke
 Invoke build-contract in current folder, use host's docker:
 ```
 docker build --tag yolean/build-contract .
-docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd)/:/source yolean/build-contract test
+docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd)/:/source  --rm --name mybuild yolean/build-contract test
 ```
 
+Or for monorepo:
+```
+docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd)/../:/source -w /source/$(basename $(pwd)) --rm --name mybuild yolean/build-contract test
+```
+
+Add `-t` for colors and Ctrl+C support.
+
+There are automated builds [solsson/build-contract](https://hub.docker.com/r/solsson/build-contract).
+
 ## Node.js monorepo support with `npm``
 
 Add scripts to `package.json` like so, and build contract will pick them up:
 
 ```
   "scripts": {
     "build-contract-predockerbuild": "./node_modules/.bin/build-contract-predockerbuild",
-    "build-contract-postdockerbuild": "./node_modules/.bin/build-contract-postdockerbuild",
+    "packagelock": "build-contract-packagelock",
 ```
+
+Paths depend on your npm install situation.

build-contract

@@ -6,7 +6,7 @@ fi
 trap "exit" INT
 
 ROOT=$PWD
-DIR=`dirname $(readlink $BASH_SOURCE || echo $BASH_SOURCE)`
+DIR=`dirname $(realpath $0)`
 if [[ "$1" == "push" ]]; then
   DO_PUSH=true
 else
@@ -90,10 +90,6 @@ done
 
 echo "  --- build-contract: Build Contract finished. ---  "
 
-MONOREPO_POST=$(cat package.json | grep '"build-contract-postdockerbuild"' | awk -F '"' '{ print $4 }')
-if [[ "$MONOREPO_POST" == "#" ]]; then $DIR/nodejs/build-contract-postdockerbuild
-elif [[ ! -z "$MONOREPO_POST" ]]; then npm run build-contract-postdockerbuild; fi
-
 # Push targets
 for compose_file in $(ls "$CONTRACTS_DIR" | grep .yml); do
   echo "  --- build-contract: $compose_file ---  "

nodejs/build-contract-packagelock

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+
+$SCRIPTPATH/build-contract-predockerbuild
+
+cd npm-monorepo/ci
+npm install --production --package-lock-only --ignore-scripts
+mv package-lock.json ../../
+cd ../../

nodejs/build-contract-predockerbuild

@@ -1,19 +1,131 @@
-#!/bin/bash
-# At next scope creep please rewrite this with nodejs, as it'll only be used with nodejs environments anyway
-if [ -z ${MONOREPO_DEPS+x} ]; then
-  MONOREPO_DEPS=$(grep '"file:../' package.json | awk -F '"' '{ print $4 }')
-fi
-if [ ! -z "$MONOREPO_DEPS" ]; then
-  mkdir -p npm-monorepo
-  cp package.json package.json.monorepo-backup
-  for FILEDEP in $MONOREPO_DEPS; do
-    DEP=$(echo $FILEDEP | awk -F':' '{ print $2 }')
-    pushd $DEP
-    TARBALL=$(npm pack | tail -n 1)
-    popd
-    cp -v $DEP/$TARBALL npm-monorepo/
-    sed -i.bak "s|$DEP|./npm-monorepo/$TARBALL|" package.json
-  done
-  echo "  --- monorepo compatibility ---  "
-  git diff package.json
-fi
+#!/usr/bin/env node
+const path = require('path');
+const fs = require('fs');
+
+const npmLib = new Promise((resolve,reject) => {
+  // ls -la $(which npm), or we could do something like https://stackoverflow.com/a/25106648/113009
+  const guesses = ['./node_modules/npm', '/usr/local/lib/node_modules/npm', '/usr/lib/node_modules/npm'];
+  const check = (path) => path ? fs.stat(path, (err, stats) => {
+    if (err) return check(guesses.shift());
+    let installed = /^(?:\.\/)?node_modules\/(.*)/.exec(path);
+    if (installed) path = installed[1];
+    require(installed ? installed[1] : path).load((err, loaded) => {
+      if (err) throw err;
+      resolve(loaded);
+    });
+  }) : reject(new Error("Failed to guess the npm lib\'s install path. Try `npm (link|install) npm`."));
+  check(guesses.shift());
+});
+
+let dir = path.resolve('.');
+let mdir = path.join(dir, 'npm-monorepo');
+let cidir = path.join(mdir, 'ci');
+let cimdir = path.join(cidir, 'npm-monorepo');
+
+/**
+ * Gets a minimal package.json with only the stuff that should
+ * trigger an invalidation of docker build cache for the npm ci layer.
+ */
+function getCiPackage(packageJson) {
+  return {
+    name: packageJson.name,
+    version: packageJson.version,
+    dependencies: packageJson.dependencies
+  }
+}
+
+/**
+ * Produces a package tarball.
+ * @param modulePath In case we find a way to avoid depending on process.cwd
+ */
+function npmPackage(modulePath, cb) {
+  if (process.cwd() !== modulePath) throw new Error('npm expected to run in ' + modulePath + ', not ' + process.cwd());
+  npmLib.then(npm => {
+    npm.commands.pack([], (err, result) => {
+      if (err) return cb(err);
+      console.debug('# pack result', modulePath, JSON.stringify(result));
+      const name = result[0];
+      fs.stat(name, (err, stats) => {
+        if (err) console.error('# npm pack failed to produce the result file', npm, process.cwd());
+        cb(err, err ? undefined : name);
+      });
+    });
+  });
+}
+
+function stringifyPackageJson(package) {
+  return JSON.stringify(package, null, '  ');
+}
+
+let package = require(path.join(dir,'package.json'));
+let monorepoDeps = Object.keys(package.dependencies).filter(
+  dep => /^file:\.\.\//.test(package.dependencies[dep]));
+
+if (!monorepoDeps.length) {
+  console.log('# Zero monorepo dependencies found');
+  process.exit(0);
+}
+
+let mk = fs.mkdir;
+mk(dir, mk.bind(null, mdir, mk.bind(null, cidir, mk.bind(null, cimdir, err => {
+  if (err) {
+    if (err.code !== 'EEXIST') throw err;
+    console.log('# Monorepo dir structure already present', cimdir);
+  }
+
+  const completed = () => {
+    process.chdir(dir); // restore after npm
+    fs.unlink(path.join(cimdir, 'package.json'), err => err && console.error('Failed to clean up after sourceless tgz pack', err));
+    fs.writeFile(path.join(mdir, 'package.json'), stringifyPackageJson(package),
+      err => { if (err) throw err; });
+    const ciPackage = getCiPackage(package);
+    fs.writeFile(path.join(cidir, 'package.json'), stringifyPackageJson(ciPackage),
+      err => { if (err) throw err; });
+    fs.unlink(path.join(cimdir, '.npmignore'), err => err && console.error(err));
+  };
+
+  // Needed for the depCiPackage part in the callback stack below
+  fs.writeFile(path.join(cimdir, '.npmignore'), "*.tgz\n", err => err && console.error(err));
+
+  const next = dep => {
+    if (!dep) return completed();
+
+    let uri = package.dependencies[dep];
+    let urimatch = /^file:(\.\.\/.*)/.exec(uri);
+    if (!urimatch) return console.error('# Unrecognized monorepo dependency URI', uri);
+    let depdir = path.normalize(path.join(dir, urimatch[1]));
+
+    process.chdir(dir); process.chdir(depdir); // for npm
+    let depPackage = require(path.resolve('./package.json'));
+    npmPackage(depdir, (err, tgzname) => {
+      if (err) throw err;
+      console.log('# Packed', tgzname, 'in', process.cwd());
+      fs.rename(tgzname, path.join(mdir, tgzname), err => {
+        if (err) throw err;
+        console.log('# Created monorepo tarball', mdir, tgzname);
+        package.dependencies[dep] = `file:npm-monorepo/${tgzname}`;
+
+        let depCiPackage = getCiPackage(depPackage);
+        fs.writeFile(path.join(cimdir, 'package.json'), stringifyPackageJson(depCiPackage), err => {
+          process.chdir(cimdir); // for npm
+          npmPackage(cimdir, (err, tgzname) => {
+            console.log('# Created monorepo sourceless tarball for npm ci', cimdir, tgzname);
+            next(monorepoDeps.shift());
+          });
+        });
+      });
+    });
+  };
+  next(monorepoDeps.shift());
+
+}))));
+
+process.on('uncaughtException', err => {
+  console.error('Uncaught exception', err);
+  process.exit(1);
+});
+
+process.on('unhandledRejection', (err, p) => {
+  console.error('Unhandled rejection', err);
+  process.exit(1);
+});

package.json

@@ -1,12 +1,12 @@
 {
   "name": "build-contract",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Defines a successful build and test run for a microservice, from source to docker push",
   "main": "build-contract",
   "bin": {
     "build-contract": "./build-contract",
     "build-contract-predockerbuild": "./nodejs/build-contract-predockerbuild",
-    "build-contract-postdockerbuild": "./nodejs/build-contract-postdockerbuild"
+    "packagelock": "./nodejs/build-contract-packagelock"
   },
   "repository": {
     "type": "git",
@@ -20,5 +20,8 @@
   "homepage": "https://github.com/Yolean/build-contract#readme",
   "dependencies": {
     "yamljs": "0.2.8"
+  },
+  "peerDependencies": {
+    "npm": "5.8.0"
   }
 }