diff --git a/package.json b/package.json index faa2ce83..901f7f37 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "async-retry": "^1.2.3", "chalk": "^2.4.2", "cli-progress": "^3.8.2", + "dockerode": "^3.2.1", "dotenv": "^8.2.0", "ethers": "^5.0.3", "figlet": "^1.2.3", @@ -66,6 +67,7 @@ "@types/async-retry": "^1.4.2", "@types/chai": "^4.2.11", "@types/cli-progress": "^3.7.0", + "@types/dockerode": "^2.5.34", "@types/figlet": "^1.2.0", "@types/inquirer": "^6.5.0", "@types/js-yaml": "^3.12.5", diff --git a/src/tasks/buildWithBuildx.ts b/src/tasks/buildWithBuildx.ts index 1b133fed..9f9c8c3b 100644 --- a/src/tasks/buildWithBuildx.ts +++ b/src/tasks/buildWithBuildx.ts @@ -3,7 +3,7 @@ import semver from "semver"; import { shell } from "../utils/shell"; import { Architecture, PackageImage, PackageImageLocal } from "../types"; import { saveAndCompressImagesCached } from "./saveAndCompressImages"; -import { getDockerVersion } from "../utils/getDockerVersion"; +import { getDocker } from "../utils/docker"; const minimumDockerVersion = "19.03.0"; const buildxInstanceName = "dappnode-multiarch-builder"; @@ -28,6 +28,8 @@ export function buildWithBuildx({ buildTimeout: number; skipSave?: boolean; }): ListrTask[] { + const docker = getDocker(); + const localImages = images.filter( (image): image is PackageImageLocal => image.type === "local" ); @@ -41,10 +43,16 @@ export function buildWithBuildx({ process.env.DOCKER_CLI_EXPERIMENTAL = "enabled"; // Make sure `docker version` is >= 19.03 - const dockerVersion = await getDockerVersion(); - if (dockerVersion && semver.lt(dockerVersion, minimumDockerVersion)) + const dockerInfo = await docker.version().catch(e => { + throw Error(`docker is not installed: ${e.message}`); + }); + const dockerVersion = dockerInfo.Version; + if ( + semver.valid(dockerVersion) && + semver.lt(dockerVersion, minimumDockerVersion) + ) throw Error( - `docker version must be at least ${minimumDockerVersion} to use buildx` + `docker version must be at least ${minimumDockerVersion} to use buildx, current version ${dockerVersion}` ); switch (architecture) { diff --git a/src/tasks/saveAndCompressImages.ts b/src/tasks/saveAndCompressImages.ts index d3797da1..9da3302d 100644 --- a/src/tasks/saveAndCompressImages.ts +++ b/src/tasks/saveAndCompressImages.ts @@ -10,6 +10,7 @@ import { PackageImage, PackageImageExternal } from "../types"; +import { getDocker, getImageByTag } from "../utils/docker"; import { shell } from "../utils/shell"; /** @@ -30,6 +31,8 @@ export function saveAndCompressImagesCached({ buildTimeout: number; skipSave?: boolean; }): ListrTask[] { + const docker = getDocker(); + const imageTags = images.map(image => image.imageTag); const externalImages = images.filter( (image): image is PackageImageExternal => image.type === "external" @@ -46,12 +49,14 @@ export function saveAndCompressImagesCached({ { onData: data => (task.output = data) } ); task.output = `Tagging ${originalImageTag} > ${imageTag}`; - await shell(`docker tag ${originalImageTag} ${imageTag}`); + + const originalImage = await getImageByTag(docker, originalImageTag); + await originalImage.tag({ tag: imageTag }); // Validate the resulting image architecture - const imageDataRaw = await shell(`docker image inspect ${imageTag}`); - const imageData = JSON.parse(imageDataRaw); - const imageArch = `${imageData[0]["Os"]}/${imageData[0]["Architecture"]}`; + const newImage = await getImageByTag(docker, imageTag); + const newTagInfo = await newImage.inspect(); + const imageArch = `${newTagInfo.Os}/${newTagInfo.Architecture}`; if (imageArch !== architecture) throw Error( `pulled image ${originalImageTag} does not have the expected architecture '${architecture}', but ${imageArch}` diff --git a/src/utils/cache.ts b/src/utils/cache.ts index 6150e788..b796c0ae 100644 --- a/src/utils/cache.ts +++ b/src/utils/cache.ts @@ -1,7 +1,7 @@ import fs from "fs"; import os from "os"; import path from "path"; -import { getImageId, getImageIds } from "./getImageId"; +import { getDocker, getImageByTag } from "./docker"; // Local cache specs. Path = $cachePath type CacheMap = Map; @@ -73,9 +73,12 @@ export function writeCache(cache: CacheMap, cachePath?: string): void { export async function getCacheKey(imageTags: string[]): Promise { return ( await Promise.all( - imageTags - .sort() - .map(async imageTag => [imageTag, await getImageId(imageTag)].join("/")) + imageTags.sort().map(async imageTag => { + const image = await getImageByTag(getDocker(), imageTag); + const info = await image.inspect(); + // info.Id = "sha256:2dbd79fff9541ba91c6ce5867840ecee8d5e335bc2465dadb39a3419e7039f96" + return [imageTag, info.Id].join("/"); + }) ) ).join(";"); } @@ -88,7 +91,9 @@ export async function pruneCache(cachePath?: string): Promise { if (!cachePath) cachePath = getCachePath(); if (fs.existsSync(cachePath)) { const cache = loadCache(cachePath); - const imageIds = await getImageIds(); + const images = await getDocker().listImages(); + // imageIds = ["sha256:2dbd79fff9541ba91c6ce5867840ecee8d5e335bc2465dadb39a3419e7039f96"] + const imageIds = images.map(image => image.Id); const prunedCache = _pruneCache(cache, imageIds); writeCache(prunedCache, cachePath); } diff --git a/src/utils/docker.ts b/src/utils/docker.ts new file mode 100644 index 00000000..b4fa6724 --- /dev/null +++ b/src/utils/docker.ts @@ -0,0 +1,32 @@ +import Docker from "dockerode"; + +let docker: Docker | null = null; + +export function getDocker(): Docker { + if (!docker) docker = new Docker({ socketPath: "/var/run/docker.sock" }); + return docker; +} + +/** + * Helper to get images by reference / repo / tag + * @param _docker + * @param imageRepo + */ +export async function getImageByTag( + _docker: Docker, + imageTag: string +): Promise { + const images = await _docker.listImages(); + const matchingImages = images.filter( + image => image.RepoTags && image.RepoTags.some(tag => tag === imageTag) + ); + if (matchingImages.length > 1) { + const imageTags = matchingImages.map(image => image.Labels[0]).join(", "); + throw Error(`More than one image match for ${imageTag}: ${imageTags}`); + } else if (matchingImages.length < 1) { + throw Error(`No image found for ${imageTag}`); + } else { + console.log(matchingImages[0].Id); + return _docker.getImage(matchingImages[0].Id); + } +} diff --git a/src/utils/getDockerVersion.ts b/src/utils/getDockerVersion.ts deleted file mode 100644 index f6ff220b..00000000 --- a/src/utils/getDockerVersion.ts +++ /dev/null @@ -1,17 +0,0 @@ -import semver from "semver"; -import { shell } from "./shell"; - -/** - * Returns parsed and validated semver of docker -v - */ -export async function getDockerVersion(): Promise { - const dockerVersion = await shell(`docker -v`).catch(e => { - throw Error(`docker is not installed: ${e.message}`); - }); - // dockerVersion = "Docker version 19.03.13-beta2, build ff3fbc9d55" - const match = dockerVersion.match(/\d+\.\d+\.\d+/g); - if (!match) return null; - const regexVersion = match[0]; - if (!semver.valid(regexVersion)) return null; - return regexVersion; -} diff --git a/src/utils/getImageId.ts b/src/utils/getImageId.ts deleted file mode 100644 index 4d671397..00000000 --- a/src/utils/getImageId.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { shell } from "./shell"; - -/** - * Returns the formated ID of a docker image - * - * @param imageTag i.e. admin.dnp.dappnode.eth:0.1.14 - * @param shell dependency - * @return formated ID: sha256:0d31e5521ef6e92a0efb6110024da8a3517daac4b1e4bbbccaf063ce96641b1b - */ -export async function getImageId(imageTag: string): Promise { - const id = await shell(`docker inspect --format='{{json .Id}}' ${imageTag}`); - return id.replace(/['"]+/g, ""); -} - -/** - * Returns current image IDs in the docker cache - * @returns [ - * "6b74b6ba423e, - * "4a67065ab84a", - * "d112c75f4189", - * "cd25b55c48fd", - * "5480cec82e92" - * ] - */ -export async function getImageIds(): Promise { - const res = await shell(`docker images -q`); - return res.trim().split(/\r?\n/); -} diff --git a/yarn.lock b/yarn.lock index 5a020e12..fdc64758 100644 --- a/yarn.lock +++ b/yarn.lock @@ -523,6 +523,13 @@ resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0" integrity sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ== +"@types/dockerode@^2.5.34": + version "2.5.34" + resolved "https://registry.yarnpkg.com/@types/dockerode/-/dockerode-2.5.34.tgz#9adb884f7cc6c012a6eb4b2ad794cc5d01439959" + integrity sha512-LcbLGcvcBwBAvjH9UrUI+4qotY+A5WCer5r43DR5XHv2ZIEByNXFdPLo1XxR+v/BjkGjlggW8qUiXuVEhqfkpA== + dependencies: + "@types/node" "*" + "@types/eslint-visitor-keys@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz#1ee30d79544ca84d68d4b3cdb0af4f205663dd2d" @@ -831,7 +838,7 @@ argparse@^1.0.7: dependencies: sprintf-js "~1.0.2" -asn1@~0.2.3: +asn1@~0.2.0, asn1@~0.2.3: version "0.2.4" resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg== @@ -890,7 +897,7 @@ base64-js@^1.0.2: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g== -bcrypt-pbkdf@^1.0.0: +bcrypt-pbkdf@^1.0.0, bcrypt-pbkdf@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4= @@ -1274,6 +1281,24 @@ diff@^4.0.1: resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== +docker-modem@^2.1.0: + version "2.1.4" + resolved "https://registry.yarnpkg.com/docker-modem/-/docker-modem-2.1.4.tgz#54ec1079287624a8d30cc3c55ab0e1201e7c46c9" + integrity sha512-vDTzZjjO1sXMY7m0xKjGdFMMZL7vIUerkC3G4l6rnrpOET2M6AOufM8ajmQoOB+6RfSn6I/dlikCUq/Y91Q1sQ== + dependencies: + debug "^4.1.1" + readable-stream "^3.5.0" + split-ca "^1.0.1" + ssh2 "^0.8.7" + +dockerode@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/dockerode/-/dockerode-3.2.1.tgz#4a2222e3e1df536bf595e78e76d3cfbf6d4d93b9" + integrity sha512-XsSVB5Wu5HWMg1aelV5hFSqFJaKS5x1aiV/+sT7YOzOq1IRl49I/UwV8Pe4x6t0iF9kiGkWu5jwfvbkcFVupBw== + dependencies: + docker-modem "^2.1.0" + tar-fs "~2.0.1" + doctrine@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-3.0.0.tgz#addebead72a6574db783639dc87a121773973961" @@ -2695,7 +2720,7 @@ read-pkg@^4.0.1: parse-json "^4.0.0" pify "^3.0.0" -readable-stream@^3.1.1, readable-stream@^3.4.0: +readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== @@ -2928,11 +2953,32 @@ spdx-license-ids@^3.0.0: resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz#3694b5804567a458d3c8045842a6358632f62654" integrity sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q== +split-ca@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/split-ca/-/split-ca-1.0.1.tgz#6c83aff3692fa61256e0cd197e05e9de157691a6" + integrity sha1-bIOv82kvphJW4M0ZfgXp3hV2kaY= + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= +ssh2-streams@~0.4.10: + version "0.4.10" + resolved "https://registry.yarnpkg.com/ssh2-streams/-/ssh2-streams-0.4.10.tgz#48ef7e8a0e39d8f2921c30521d56dacb31d23a34" + integrity sha512-8pnlMjvnIZJvmTzUIIA5nT4jr2ZWNNVHwyXfMGdRJbug9TpI3kd99ffglgfSWqujVv/0gxwMsDn9j9RVst8yhQ== + dependencies: + asn1 "~0.2.0" + bcrypt-pbkdf "^1.0.2" + streamsearch "~0.1.2" + +ssh2@^0.8.7: + version "0.8.9" + resolved "https://registry.yarnpkg.com/ssh2/-/ssh2-0.8.9.tgz#54da3a6c4ba3daf0d8477a538a481326091815f3" + integrity sha512-GmoNPxWDMkVpMFa9LVVzQZHF6EW3WKmBwL+4/GeILf2hFmix5Isxm7Amamo8o7bHiU0tC+wXsGcUXOxp8ChPaw== + dependencies: + ssh2-streams "~0.4.10" + sshpk@^1.7.0: version "1.16.1" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877" @@ -2948,6 +2994,11 @@ sshpk@^1.7.0: safer-buffer "^2.0.2" tweetnacl "~0.14.0" +streamsearch@~0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a" + integrity sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo= + string-width@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" @@ -3100,6 +3151,16 @@ tar-fs@^2.0.0: pump "^3.0.0" tar-stream "^2.0.0" +tar-fs@~2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.1.tgz#e44086c1c60d31a4f0cf893b1c4e155dabfae9e2" + integrity sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA== + dependencies: + chownr "^1.1.1" + mkdirp-classic "^0.5.2" + pump "^3.0.0" + tar-stream "^2.0.0" + tar-stream@^2.0.0: version "2.1.2" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.1.2.tgz#6d5ef1a7e5783a95ff70b69b97455a5968dc1325"