fix: updates ipld-dag-pb dep to version without .cid properties

Follows on from ipld/js-ipld-dag-pb#99 and
updates this module to not rely on DAGNodes having knowledge of
their CIDs.

Bonus: also removes use of js-ipfs from this module breaking another
circular dependency from the project.

Author: achingbrain

package.json

@@ -40,7 +40,6 @@
     "aegir": "^17.0.0",
     "chai": "^4.2.0",
     "dirty-chai": "^2.0.1",
-    "ipfs": "~0.32.3",
     "ipfs-block-service": "~0.15.1",
     "ipfs-repo": "~0.25.0",
     "ipld": "~0.19.1",
@@ -58,7 +57,7 @@
     "cids": "~0.5.5",
     "deep-extend": "~0.6.0",
     "ipfs-unixfs": "~0.1.16",
-    "ipld-dag-pb": "~0.14.11",
+    "ipld-dag-pb": "ipld/js-ipld-dag-pb#remove-cid-property-from-dagnodes",
     "left-pad": "^1.3.0",
     "multihashing-async": "~0.5.1",
     "pull-batch": "^1.0.0",

src/builder/builder.js

@@ -7,8 +7,7 @@ const through = require('pull-through')
 const parallel = require('async/parallel')
 const waterfall = require('async/waterfall')
 const dagPB = require('ipld-dag-pb')
-const CID = require('cids')
-const multihash = require('multihashing-async')
+const persist = require('../utils/persist')
 
 const reduce = require('./reduce')
 
@@ -27,7 +26,7 @@ const defaultOptions = {
 
 module.exports = function builder (createChunker, ipld, createReducer, _options) {
   const options = extend({}, defaultOptions, _options)
-  options.cidVersion = options.cidVersion || options.cidVersion
+  options.cidVersion = options.cidVersion || defaultOptions.cidVersion
   options.hashAlg = options.hashAlg || defaultOptions.hashAlg
 
   if (options.hashAlg !== 'sha2-256') {
@@ -71,33 +70,17 @@ module.exports = function builder (createChunker, ipld, createReducer, _options)
     const d = new UnixFS('directory')
 
     waterfall([
-      (cb) => DAGNode.create(d.marshal(), [], options.hashAlg, cb),
-      (node, cb) => {
-        if (options.onlyHash) {
-          return cb(null, node)
-        }
-
-        const cid = new CID(options.cidVersion, 'dag-pb', node.multihash)
-
-        node = new DAGNode(
-          node.data,
-          node.links,
-          node.serialized,
-          cid
-        )
-
-        ipld.put(node, {
-          cid
-        }, (err) => cb(err, node))
-      }
-    ], (err, node) => {
+      (cb) => DAGNode.create(d.marshal(), [], cb),
+      (node, cb) => persist(node, ipld, options, cb)
+    ], (err, result) => {
       if (err) {
         return callback(err)
       }
+
       callback(null, {
         path: item.path,
-        multihash: node.multihash,
-        size: node.size
+        multihash: result.cid.buffer,
+        size: result.node.size
       })
     })
   }
@@ -134,55 +117,42 @@ module.exports = function builder (createChunker, ipld, createReducer, _options)
       }),
       pull.asyncMap((buffer, callback) => {
         if (options.rawLeaves) {
-          return multihash(buffer, options.hashAlg, (error, hash) => {
-            if (error) {
-              return callback(error)
-            }
-
-            return callback(null, {
-              multihash: hash,
-              size: buffer.length,
-              leafSize: buffer.length,
-              cid: new CID(1, 'raw', hash),
-              data: buffer
-            })
+          return callback(null, {
+            size: buffer.length,
+            leafSize: buffer.length,
+            data: buffer
           })
         }
 
         const file = new UnixFS(options.leafType, buffer)
 
-        DAGNode.create(file.marshal(), [], options.hashAlg, (err, node) => {
+        DAGNode.create(file.marshal(), [], (err, node) => {
           if (err) {
             return callback(err)
           }
 
           callback(null, {
-            multihash: node.multihash,
             size: node.size,
             leafSize: file.fileSize(),
-            cid: new CID(options.cidVersion, 'dag-pb', node.multihash),
             data: node
           })
         })
       }),
       pull.asyncMap((leaf, callback) => {
-        if (options.onlyHash) {
-          return callback(null, leaf)
-        }
+        persist(leaf.data, ipld, options, (error, results) => {
+          if (error) {
+            return callback(error)
+          }
 
-        ipld.put(leaf.data, {
-          cid: leaf.cid
-        }, (error) => callback(error, leaf))
-      }),
-      pull.map((leaf) => {
-        return {
-          path: file.path,
-          multihash: leaf.cid.buffer,
-          size: leaf.size,
-          leafSize: leaf.leafSize,
-          name: '',
-          cid: leaf.cid
-        }
+          callback(null, {
+            size: leaf.size,
+            leafSize: leaf.leafSize,
+            data: results.node,
+            multihash: results.cid.buffer,
+            path: leaf.path,
+            name: ''
+          })
+        })
       }),
       through( // mark as single node if only one single node
         function onData (data) {

src/builder/reduce.js

@@ -3,7 +3,7 @@
 const waterfall = require('async/waterfall')
 const dagPB = require('ipld-dag-pb')
 const UnixFS = require('ipfs-unixfs')
-const CID = require('cids')
+const persist = require('../utils/persist')
 
 const DAGLink = dagPB.DAGLink
 const DAGNode = dagPB.DAGNode
@@ -14,10 +14,10 @@ module.exports = function reduce (file, ipld, options) {
       const leaf = leaves[0]
 
       return callback(null, {
-        path: file.path,
-        multihash: leaf.multihash,
         size: leaf.size,
         leafSize: leaf.leafSize,
+        multihash: leaf.multihash,
+        path: file.path,
         name: leaf.name
       })
     }
@@ -28,44 +28,23 @@ module.exports = function reduce (file, ipld, options) {
     const links = leaves.map((leaf) => {
       f.addBlockSize(leaf.leafSize)
 
-      let cid = leaf.cid
-
-      if (!cid) {
-        // we are an intermediate node
-        cid = new CID(options.cidVersion, 'dag-pb', leaf.multihash)
-      }
-
-      return new DAGLink(leaf.name, leaf.size, cid.buffer)
+      return new DAGLink(leaf.name, leaf.size, leaf.multihash)
     })
 
     waterfall([
-      (cb) => DAGNode.create(f.marshal(), links, options.hashAlg, cb),
-      (node, cb) => {
-        const cid = new CID(options.cidVersion, 'dag-pb', node.multihash)
-
-        if (options.onlyHash) {
-          return cb(null, {
-            node, cid
-          })
-        }
-
-        ipld.put(node, {
-          cid
-        }, (error) => cb(error, {
-          node, cid
-        }))
-      }
+      (cb) => DAGNode.create(f.marshal(), links, cb),
+      (node, cb) => persist(node, ipld, options, cb)
     ], (error, result) => {
       if (error) {
         return callback(error)
       }
 
       callback(null, {
-        name: '',
-        path: file.path,
-        multihash: result.cid.buffer,
         size: result.node.size,
-        leafSize: f.fileSize()
+        leafSize: f.fileSize(),
+        multihash: result.cid.buffer,
+        path: file.path,
+        name: ''
       })
     })
   }

src/exporter/dir-flat.js

@@ -13,7 +13,7 @@ function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, paren
     name: name,
     depth: depth,
     path: path,
-    hash: cid,
+    multihash: cid.buffer,
     size: node.size,
     type: 'dir'
   }
@@ -26,7 +26,7 @@ function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, paren
         size: link.size,
         name: link.name,
         path: path + '/' + link.name,
-        multihash: link.multihash,
+        multihash: link.cid.buffer,
         linkName: link.name,
         pathRest: pathRest.slice(1),
         type: 'dir'

src/exporter/dir-hamt-sharded.js

@@ -13,7 +13,7 @@ function shardedDirExporter (cid, node, name, path, pathRest, resolve, size, dag
       name: name,
       depth: depth,
       path: path,
-      hash: cid,
+      multihash: cid.buffer,
       size: node.size,
       type: 'dir'
     }
@@ -36,7 +36,7 @@ function shardedDirExporter (cid, node, name, path, pathRest, resolve, size, dag
             depth: depth + 1,
             name: p,
             path: pp,
-            multihash: link.multihash,
+            multihash: link.cid.buffer,
             pathRest: p ? pathRest.slice(1) : pathRest,
             parent: dir || parent
           }

src/exporter/file.js

@@ -2,7 +2,6 @@
 
 const traverse = require('pull-traverse')
 const UnixFS = require('ipfs-unixfs')
-const CID = require('cids')
 const pull = require('pull-stream')
 const paramap = require('pull-paramap')
 const extractDataFromBlock = require('./extract-data-from-block')
@@ -43,7 +42,7 @@ module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, d
       content: pull.once(Buffer.alloc(0)),
       name: name,
       path: path,
-      hash: cid,
+      multihash: cid.buffer,
       size: fileSize,
       type: 'file'
     })
@@ -64,7 +63,7 @@ module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, d
     content: content,
     name: name,
     path: path,
-    hash: cid,
+    multihash: cid.buffer,
     size: fileSize,
     type: 'file'
   }])
@@ -142,7 +141,7 @@ function getChildren (dag, offset, end) {
     return pull(
       pull.values(filteredLinks),
       paramap((child, cb) => {
-        dag.get(new CID(child.link.multihash), (error, result) => cb(error, {
+        dag.get(child.link.cid, (error, result) => cb(error, {
           start: child.start,
           end: child.end,
           node: result && result.value,

src/exporter/index.js

@@ -54,9 +54,11 @@ module.exports = (path, dag, options) => {
   const pathLengthToCut = join(
     [dPath.base].concat(dPath.rest.slice(0, dPath.rest.length - 1))).length
 
+  const cid = new CID(dPath.base)
+
   return pull(
     pull.values([{
-      multihash: new CID(dPath.base),
+      multihash: cid.buffer,
       name: dPath.base,
       path: dPath.base,
       pathRest: dPath.rest,
@@ -70,7 +72,7 @@ module.exports = (path, dag, options) => {
         name: node.name,
         path: finalPathFor(node),
         size: node.size,
-        hash: node.hash || node.multihash,
+        hash: node.multihash,
         content: node.content,
         type: node.type
       }

src/importer/dir-flat.js

@@ -2,12 +2,12 @@
 
 const asyncEachSeries = require('async/eachSeries')
 const waterfall = require('async/waterfall')
-const CID = require('cids')
 const dagPB = require('ipld-dag-pb')
 const UnixFS = require('ipfs-unixfs')
 const DAGLink = dagPB.DAGLink
 const DAGNode = dagPB.DAGNode
 const Dir = require('./dir')
+const persist = require('../utils/persist')
 
 class DirFlat extends Dir {
   constructor (props, _options) {
@@ -56,28 +56,17 @@ class DirFlat extends Dir {
       })
 
     const dir = new UnixFS('directory')
-    const options = this._options
 
     waterfall(
       [
-        (callback) => DAGNode.create(dir.marshal(), links, options.hashAlg, callback),
-        (node, callback) => {
-          if (options.onlyHash) return callback(null, node)
-
-          let cid = new CID(node.multihash)
-
-          if (options.cidVersion === 1) {
-            cid = cid.toV1()
-          }
-
-          ipld.put(node, { cid }, (err) => callback(err, node))
-        },
-        (node, callback) => {
-          this.multihash = node.multihash
+        (callback) => DAGNode.create(dir.marshal(), links, callback),
+        (node, callback) => persist(node, ipld, this._options, callback),
+        ({cid, node}, callback) => {
+          this.multihash = cid.buffer
           this.size = node.size
           const pushable = {
             path: path,
-            multihash: node.multihash,
+            multihash: cid.buffer,
             size: node.size
           }
           source.push(pushable)