Merge pull request #291 from Fateslayer/multiple-column-delimiters

add multiple column delimiter support

Author: wdavidw

lib/index.js

@@ -95,21 +95,27 @@ class Parser extends Transform {
       }
     }
     // Normalize option `delimiter`
-    if(options.delimiter === undefined || options.delimiter === null || options.delimiter === false){
-      options.delimiter = Buffer.from(',')
-    }else if(typeof options.delimiter === 'string' && options.delimiter.length !== 0){
-      options.delimiter = Buffer.from(options.delimiter)
-    }else if(
-      (Buffer.isBuffer(options.delimiter) && options.delimiter.length === 0) ||
-      (typeof options.delimiter === 'string' && options.delimiter.length === 0) ||
-      (!Buffer.isBuffer(options.delimiter) && typeof options.delimiter !== 'string')
-    ){
+    const delimiter_json = JSON.stringify(options.delimiter)
+    if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]
+    if(options.delimiter.length === 0){
       throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [
         'Invalid option delimiter:',
-        'delimiter must be a non empty string or buffer,',
-        `got ${JSON.stringify(options.delimiter)}`
+        'delimiter must be a non empty string or buffer or array of string|buffer,',
+        `got ${delimiter_json}`
       ])
     }
+    options.delimiter = options.delimiter.map(function(del){
+      if(del === undefined || del === null || del === false){
+        return Buffer.from(',')
+      }else if((typeof del === 'string' || Buffer.isBuffer(del)) && del.length !== 0){
+        return Buffer.from(del)
+      }
+      throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [
+        'Invalid option delimiter:',
+        'delimiter must be a non empty string or buffer or array of string|buffer,',
+        `got ${delimiter_json}`
+      ])
+    })
     // Normalize option `escape`
     if(options.escape === undefined || options.escape === null){
       options.escape = Buffer.from('"')
@@ -923,12 +929,16 @@ class Parser extends Transform {
   }
   __isDelimiter(chr, buf, pos){
     const {delimiter} = this.options
-    const delLength = delimiter.length
-    if(delimiter[0] !== chr) return 0
-    for(let i = 1; i < delLength; i++){
-      if(delimiter[i] !== buf[pos+i]) return 0
+    loop1: for(let i = 0; i < delimiter.length; i++){
+      const del = delimiter[i]
+      if(del[0] === chr){
+        for(let j = 1; j < del.length; j++){
+          if(del[j] !== buf[pos+j]) continue loop1
+        }
+        return del.length
+      }
     }
-    return delimiter.length
+    return 0
   }
   __isRecordDelimiter(chr, buf, pos){
     const {record_delimiter} = this.options

test/option.delimiter.coffee

@@ -9,17 +9,32 @@ describe 'Option `delimiter`', ->
     (->
       parse '', delimiter: '', (->)
     ).should.throw
-      message: 'Invalid option delimiter: delimiter must be a non empty string or buffer, got ""'
+      message: 'Invalid option delimiter: delimiter must be a non empty string or buffer or array of string|buffer, got ""'
       code: 'CSV_INVALID_OPTION_DELIMITER'
     (->
       parse '', delimiter: Buffer.from(''), (->)
     ).should.throw
-      message: 'Invalid option delimiter: delimiter must be a non empty string or buffer, got {"type":"Buffer","data":[]}'
+      message: 'Invalid option delimiter: delimiter must be a non empty string or buffer or array of string|buffer, got {"type":"Buffer","data":[]}'
       code: 'CSV_INVALID_OPTION_DELIMITER'
     (->
       parse '', delimiter: true, (->)
     ).should.throw
-      message: 'Invalid option delimiter: delimiter must be a non empty string or buffer, got true'
+      message: 'Invalid option delimiter: delimiter must be a non empty string or buffer or array of string|buffer, got true'
+      code: 'CSV_INVALID_OPTION_DELIMITER'
+    (->
+      parse '', delimiter: [], (->)
+    ).should.throw
+      message: 'Invalid option delimiter: delimiter must be a non empty string or buffer or array of string|buffer, got []'
+      code: 'CSV_INVALID_OPTION_DELIMITER'
+    (->
+      parse '', delimiter: [''], (->)
+    ).should.throw
+      message: 'Invalid option delimiter: delimiter must be a non empty string or buffer or array of string|buffer, got [""]'
+      code: 'CSV_INVALID_OPTION_DELIMITER'
+    (->
+      parse '', delimiter: [',',''], (->)
+    ).should.throw
+      message: 'Invalid option delimiter: delimiter must be a non empty string or buffer or array of string|buffer, got [",",""]'
       code: 'CSV_INVALID_OPTION_DELIMITER'
 
   it 'using default comma', (next) ->
@@ -69,3 +84,39 @@ describe 'Option `delimiter`', ->
         [ '','1974','8.8392926E7','','']
       ]
       next()
+  
+  it 'using array of a single delimiter', (next) ->
+    parse """
+    abc,,123,
+    ,def,,
+    """, delimiter: [','], (err, data) ->
+      return next err if err
+      data.should.eql [
+        [ 'abc','','123','']
+        [ '','def','','']
+      ]
+      next()
+  
+  it 'using array of a single delimiter of multiple characters', (next) ->
+    parse """
+    !#
+    !#
+    """, delimiter: ['!#'], (err, data) ->
+      return next err if err
+      data.should.eql [
+        [ '', '']
+        [ '', '']
+      ]
+      next()
+  
+  it 'using array of a multiple delimiters of variable length', (next) ->
+    parse """
+    abc,;;123;;
+    ;;def;;,
+    """, delimiter: [',', ';;'], (err, data) ->
+      return next err if err
+      data.should.eql [
+        [ 'abc','','123','']
+        [ '','def','','']
+      ]
+      next()