Streaming parse?

[Mike Bostock]

Like the new shapefile.

[Mike Bostock]

This is fine for Node: https://github.com/mafintosh/csv-parser

Would still be nice to have streaming CSV parsing in the browser.

[Christopher Pietsch]

you probably mean something else, but I wrote a little class that "streams" csv rows as they are loaded through the progress event:

import { request, csvParse, csvParseRows } from 'd3'
import EventEmitter from 'eventemitter3'

export default class CsvLoader extends EventEmitter {
  constructor (url) {
    super()
    this.loadedChars = 0
    this.header = undefined
    this.data = []
    this.xhr = undefined

    this.request = request(url)
      .mimeType('text')
      .on('progress', this.onProgress)
      .on('load', () => this.emit('loaded'))

    return this
  }

  get () {
    this.request.get()
    return this
  }

  onProgress = (p) => {
    const progress = p.total / p.loaded
    const responseText = p.target.responseText
    // console.log(responseText.length)
    const lineBreak = responseText.lastIndexOf('\n') + 1
    const char = responseText.slice(this.loadedChars, lineBreak)
    let parsed = csvParseRows(char)
    if (!this.header) {
      this.header = parsed.shift()
    }
    const obj = parsed.map(p =>
      this.header.reduce((a, b, i) => {
        a[b] = p[i]
        return a
      }, {})
    )
    this.loadedChars = lineBreak
    this.emit('row', obj)
    this.emit('progress', progress)
  }
}

[Philippe Rivière]

https://observablehq.com/@mbostock/streaming-csv