ironSource / aws-api-read-stream

Turn an AWS api call into a readable stream

aws-api-read-stream

Turn an aws api call into a readable stream.

Install

npm i aws-api-read-stream

example

Piping the result of s3.listObjectsV2()

Take care to use NextToken or ContinuationToken accordingly.

const aws = require('aws-sdk')
const APIStream = require('aws-api-read-stream')
const { promisify } = require('util')
const pipeline = promisify(require('stream').pipeline)

async function main() {
    const s3 = new aws.S3()

    const s = APIStream.from((nextToken) => {
        return s3.listObjectsV2({
            Bucket: 'your-bucket-here',
            ContinuationToken: nextToken
        }).promise()
    })

    // convert the object stream to strings using async generator
    // (node 13.* and above)
    const transform = async function*(source) {
        for await (const chunk of source) {
            yield JSON.stringify(chunk)
        }
    }

    await pipeline(s, transform, process.stdout)
}

main()

Keep reading until the stream finishes. This will buffer the results in an internal array, be wary though, because this might crash the process if it runs out of memory

const aws = require('aws-sdk')
const APIStream = require('aws-api-read-stream')
const { promisify } = require('util')
const pipeline = promisify(require('stream').pipeline)

async function main() {
    const s3 = new aws.S3()

    const s = APIStream.from((nextToken) => {
        return s3.listObjectsV2({
            Bucket: 'your-bucket-here',
            ContinuationToken: nextToken
        }).promise()
    })

    const results = await s.readAll()
}

main()

Provide Readable stream options during initialization. objectMode will always be set to true

const s = APIStream.from((nextToken) => {
    return s3.listObjectsV2({
        Bucket: 'your-bucket-here',
        ContinuationToken: nextToken
    }).promise()
}, { options: { ... your options here } })

Start with an existing nextToken

const s = APIStream.from((nextToken) => {
    return s3.listObjectsV2({
        Bucket: 'your-bucket-here',
        ContinuationToken: nextToken
    }).promise()
}, { nextToken: '123123' })