mozilla / DeepSpeech

DeepSpeech is an open source embedded (offline, on-device) speech-to-text engine which can run in real time on devices ranging from a Raspberry Pi 4 to high power GPU servers.

Geek Repo:Geek Repo

Github PK Tool:Github PK Tool

Speedup / make more reliable NodeJS + ElectronJS builds

lissyx opened this issue · comments

NodeJS / ElectronJS builds can be slower because of:

  • node_modules dependencies to fetch
  • nodejs / electronjs headers to download
  • both those also make it more sensible to infra issues

It would be nice to leverage caching capabilities of GitHub Actions to help with that

  • caching the nodejs/electronjs headers: --devdir parameter in
    NODE_DEVDIR=--devdir=tmp/headers/nodejs/${node} \
  • caching the node_modules from build time:
    build-nodejs-macOS:
    name: "Build NodeJS and ElectronJS for macOS"
    runs-on: macos-10.15
    needs: [ build-lib_macOS, swig_macOS ]
    strategy:
    matrix:
    build-flavor: ["tf", "tflite"]
    steps:
    - uses: actions/checkout@v2
    with:
    fetch-depth: 1
    - uses: actions/download-artifact@v2
    with:
    name: "native_client.${{ matrix.build-flavor }}.tar.xz"
    path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
    - run: |
    cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
    tar xf native_client.tar.xz
    ls -hal
    cd ${{ github.workspace }}/
    - uses: actions/download-artifact@v2
    with:
    name: "swig_macOS"
    path: ${{ github.workspace }}/native_client/ds-swig/
    - run: |
    ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
    ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
    chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
    - uses: actions/setup-node@v2
    with:
    node-version: 12
    - uses: ./.github/actions/node-build
    with:
    nodejs_versions: "10.0.0 11.0.0 12.7.0 13.0.0 14.0.0 15.0.0"
    electronjs_versions: "5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1 9.1.0 9.2.0 10.0.0 10.1.0 11.0.0 12.0.0"
    - uses: actions/upload-artifact@v2
    with:
    name: "nodewrapper-${{ matrix.build-flavor }}.tar.gz"
    path: ${{ github.workspace }}/native_client/javascript/wrapper.tar.gz
    - uses: actions/upload-artifact@v2
    with:
    name: "deepspeech-${{ matrix.build-flavor }}.tgz"
    path: ${{ github.workspace }}/native_client/javascript/deepspeech-*.tgz
  • caching the node_modules from test runs:
    test-nodejs-macOS:
    name: "Test NodeJS bindings on macOS"
    runs-on: macos-10.15
    needs: [ build-nodejs-macOS, train-test-model ]
    if: ${{ github.event_name == 'pull_request' }}
    strategy:
    matrix:
    # https://nodejs.org/en/about/releases/
    nodejs-version: [10, 12, 14, 15]
    build-flavor: ["tf", "tflite"]
    models: ["test"]
    bitrate: ["16k"]
    env:
    TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/
    DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
    DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
    DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
    EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
    steps:
    - uses: actions/checkout@v2
    with:
    fetch-depth: 1
    - uses: actions/setup-node@v2
    with:
    node-version: ${{ matrix.nodejs-version }}
    - uses: actions/download-artifact@v2
    with:
    name: "deepspeech-${{ matrix.build-flavor }}.tgz"
    path: ${{ env.TASKCLUSTER_TMP_DIR }}
    - uses: actions/download-artifact@v2
    with:
    name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
    path: ${{ env.TASKCLUSTER_TMP_DIR }}
    if: matrix.models == 'test'
    - run: |
    ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
    if: matrix.models == 'test'
    - run: |
    ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
    npm install ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.tgz
    - uses: ./.github/actions/run-tests
    with:
    runtime: "node"
    build-flavor: ${{ matrix.build-flavor }}
    bitrate: ${{ matrix.bitrate }}
    model-kind: ${{ matrix.models }}
    test-electronjs-macOS:
    name: "Test ElectronJS bindings on macOS"
    runs-on: macos-10.15
    needs: [ build-nodejs-macOS, train-test-model ]
    if: ${{ github.event_name == 'pull_request' }}
    strategy:
    matrix:
    electronjs-version: [5.0.13, 6.1.7, 7.1.8, 8.0.1, 9.2.0, 10.1.0, 11.0.0, 12.0.0]
    build-flavor: ["tf", "tflite"]
    models: ["test"]
    bitrate: ["16k"]
    env:
    TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/
    DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
    DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
    DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
    EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
    steps:
    - uses: actions/checkout@v2
    with:
    fetch-depth: 1
    - uses: actions/setup-node@v2
    with:
    node-version: 12
    - uses: actions/download-artifact@v2
    with:
    name: "deepspeech-${{ matrix.build-flavor }}.tgz"
    path: ${{ env.TASKCLUSTER_TMP_DIR }}
    - uses: actions/download-artifact@v2
    with:
    name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
    path: ${{ env.TASKCLUSTER_TMP_DIR }}
    if: matrix.models == 'test'
    - run: |
    ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
    if: matrix.models == 'test'
    - run: |
    ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
    npm install ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.tgz
    - run: |
    npm install electron@${{ matrix.electronjs-version }}
    - uses: ./.github/actions/run-tests
    with:
    runtime: "electronjs"
    build-flavor: ${{ matrix.build-flavor }}
    bitrate: ${{ matrix.bitrate }}
    model-kind: ${{ matrix.models }}
    timeout-minutes: 5