Speedup / make more reliable NodeJS + ElectronJS builds
lissyx opened this issue · comments
lissyx commented
NodeJS / ElectronJS builds can be slower because of:
node_modules
dependencies to fetch- nodejs / electronjs headers to download
- both those also make it more sensible to infra issues
It would be nice to leverage caching capabilities of GitHub Actions to help with that
- caching the nodejs/electronjs headers:
--devdir
parameter inNODE_DEVDIR=--devdir=tmp/headers/nodejs/${node} \ - caching the
node_modules
from build time:DeepSpeech/.github/workflows/macOS-amd64.yml
Lines 301 to 343 in 0ce835a
build-nodejs-macOS: name: "Build NodeJS and ElectronJS for macOS" runs-on: macos-10.15 needs: [ build-lib_macOS, swig_macOS ] strategy: matrix: build-flavor: ["tf", "tflite"] steps: - uses: actions/checkout@v2 with: fetch-depth: 1 - uses: actions/download-artifact@v2 with: name: "native_client.${{ matrix.build-flavor }}.tar.xz" path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/ - run: | cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/ tar xf native_client.tar.xz ls -hal cd ${{ github.workspace }}/ - uses: actions/download-artifact@v2 with: name: "swig_macOS" path: ${{ github.workspace }}/native_client/ds-swig/ - run: | ls -hal ${{ github.workspace }}/native_client/ds-swig/bin ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig - uses: actions/setup-node@v2 with: node-version: 12 - uses: ./.github/actions/node-build with: nodejs_versions: "10.0.0 11.0.0 12.7.0 13.0.0 14.0.0 15.0.0" electronjs_versions: "5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1 9.1.0 9.2.0 10.0.0 10.1.0 11.0.0 12.0.0" - uses: actions/upload-artifact@v2 with: name: "nodewrapper-${{ matrix.build-flavor }}.tar.gz" path: ${{ github.workspace }}/native_client/javascript/wrapper.tar.gz - uses: actions/upload-artifact@v2 with: name: "deepspeech-${{ matrix.build-flavor }}.tgz" path: ${{ github.workspace }}/native_client/javascript/deepspeech-*.tgz - caching the
node_modules
from test runs:DeepSpeech/.github/workflows/macOS-amd64.yml
Lines 430 to 523 in 0ce835a
test-nodejs-macOS: name: "Test NodeJS bindings on macOS" runs-on: macos-10.15 needs: [ build-nodejs-macOS, train-test-model ] if: ${{ github.event_name == 'pull_request' }} strategy: matrix: # https://nodejs.org/en/about/releases/ nodejs-version: [10, 12, 14, 15] build-flavor: ["tf", "tflite"] models: ["test"] bitrate: ["16k"] env: TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/ DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988" steps: - uses: actions/checkout@v2 with: fetch-depth: 1 - uses: actions/setup-node@v2 with: node-version: ${{ matrix.nodejs-version }} - uses: actions/download-artifact@v2 with: name: "deepspeech-${{ matrix.build-flavor }}.tgz" path: ${{ env.TASKCLUSTER_TMP_DIR }} - uses: actions/download-artifact@v2 with: name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" path: ${{ env.TASKCLUSTER_TMP_DIR }} if: matrix.models == 'test' - run: | ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ if: matrix.models == 'test' - run: | ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ npm install ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.tgz - uses: ./.github/actions/run-tests with: runtime: "node" build-flavor: ${{ matrix.build-flavor }} bitrate: ${{ matrix.bitrate }} model-kind: ${{ matrix.models }} test-electronjs-macOS: name: "Test ElectronJS bindings on macOS" runs-on: macos-10.15 needs: [ build-nodejs-macOS, train-test-model ] if: ${{ github.event_name == 'pull_request' }} strategy: matrix: electronjs-version: [5.0.13, 6.1.7, 7.1.8, 8.0.1, 9.2.0, 10.1.0, 11.0.0, 12.0.0] build-flavor: ["tf", "tflite"] models: ["test"] bitrate: ["16k"] env: TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/ DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988" steps: - uses: actions/checkout@v2 with: fetch-depth: 1 - uses: actions/setup-node@v2 with: node-version: 12 - uses: actions/download-artifact@v2 with: name: "deepspeech-${{ matrix.build-flavor }}.tgz" path: ${{ env.TASKCLUSTER_TMP_DIR }} - uses: actions/download-artifact@v2 with: name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" path: ${{ env.TASKCLUSTER_TMP_DIR }} if: matrix.models == 'test' - run: | ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ if: matrix.models == 'test' - run: | ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ npm install ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.tgz - run: | npm install electron@${{ matrix.electronjs-version }} - uses: ./.github/actions/run-tests with: runtime: "electronjs" build-flavor: ${{ matrix.build-flavor }} bitrate: ${{ matrix.bitrate }} model-kind: ${{ matrix.models }} timeout-minutes: 5