very slow to package and deploy service

[Rodrigo Reis]

I have a service with 20 functions in it and when I do sls deploy it optimizes, packages all the functions and deploy.
This is taking more than 30 minutes now to run on my MacOS.. I'm not sure the problem is with this plugin or the serverless framework itself. Anyone experiencing this for large services with more than 15 functions?

My package.json:

{
  "name": "arena-node-api-service",
  "version": "1.0.0",
  "description": "",
  "main": "handler.js",
  "directories": {
    "lib": "lib"
  },
  "scripts": {
    "test": "jest --forceExit"
  },
  "jest": {
    "testEnvironment": "node"
  },
  "author": "",
  "license": "ISC",
  "devDependencies": {
    "jest": "^23.6.0",
    "rewire": "^4.0.1",
    "serverless-plugin-optimize": "^3.1.1-rc.1"
  },
  "dependencies": {
    "@iopipe/iopipe": "^1.11.1",
    "algoliasearch": "^3.30.0",
    "aws-sdk": "^2.351.0",
    "cheerio": "^1.0.0-rc.2",
    "elasticsearch": "^15.1.1",
    "firebase-admin": "^6.3.0",
    "jsonwebtoken": "^8.4.0",
    "lambda-api": "^0.8.1",
    "langmap": "0.0.16",
    "lodash": "^4.17.11",
    "mongoose": "5.2.5",
    "mongoose-findorcreate": "^3.0.0",
    "open-graph-scraper": "^3.5.1",
    "pretty-ms": "^4.0.0",
    "redis": "^2.8.0",
    "request": "^2.88.0",
    "request-promise": "^4.2.2",
    "stripe": "^6.17.0",
    "twitter": "^1.7.1",
    "ua-parser-js": "^0.7.19"
  }
}

My serverless.yml:

service: arena-node-api

# You can pin your service to only deploy with a specific Serverless version
# Check out our docs for more details
# frameworkVersion: "=X.X.X"

custom:
  stage: "${opt:stage, self:provider.stage}"
  optimize:
    debug: false
    external: ['mongoose']
  presence-frequency-dev: 10
  presence-frequency-prd: 2

provider:
  name: aws
  stage: dev
  runtime: nodejs8.10
  profile: default
  region: us-west-2
  memorySize: 256
  vpc:
    securityGroupIds:
      - XX-redacted
    subnetIds:
      - XX-redacted
      - XX-redacted
      - XX-redacted
  environment:
      env: ${self:custom.stage}
  apiKeys:
    - admin-apikey-${self:custom.stage}

# you can add packaging information here
package:
  individually: true

functions:
  billing:
    handler: handler_billing_api.handler
    timeout: 12
    events:
      - http:
          path: billing
          method: ANY
          cors:
            origin: '*'
            headers:
              - Content-Type
              - X-Amz-Date
              - Authorization
              - X-Api-Key
              - X-Amz-Security-Token
              - X-Amz-User-Agent
              - Preferred-Language
      - http:
          path: billing/{proxy+}
          method: ANY
          cors:
            origin: '*'
            headers:
              - Content-Type
              - X-Amz-Date
              - Authorization
              - X-Api-Key
              - X-Amz-Security-Token
              - X-Amz-User-Agent
              - Preferred-Language
  oembed:
    handler: handler_oembed.oembed
    timeout: 15
    events:
      - http:
          path: oembed/{proxy+}
          method: ANY
          cors:
            origin: '*'
            headers:
              - Content-Type
              - X-Amz-Date
              - Authorization
              - X-Api-Key
              - X-Amz-Security-Token
              - X-Amz-User-Agent
              - Preferred-Language
  metrics:
    handler: handler_metrics_api.handler
    events:
      - http:
          path: metrics/{proxy+}
          method: ANY
          cors:
            origin: '*'
            headers:
              - Content-Type
              - X-Amz-Date
              - Authorization
              - X-Api-Key
              - X-Amz-Security-Token
              - X-Amz-User-Agent
              - Preferred-Language
  integrations:
    handler: handler_integrations_api.handler
    events:
      - http:
          path: integrations/{proxy+}
          method: ANY
          cors:
            origin: '*'
            headers:
              - Content-Type
              - X-Amz-Date
              - Authorization
              - X-Api-Key
              - X-Amz-Security-Token
              - X-Amz-User-Agent
              - Preferred-Language
  contacts:
    handler: handler_contacts_api.handler
    timeout: 30
    events:
      - http:
          path: contacts/{proxy+}
          method: ANY
          cors:
            origin: '*'
            headers:
              - Content-Type
              - X-Amz-Date
              - Authorization
              - X-Api-Key
              - X-Amz-Security-Token
              - X-Amz-User-Agent
              - Preferred-Language
  content-manager:
    handler: handler_content_manager_api.handler
    timeout: 30
    memorySize: 2536
    optimize:
      external: ['@google-cloud/firestore', 'mongoose']
    events:
      - http:
          path: cm/{proxy+}
          method: ANY
          cors:
            origin: '*'
            headers:
              - Content-Type
              - X-Amz-Date
              - Authorization
              - X-Api-Key
              - X-Amz-Security-Token
              - X-Amz-User-Agent
              - Preferred-Language
  stream:
    handler: handler_stream_api.handler
    timeout: 30
    events:
      - http:
          path: stream/{proxy+}
          method: ANY
          cors:
            origin: '*'
            headers:
              - Content-Type
              - X-Amz-Date
              - Authorization
              - X-Api-Key
              - X-Amz-Security-Token
              - X-Amz-User-Agent
              - Preferred-Language
  admin-api:
    handler: handler_admin_api.handler
    timeout: 30
    optimize:
      external: ['@google-cloud/firestore']
    events:
      - http:
          path: admin/{proxy+}
          method: ANY
          private: true
  live-score-update:
    handler: handler_live_score_update.handler
  billing-job:
    handler: handler_billing_job.handler
    timeout: 30
    reservedConcurrency: 5
  update-intercom:
    handler: handler_update_intercom.handler
    memorySize: 128
    timeout: 300
    reservedConcurrency: 5
    vpc:
      securityGroupIds: []
      subnetIds: []
  track-collect:
    handler: handler_track_collect.handler
    timeout: 300
    events:
      - sqs: arn:aws:sqs:us-west-2:XXXX:events_${self:custom.stage}
  stream-autopost:
      handler: handler_stream_autopost.handler
      memorySize: 512
      optimize:
        external: ['@google-cloud/firestore', 'mongoose']
      timeout: 300
  presence-main-job:
      handler: handler_presence_job.main
      memorySize: 2536
      timeout: 300
      events:
        - schedule: rate(${self:custom.presence-frequency-${self:custom.stage}} minutes)
  presence-update-presence:
      handler: handler_presence_job.update_presence
      memorySize: 512
      timeout: 300
  presence-update-publisher:
      handler: handler_presence_job.update_publisher
      memorySize: 512
      timeout: 300
  presence-check-stale-sessions:
      handler: handler_presence_job.check_stale_sessions
      memorySize: 512
      timeout: 300
      events:
        - schedule: rate(4 hours)
  classify-asset:
      handler: handler_classify_asset.handler
      timeout: 300
      events:
        - s3:
            bucket: arena-cm-upload-${self:custom.stage}
            event: s3:ObjectCreated:*
  sync-fb-likes:
    handler: handler_sync_fb_likes.handler
    memorySize: 512
    timeout: 300
  admin-op:
    handler: handler_admin_api.handler_op
    timeout: 10
    optimize:
      external: ['@google-cloud/firestore']
plugins:
  - serverless-plugin-optimize

serverless cli version: 1.30.1

[Martin]

I have the same issue. It is taking more than 30 minutes to deploy.

[katie]

Me 3

I have 63 lambda functions, and it takes about a half hour to package.
Very annoying

[Vicary A.]

We ended up switching from serverless-plugin-typescript and serverless-plugin-optimize to serverless-webpack and the bundling time is reduced exponentially by size.

[clethrill]

@vicary could you share more details about your old and new setup? Our deploy takes up to 90 minutes! And we really need to find a way to speed up the process.

[Vicary A.]

@clethrill It takes some time to digest, took me 3 weeks for this and I made two repos in the process.

We have 300 lambdas so normal webpack takes more than 90 mins, we are now in 12 mins.

Summary

Basically you connect all the following pieces together,

1. serverless-webpack To prevent fd limit explosion, you may either use the official serializedCompile or my concurrency. I forked before this option is merged upstream but mine directly conflicts with the PR so I don't bother going through all the people involved. Use my version github:vicary/serverless-webpack#feat/concurrency in your package.json if you want multiple concurrent builds instead of only 1.
2. fork-ts-checker-webpack-plugin This is to prevent heap explosion. Use this in chain with transpileOnly: true in webpack, because the built-in way is to type check before build, that makes the build time ~2.5x longer. Spawning other processes for type-checking allows more efficient use in servers with many cores, reducing it down to ~1.2x when compared with no type checks.
3. fork-ts-checker-webpack-plugin-limiter This depends on the plugin above, limiting it from spawning all entries at once to prevent your heap from blowing up. The limiter takes one concurrency options which defaults to the number of CPUs in the system, I usually use the default number for both webpack and the limiter. Author of the plugin above has added concurrency support.
4. ts-loader and of cause you need this for TypeScript.

serverless.yml

I am only adding related options here, please study and merge with your existing entries.

plugins:
  - serverless-webpack

custom:
  webpack:
    webpackConfig: serverless/plugin-webpack-config.js
    serializedCompile: true # This is the official one
    concurrency: 5          # This is from my fork, customize and experiment yourself

serverless/plugin-webpack-config.js

This is a reduced version of my configuration.

const serverlessWebpack = require("serverless-webpack");
const nodeExternals = require("webpack-node-externals");
const ForkTsCheckerWebpackPlugin = require("fork-ts-checker-webpack-plugin");
const ForkTsCheckerWebpackPluginLimiter = require("fork-ts-checker-webpack-plugin-limiter");
const webpack = require("webpack");

const TSCONFIG_PATH = "plugin-webpack-tsconfig.json";

const {
  lib: {
    entries,
    serverless: { service: { custom: { webpack: { concurrency } = {} } = {} } = {} } = {},
    webpack: { isLocal },
  } = {},
} = serverlessWebpack;

module.exports = async () => ({
  entry: entries,
  externals: [nodeExternals()],
  module: {
    rules: [
      {
        test: /\.(t|j)s$/,
        exclude: /(node_modules|bower_components)/,
        use: [
          {
            loader: "ts-loader",
            options: {
              configFile: TSCONFIG_PATH,
              transpileOnly: true,
            },
          },
        ],
      },
    ],
  },
  plugins: [
    new ForkTsCheckerWebpackPlugin({ typescript: { tsconfig: TSCONFIG_PATH } }),
    new ForkTsCheckerWebpackPluginLimiter({ concurrency }), // in theory, pairing this concurrency with the number of webpack threads should gives the best performance.
    // new webpack.DefinePlugin({ "global.GENTLY": false }), // this solves something else I already forgot, keeping it here in case you need it
    ... // other plugins
  ],
  resolve: {
    extensions: [".js", ".ts"],
  },
  ... // other options
});

serverless/plugin-webpack-tsconfig.json

This is just normal tsconfig, it's totally project dependent so I'm not sharing it here.