Running swarmprom behind traefik

[Jonathan Kirszling]

I want to be able to use swarmprom behind reverse proxy.
First problem is grafana, when i query https://supervision.infra.domain.fr/grafana
I get

If you're seeing this Grafana has failed to load its application files
1. This could be caused by your reverse proxy settings.
2. If you host grafana under subpath make sure your grafana.ini root_path setting includes subpath
3. If you have a local dev build make sure you build frontend using: npm run dev, npm run watch, or npm run build
4. Sometimes restarting grafana-server can help

I found solution here https://grafana.com/docs/grafana/latest/installation/behind_proxy/
I need to be able to change root_url var

docker exec -it d13a4a534e36 /bin/cat conf/defaults.ini | grep root_url
root_url = %(protocol)s://%(domain)s:%(http_port)s/

Is there a way to do this ?
I created a config file defaults.ini bind on /usr/share/grafana/conf/defaults.ini
And changed value as

root_url = %(protocol)s://%(domain)s:%(http_port)s/grafana/

But i still get the same msg

Here my complete docker-stack file

version: "3.3"

networks:
  net:
    driver: overlay
    attachable: true
  traefik-public:
    external: true

volumes:
    prometheus: {}
    grafana: {}
    alertmanager: {}

configs:
  dockerd_config:
    file: ./dockerd-exporter/Caddyfile
  node_rules:
    file: ./prometheus/rules/swarm_node.rules.yml
  task_rules:
    file: ./prometheus/rules/swarm_task.rules.yml
  grafana_config:
    file: ./grafana/defaults.ini

services:
  dockerd-exporter:
    image: stefanprodan/caddy
    networks:
      - net
    environment:
      - DOCKER_GWBRIDGE_IP=172.18.0.1
    configs:
      - source: dockerd_config
        target: /etc/caddy/Caddyfile
    deploy:
      mode: global
      resources:
        limits:
          memory: 128M
        reservations:
          memory: 64M

  cadvisor:
    image: google/cadvisor
    networks:
      - net
    command: -logtostderr -docker_only
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /:/rootfs:ro
      - /var/run:/var/run
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
    deploy:
      mode: global
      resources:
        limits:
          memory: 128M
        reservations:
          memory: 64M

  grafana:
    image: stefanprodan/swarmprom-grafana:5.3.4
    networks:
      - default
      - net
      - traefik-public
    environment:
      - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
      - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-password}
      - GF_USERS_ALLOW_SIGN_UP=false
      #- GF_SERVER_ROOT_URL=${GF_SERVER_ROOT_URL:-localhost}
      #- GF_SMTP_ENABLED=${GF_SMTP_ENABLED:-false}
      #- GF_SMTP_FROM_ADDRESS=${GF_SMTP_FROM_ADDRESS:[e-mail]}
      #- GF_SMTP_FROM_NAME=${GF_SMTP_FROM_NAME:-Grafana}
      #- GF_SMTP_HOST=${GF_SMTP_HOST:-smtp:25}
      #- GF_SMTP_USER=${GF_SMTP_USER}
      #- GF_SMTP_PASSWORD=${GF_SMTP_PASSWORD}
    configs:
      - source: grafana_config
        target: /usr/share/grafana/conf/defaults.ini
    volumes:
      - grafana:/var/lib/grafana
    deploy:
      mode: replicated
      replicas: 1
      placement:
        constraints:
          - node.role == manager
      resources:
        limits:
          memory: 128M
        reservations:
          memory: 64M
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.grafana.entrypoints=https"
        - "traefik.http.routers.grafana.tls=true"
        - "traefik.http.routers.grafana.rule=Host(`tspeda-swarm-supervision.infra.domain.fr`) && PathPrefix(`/grafana`)"
        - "traefik.http.services.grafana.loadbalancer.server.port=3000"

  alertmanager:
    image: stefanprodan/swarmprom-alertmanager:v0.14.0
    networks:
      - default
      - net
      - traefik-public
    environment:
      - SLACK_URL=${SLACK_URL:-https://hooks.slack.com/services/TOKEN}
      - SLACK_CHANNEL=${SLACK_CHANNEL:-general}
      - SLACK_USER=${SLACK_USER:-alertmanager}
    command:
      - '--config.file=/etc/alertmanager/alertmanager.yml'
      - '--storage.path=/alertmanager'
    volumes:
      - alertmanager:/alertmanager
    deploy:
      mode: replicated
      replicas: 1
      placement:
        constraints:
          - node.role == manager
      resources:
        limits:
          memory: 128M
        reservations:
          memory: 64M
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.alertmanager.entrypoints=https"
        - "traefik.http.routers.alertmanager.tls=true"
        - "traefik.http.routers.alertmanager.rule=Host(`tspeda-swarm-supervision.infra.domain.fr`) && PathPrefix(`/alertmanager`)"
        - "traefik.http.services.alertmanager.loadbalancer.server.port=9093"

  unsee:
    image: cloudflare/unsee:v0.8.0
    networks:
      - default
      - net
      - traefik-public
    environment:
      - "ALERTMANAGER_URIS=default:http://alertmanager:9093"
    deploy:
      mode: replicated
      replicas: 1
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.unsee.entrypoints=https"
        - "traefik.http.routers.unsee.tls=true"
        - "traefik.http.routers.unsee.rule=Host(`tspeda-swarm-supervision.infra.domain.fr`) && PathPrefix(`/unsee`)"
        - "traefik.http.services.unsee.loadbalancer.server.port=8080"

  node-exporter:
    image: stefanprodan/swarmprom-node-exporter:v0.16.0
    networks:
      - net
    environment:
      - NODE_ID={{.Node.ID}}
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
      - /etc/hostname:/etc/nodename
    command:
      - '--path.sysfs=/host/sys'
      - '--path.procfs=/host/proc'
      - '--collector.textfile.directory=/etc/node-exporter/'
      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
      - '--no-collector.ipvs'
    deploy:
      mode: global
      resources:
        limits:
          memory: 128M
        reservations:
          memory: 64M

  prometheus:
    image: stefanprodan/swarmprom-prometheus:v2.5.0
    networks:
      - default
      - net
      - traefik-public
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention=${PROMETHEUS_RETENTION:-24h}'
    volumes:
      - prometheus:/prometheus
    configs:
      - source: node_rules
        target: /etc/prometheus/swarm_node.rules.yml
      - source: task_rules
        target: /etc/prometheus/swarm_task.rules.yml
    deploy:
      mode: replicated
      replicas: 1
      placement:
        constraints:
          - node.role == manager
      resources:
        limits:
          memory: 2048M
        reservations:
          memory: 128M
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.prometheus.entrypoints=https"
        - "traefik.http.routers.prometheus.tls=true"
        - "traefik.http.routers.prometheus.rule=Host(`tspeda-swarm-supervision.infra.domain.fr`) && PathPrefix(`/prometheus`)"
        - "traefik.http.services.prometheus.loadbalancer.server.port=9090"

[Lee Tuan]

I've got the same issue!!!

[Jonathan Kirszling]

I did change my configuration to match requirements : no subdirs but one fqdn by service.

[Jack Henschel]

I also wanted to run swarmprom's Web UIs in a sub-path (using Traefik as a reverse proxy). Here's what I had to do:

1. Add --web.external-url=/prometheus/ to the prometheus command (in docker-compose.yml)
2. Set the Traefik rule for prometheus to: traefik...rule="Host('example.com') && PathPrefix('/prometheus/')")
   (or if you are using Traefik v1: traefik...rule=Host:example.com;PathPrefix:/prometheus/)
3. Add GF_SERVER_ROOT_URL=https://example.com/grafana/ environment variable to grafana service in docker-compose.yml
4. Set the Traefik rule for Grafana to PathPrefixStrip('/grafana/')

See my post here for more details: https://blog.cubieserver.de/2020/configure-prometheus-on-a-sub-path-behind-reverse-proxy/