tarantool / prometheus

Prometheus library to collect metrics from Tarantool

Geek Repo:Geek Repo

Github PK Tool:Github PK Tool

monitoring error Prometheus

lexoff92 opened this issue · comments

Installed Tarantool 2.1:
yum install tarantool
For monitoring decided to use Prometneus:

yum install tarantool-http -y
tarantoolctl rocks install prometheus

Created a LUA script:

box.cfg{
  listen = 3301,
}
box.once("schema", function()
  box.schema.user.grant('guest', 'read,write,execute', 'universe')
  box.schema.space.create("test")
  box.space.test:create_index("primary")

local http = require('http.server')

local prometheus = require('prometheus')

prometheus.init()

local httpd = http.new('0.0.0.0', 8080)



local memory_limit_bytes = prometheus.gauge(
    'tarantool_memory_limit_bytes',
    'Maximum amount of memory Tarantool can use')
local memory_used_bytes = prometheus.gauge(
    'tarantool_memory_used_bytes',
    'Amount of memory currently used by Tarantool')
local tuples_memory_bytes = prometheus.gauge(
    'tarantool_tuples_memory_bytes',
    'Amount of memory allocated for Tarantool tuples')
local system_memory_bytes = prometheus.gauge(
    'tarantool_system_memory_bytes',
    'Amount of memory used by Tarantool indexes and system')

local requests_total = prometheus.gauge(
    'tarantool_requests_total',
    'Total number of requests by request type',
    {'request_type'})

local uptime_seconds = prometheus.gauge(
    'tarantool_uptime_seconds',
    'Number of seconds since the server started')

local tuples_total = prometheus.gauge(
    'tarantool_space_tuples_total',
    'Total number of tuples in a space',
    {'space_name'})

local replication_lag = prometheus.gauge(
    'tarantool_replication_lag',
    'The time difference between the instance and the master',
    {'uuid'})
local replication_state_normal = prometheus.gauge(
    'tarantool_is_replication_healthy',
    'Is replication healthy?')


local function measure_tarantool_memory_usage()
    local slabs = box.slab.info()
    local memory_limit = slabs.quota_size
    local memory_used = slabs.quota_used
    local tuples_memory = slabs.arena_used
    local system_memory = memory_used - tuples_memory

    memory_limit_bytes:set(memory_limit)
    memory_used_bytes:set(memory_used)
    tuples_memory_bytes:set(tuples_memory)
    system_memory_bytes:set(system_memory)
end

local function measure_tarantool_request_stats()
    local stat = box.stat()
    local request_types = {'delete', 'select', 'insert', 'eval', 'call',
                           'replace', 'upsert', 'auth', 'error', 'update'}

    for _, request_type in ipairs(request_types) do
        requests_total:set(stat[string.upper(request_type)].total,
                           {request_type})
    end
end

local function measure_tarantool_uptime()
    uptime_seconds:set(box.info.uptime)
end

local function measure_tarantool_space_stats()
    for _, space in box.space._space:pairs() do
        local space_name = space[3]

        if string.sub(space_name, 1,1) ~= '_' then
            tuples_total:set(box.space[space_name]:len(), {space_name})
        end
    end
end

local function measure_tarantool_replication_lag()
    local idle = 0

    for _, replica in ipairs(box.info.replication) do
        if replica.upstream ~= nil then
            replication_lag:set(replica.upstream.lag, { replica.uuid })
            if replica.upstream.idle > idle then
                idle = replica.upstream.idle
            end
        end
    end

    if idle ~= 0 then
        local replication_timeout = box.cfg.replication_timeout
        if idle <= replication_timeout then
            replication_state_normal:set(1)
        else
            replication_state_normal:set(0)
        end
    end
end

local function measure_tarantool_metrics()
    if type(box.cfg) ~= 'function' then
        measure_tarantool_memory_usage()
        measure_tarantool_request_stats()
        measure_tarantool_uptime()
        measure_tarantool_space_stats()
        measure_tarantool_replication_lag()
    end
end

httpd:route( { path = '/metrics' }, prometheus.collect_http)

httpd:start()

return {measure_tarantool_metrics=measure_tarantool_metrics}

Which creates a database and starts collecting metrics with the help of the prometneus module.

If run like:
tarantool test.lua
it works

But if you run like this:
tarantoolctl start test
I get an error:

May 30 13:50:24 tarantool02 tarantoolctl[14478]: Run console at unix/:/var/run/tarantool/replica.control
May 30 13:50:24 tarantool02 tarantoolctl[14478]: started
May 30 13:50:24 tarantool02 tarantoolctl[14478]: no file '/usr/local/share/lua/5.1/prometheus.lua'
May 30 13:50:24 tarantool02 tarantoolctl[14478]: no file '/usr/local/share/lua/5.1/prometheus/init.lua'
May 30 13:50:24 tarantool02 tarantoolctl[14478]: no file '/usr/share/lua/5.1/prometheus.lua'
May 30 13:50:24 tarantool02 tarantoolctl[14478]: no file '/usr/share/lua/5.1/prometheus/init.lua'
May 30 13:50:24 tarantool02 tarantoolctl[14478]: no file '/var/lib/tarantool/.luarocks/lib/lua/5.1/prometheus.so'
May 30 13:50:24 tarantool02 tarantoolctl[14478]: no file '/var/lib/tarantool/.luarocks/lib/lua/prometheus.so'
May 30 13:50:24 tarantool02 tarantoolctl[14478]: no file '/usr/local/lib64/tarantool/prometheus.so'
May 30 13:50:24 tarantool02 tarantoolctl[14478]: no file '/usr/lib64/tarantool/prometheus.so'
May 30 13:50:24 tarantool02 tarantoolctl[14478]: no file '/usr/local/lib64/lua/5.1/prometheus.so'
May 30 13:50:24 tarantool02 tarantoolctl[14478]: no file '/usr/lib64/lua/5.1/prometheus.so'
May 30 13:50:24 tarantool02 systemd[1]: tarantool@replica.service: main process exited, code=exited, status=1/FAILURE
May 30 13:50:24 tarantool02 tarantoolctl[14487]: Stopping instance replica...
May 30 13:50:24 tarantool02 tarantoolctl[14487]: Process is not running (pid: /var/run/tarantool/replica.pid)
May 30 13:50:24 tarantool02 systemd[1]: Failed to start Tarantool Database Server.

commands local prometheus = require('prometheus') fails....

How to solve this problem?

Try to move this to the beginning of file, outside of box.once:

local prometheus = require('prometheus')

Try to move this to the beginning of file, outside of box.once:

local prometheus = require('prometheus')

in this case:

local prometheus = require('prometheus')

box.cfg{
  listen = 3301,
...........

error:

May 31 09:43:28 tarantool02 tarantoolctl[30109]: Starting instance replica...
May 31 09:43:28 tarantool02 tarantoolctl[30109]: Start failed: /etc/tarantool/instances.available/replica.lua:1: module 'prometheus' not found:
May 31 09:43:28 tarantool02 tarantoolctl[30109]: no field package.preload['prometheus']
May 31 09:43:28 tarantool02 tarantoolctl[30109]: no file './prometheus.lua'
May 31 09:43:28 tarantool02 tarantoolctl[30109]: no file './prometheus/init.lua'
May 31 09:43:28 tarantool02 tarantoolctl[30109]: no file './prometheus.so'
May 31 09:43:28 tarantool02 tarantoolctl[30109]: no file '//.rocks/share/tarantool/prometheus.lua'
May 31 09:43:28 tarantool02 tarantoolctl[30109]: no file '//.rocks/share/tarantool/prometheus/init.lua'
May 31 09:43:28 tarantool02 tarantoolctl[30109]: no file '/.rocks/share/tarantool/prometheus.lua'
May 31 09:43:28 tarantool02 tarantoolctl[30109]: no file '/.rocks/share/tarantool/prometheus/init.lua'
May 31 09:43:28 tarantool02 tarantoolctl[30109]: no file '//.rocks/lib/tarantool/prometheus.so'
May 31 09:43:28 tarantool02 tarantoolctl[30109]: no file '/.rocks/lib/tarantool/prometheus.so'

I noticed the following feature:
I launch instans without module prometneus. And try to execute "local prometheus = require ('prometheus')" in the following cases:

[root@tarantool02 instances.available]# ps -ef | grep tarantool
taranto+ 30308     1  0 09:46 ?        00:00:00 tarantool replica.lua <running>
root     30403 11467  0 09:48 pts/0    00:00:00 grep --color=auto tarantool
[root@tarantool02 instances.available]#
[root@tarantool02 instances.available]#
[root@tarantool02 instances.available]# tarantool
Tarantool 2.1.2-91-g70f9eb1
type 'help' for interactive help
tarantool> local prometheus = require('prometheus')
---
...

tarantool> [root@tarantool02 instances.available]#
[root@tarantool02 instances.available]#
[root@tarantool02 instances.available]#
[root@tarantool02 instances.available]# tarantoolctl connect localhost:3301
connected to localhost:3301
localhost:3301> local prometheus = require('prometheus')
---
- error: "[string \"local prometheus = require('prometheus')\"]:1: module 'prometheus'
    not found:\n\tno field package.preload['prometheus']\n\tno file './prometheus.lua'\n\tno
    file './prometheus/init.lua'\n\tno file './prometheus.so'\n\tno file '//.rocks/share/tarantool/prometheus.lua'\n\tno
    file '//.rocks/share/tarantool/prometheus/init.lua'\n\tno file '/.rocks/share/tarantool/prometheus.lua'\n\tno
    file '/.rocks/share/tarantool/prometheus/init.lua'\n\tno file '//.rocks/lib/tarantool/prometheus.so'\n\tno
    file '/.rocks/lib/tarantool/prometheus.so'\n\tno file '/var/lib/tarantool/.luarocks/share/lua/5.1/prometheus.lua'\n\tno
    file '/var/lib/tarantool/.luarocks/share/lua/5.1/prometheus/init.lua'\n\tno file
    '/var/lib/tarantool/.luarocks/share/lua/prometheus.lua'\n\tno file '/var/lib/tarantool/.luarocks/share/lua/prometheus/init.lua'\n\tno
    file '/usr/local/share/tarantool/prometheus.lua'\n\tno file '/usr/local/share/tarantool/prometheus/init.lua'\n\tno
    file '/usr/share/tarantool/prometheus.lua'\n\tno file '/usr/share/tarantool/prometheus/init.lua'\n\tno
    file '/usr/local/share/lua/5.1/prometheus.lua'\n\tno file '/usr/local/share/lua/5.1/prometheus/init.lua'\n\tno
    file '/usr/share/lua/5.1/prometheus.lua'\n\tno file '/usr/share/lua/5.1/prometheus/init.lua'\n\tno
    file '/var/lib/tarantool/.luarocks/lib/lua/5.1/prometheus.so'\n\tno file '/var/lib/tarantool/.luarocks/lib/lua/prometheus.so'\n\tno
    file '/usr/local/lib64/tarantool/prometheus.so'\n\tno file '/usr/lib64/tarantool/prometheus.so'\n\tno
    file '/usr/local/lib64/lua/5.1/prometheus.so'\n\tno file '/usr/lib64/lua/5.1/prometheus.so'"
...

localhost:3301>

in the first case, the team successfully worked.
But when connected to an instance - no.

You have prometheus installed locally into your home luarocks dir, and not system-wide, this is why it is not available to the system-wide instance.
The default of luarocks install is --local, so please install prometheus globally or launch your instances from your local directory by addint .config/tarantool/tarantool:

ostja@atlas ~/.config/tarantool
 % ls
instances.available  instances.enabled  tarantool
kostja@atlas ~/.config/tarantool
 % cat tarantool 
-- Options for Tarantool
default_cfg = {
    pid_file   = "/home/kostja/local/var/", -- will become pid_file .. instance .. '.pid'
    wal_dir    = "/home/kostja/local/var/", -- will become wal_dir/instance/
    snap_dir   = "/home/kostja/local/var/", -- snap_dir/instance/
    sophia_dir = "/home/kostja/local/var/", -- will become sophia_dir/sophia/instance/
    logger     = "/home/kostja/local/var/", -- logger/instance .. '.log'
--    username   = "sysadm",
}
instance_dir = "/home/kostja/local/var"

-- vim: set ft=lua :

You have prometheus installed locally into your home luarocks dir, and not system-wide, this is why it is not available to the system-wide instance.
The default of luarocks install is --local, so please install prometheus globally or launch your instances from your local directory by addint .config/tarantool/tarantool:

ostja@atlas ~/.config/tarantool
 % ls
instances.available  instances.enabled  tarantool
kostja@atlas ~/.config/tarantool
 % cat tarantool 
-- Options for Tarantool
default_cfg = {
    pid_file   = "/home/kostja/local/var/", -- will become pid_file .. instance .. '.pid'
    wal_dir    = "/home/kostja/local/var/", -- will become wal_dir/instance/
    snap_dir   = "/home/kostja/local/var/", -- snap_dir/instance/
    sophia_dir = "/home/kostja/local/var/", -- will become sophia_dir/sophia/instance/
    logger     = "/home/kostja/local/var/", -- logger/instance .. '.log'
--    username   = "sysadm",
}
instance_dir = "/home/kostja/local/var"

-- vim: set ft=lua :

i installed prometneus in this way:
tarantoolctl rocks install prometheus
Tell me, please, how to install it global?

my Tarantool config:

 cat /etc/sysconfig/tarantool
default_cfg = {
    pid_file   = "/var/run/tarantool", -- /var/run/tarantool/${INSTANCE}.pid
    wal_dir    = "/var/lib/tarantool", -- /var/lib/tarantool/${INSTANCE}/
    memtx_dir  = "/var/lib/tarantool", -- /var/lib/tarantool/${INSTANCE}
    vinyl_dir  = "/var/lib/tarantool", -- /var/lib/tarantool/${INSTANCE}
    log        = "/var/log/tarantool", -- /var/log/tarantool/${INSTANCE}.log
    username   = "tarantool",
    language   = "lua",
}
instance_dir = "/etc/tarantool/instances.available"
-- vim: set ft=lua :

Try running it from the super user (su -), please also verify where the package is installed afterwards.

Try running it from the super user (su -), please also verify where the package is installed afterwards.

Yes, I ran the command "tarantoolctl rocks install prometheus
" as root user in the directory" /etc/tarantool/instances.available/ ". There appeared a module:

[root@tarantool02 instances.available]# pwd
/etc/tarantool/instances.available
[root@tarantool02 instances.available]# ll -a
total 20
drwxr-xr-x 3 root      root        85 May 31 11:19 .
drwxr-xr-x 3 root      root        33 May 29 22:31 ..
-rw-r--r-- 1 root      root      7058 May 29 22:26 example.lua
-rw-r--r-- 1 tarantool tarantool 2012 May 30 16:42 replica.lua
-rw-r--r-- 1 root      root      5670 May 31 09:55 replica.lua.metrics
drwxr-xr-x 3 root      root        19 May 30 12:31 .rocks

I installed Tarantool from packages:
yum install tarantool
And I run the instance under the "root"
tarantoolctl start replica

in the processes it is launched from "tarantool":

[root@tarantool02 instances.available]# ps -ef | grep tarantool
taranto+ 30308     1  0 09:46 ?        00:00:00 tarantool replica.lua <running>

i did the following:

[root@tarantool02 tarantool]# su -s /bin/bash tarantool
bash-4.2$ cd /etc/tarantool/instances.available/
bash-4.2$ tarantoolctl rocks install prometheus
Installing http://rocks.tarantool.org/prometheus-scm-1.rockspec
Cloning into 'prometheus'...
remote: Enumerating objects: 19, done.
remote: Counting objects: 100% (19/19), done.
remote: Compressing objects: 100% (19/19), done.
remote: Total 19 (delta 2), reused 5 (delta 0), pack-reused 0
Receiving objects: 100% (19/19), 10.67 KiB | 0 bytes/s, done.
Resolving deltas: 100% (2/2), done.
No existing manifest. Attempting to rebuild...
prometheus scm-1 is now installed in /etc/tarantool/instances.available/.rocks (license: BSD)

but it did not help....

In my case, you probably need to install "prometneus" from the YUM repository.
But there is no this package:

[root@tarantool02 instances.available]# yum search tarantool-
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
 * base: mirror.reconn.ru
 * epel: mirror.logol.ru
 * epel-debuginfo: mirror.logol.ru
 * epel-source: mirror.logol.ru
 * extras: mirror.sale-dedic.com
 * updates: mirror.reconn.ru
========================================================================================= N/S matched: tarantool- =========================================================================================
tarantool-argon2-debuginfo.x86_64 : Debug information for package tarantool-argon2
tarantool-avro-schema-debuginfo.x86_64 : Debug information for package tarantool-avro-schema
tarantool-c-debuginfo.x86_64 : Debug information for package tarantool-c
tarantool-gis-debuginfo.x86_64 : Debug information for package tarantool-gis
tarantool-http-debuginfo.x86_64 : Debug information for package tarantool-http
tarantool-lrexlib-debuginfo.x86_64 : Debug information for package tarantool-lrexlib
tarantool-mqtt-debuginfo.x86_64 : Debug information for package tarantool-mqtt
tarantool-mysql-debuginfo.x86_64 : Debug information for package tarantool-mysql
tarantool-pg-debuginfo.x86_64 : Debug information for package tarantool-pg
tarantool-shard-debuginfo.x86_64 : Debug information for package tarantool-shard
tarantool-zookeeper-debuginfo.x86_64 : Debug information for package tarantool-zookeeper
tarantool-argon2.x86_64 : Password hash Argon2, winner of PHC
tarantool-avro-schema.x86_64 : Apache Avro bindings for Tarantool
tarantool-c.x86_64 : Tarantool C connector
tarantool-c-devel.x86_64 : Development files for C libtnt
tarantool-checks.noarch : Persistent in-memory queues for Tarantool
tarantool-debuginfo.x86_64 : Debug information for package tarantool
tarantool-devel.x86_64 : Server development files for tarantool
tarantool-dump.noarch : Logical backup and restore for Tarantool
tarantool-expirationd.noarch : Expiration daemon for Tarantool
tarantool-gis.x86_64 : Full-featured geospatial extension for Tarantool Database
tarantool-graphql.noarch : Set of adapters for GraphQL query language to the Tarantool data model
tarantool-http.x86_64 : HTTP server for Tarantool
tarantool-lrexlib-gnu.x86_64 : Lua binding of GNU library
tarantool-lrexlib-oniguruma.x86_64 : Lua binding of Oniguruma library
tarantool-lrexlib-pcre.x86_64 : Lua binding of PCRE library
tarantool-lrexlib-pcre2.x86_64 : Lua binding of PCRE-2 library
tarantool-lrexlib-posix.x86_64 : Lua binding of POSIX library
tarantool-luacheck.noarch : A static analyzer and a linter for Lua
tarantool-lulpeg.noarch : LuLPeg
tarantool-mqtt.x86_64 : Tarantool MQTT module
tarantool-mysql.x86_64 : MySQL connector for Tarantool
tarantool-pg.x86_64 : PostgreSQL connector for Tarantool
tarantool-python.noarch : Python client library for Tarantool Database
tarantool-queue.noarch : Persistent in-memory queues for Tarantool
tarantool-shard.x86_64 : Tarantool sharding module
tarantool-try.noarch : try.tarantool.org service
tarantool-zookeeper.x86_64 : Tarantool Zookeeper module

  Name and summary matches only, use "search all" for everything.
[root@tarantool02 instances.available]#
[root@tarantool02 instances.available]#
[root@tarantool02 instances.available]# yum search tarantool- | grep prom
[root@tarantool02 instances.available]#
[root@tarantool02 instances.available]#
[root@tarantool02 instances.available]#

It turns out that I can successfully run the instance only there:
tarantool replica.lua
but not:
tarantoolctl start replica

but in the first case there is no way to correctly stop the instance! or is there?

Take this file and place it in the same directory with test.lua:

https://gist.github.com/knazarov/48f8af717ce3d36d8e38f91ef0d281f2

Then in your test.lua at the top of file, before all requires, write this:

-- This is necessary so that we can load dependencies
-- even when we are not in the same directory with them
local script_dir = debug.getinfo(1, "S").source:sub(2):match("(.*/)") or './'
dofile(script_dir ..'/env.lua')
-- This is necessary so that we can load dependencies
-- even when we are not in the same directory with them
local script_dir = debug.getinfo(1, "S").source:sub(2):match("(.*/)") or './'
dofile(script_dir ..'/env.lua')
[root@tarantool02 instances.available]# tarantoolctl rocks list

Installed rocks:
----------------

prometheus
   scm-1 (installed) - /etc/tarantool/instances.available/.rocks/share/tarantool/rocks

[root@tarantool02 instances.available]# pwd
/etc/tarantool/instances.available
[root@tarantool02 instances.available]# ll
total 24
-rw-r--r-- 1 tarantool tarantool 1846 May 31 13:04 env.lua
-rw-r--r-- 1 tarantool tarantool 7058 May 29 22:26 example.lua
-rw-r--r-- 1 tarantool tarantool 5891 May 31 13:05 replica.lua
-rw-r--r-- 1 tarantool tarantool 2012 May 30 16:42 replica.lua.bak
[root@tarantool02 instances.available]# tarantoolctl start replica
Starting instance replica...
Forwarding to 'systemctl start tarantool@replica'
Job for tarantool@replica.service failed because the control process exited with error code. See "systemctl status tarantool@replica.service" and "journalctl -xe" for details.
[root@tarantool02 instances.available]#

error:

May 31 13:07:32 tarantool02 tarantoolctl[11210]: Starting instance replica...
May 31 13:07:32 tarantool02 tarantoolctl[11210]: Run console at unix/:/var/run/tarantool/replica.control
May 31 13:07:32 tarantool02 tarantoolctl[11210]: started
May 31 13:07:32 tarantool02 tarantoolctl[11210]: no file '/var/lib/tarantool/.luarocks/lib/lua/5.1/prometheus.so'
May 31 13:07:32 tarantool02 tarantoolctl[11210]: no file '/var/lib/tarantool/.luarocks/lib/lua/prometheus.so'
May 31 13:07:32 tarantool02 tarantoolctl[11210]: no file '/usr/local/lib64/tarantool/prometheus.so'
May 31 13:07:32 tarantool02 tarantoolctl[11210]: no file '/usr/lib64/tarantool/prometheus.so'
May 31 13:07:32 tarantool02 tarantoolctl[11210]: no file '/usr/local/lib64/lua/5.1/prometheus.so'
May 31 13:07:32 tarantool02 tarantoolctl[11210]: no file '/usr/lib64/lua/5.1/prometheus.so'
May 31 13:07:32 tarantool02 tarantoolctl[11210]: no file '//prometheus.dylib'
May 31 13:07:32 tarantool02 tarantoolctl[11210]: no file '//prometheus.so'
May 31 13:07:32 tarantool02 tarantoolctl[11210]: no file '//.rocks/lib/tarantool/prometheus.dylib'
May 31 13:07:32 tarantool02 tarantoolctl[11210]: no file '//.rocks/lib/tarantool/prometheus.so'
May 31 13:07:32 tarantool02 systemd[1]: tarantool@replica.service: main process exited, code=exited, status=1/FAILURE
May 31 13:07:32 tarantool02 tarantoolctl[11219]: Stopping instance replica...
May 31 13:07:32 tarantool02 tarantoolctl[11219]: Process is not running (pid: /var/run/tarantool/replica.pid)
May 31 13:07:32 tarantool02 systemd[1]: Failed to start Tarantool Database Server.

no file '//.rocks/lib/tarantool/prometheus.so'

[root@tarantool02 .rocks]# pwd
/etc/tarantool/instances.available/.rocks
[root@tarantool02 .rocks]# ls
share
[root@tarantool02 .rocks]# ll share/tarantool/
prometheus/     prometheus.lua  rocks/

Looks like I made a mistake in my env.lua. Take the new version from here:

https://gist.github.com/knazarov/48f8af717ce3d36d8e38f91ef0d281f2

Looks like I made a mistake in my env.lua. Take the new version from here:

it works, thanks!

We are going to solve this problem soon, see tarantool/tarantool#4193