grafana / helm-charts

Geek Repo:Geek Repo

Github PK Tool:Github PK Tool

Loki ruler SERVER_ERROR out of memory storing object warnings

slimes28 opened this issue · comments

Hello,
I am using loki-distributed helm chart with configured memcachedChunks, memcachedFrontend and memcachedIndexQueries but the ruler has a lot of warnings: memcache: unexpected response line from \"set\": \"SERVER_ERROR out of memory storing object\\r\\n\.

Here is how memcached is configured:

         memcachedChunks:
            enabled: true
            extraArgs:
              - -m 2048
              - -I 2m
              - -v
            resources:
              limits:
                cpu: "2"
                memory: 3Gi
              requests:
                cpu: 500m
                memory: 3Gi

          memcachedFrontend:
            enabled: true
            resources:
              limits:
                cpu: 100m
                memory: 200Mi
              requests:
                cpu: 10m
                memory: 10Mi

          memcachedIndexQueries:
            enabled: true
            resources:
              limits:
                cpu: "1"
                memory: 200Mi
              requests:
                cpu: 100m
                memory: 100Mi
       
          loki:
            structuredConfig:
              storage_config:
                index_queries_cache_config:
                  memcached:
                    batch_size: 100
                    parallelism: 100
                  memcached_client:
                    consistent_hash: true

              chunk_store_config:
                chunk_cache_config:
                  memcached:
                    batch_size: 100
                    parallelism: 100
                  memcached_client:
                    consistent_hash: true
                    timeout: 60s
                  background:
                    writeback_goroutines: 1
                    writeback_buffer: 1000
                    writeback_size_limit: 500MB

              query_range:
                parallelise_shardable_queries: true
                results_cache:
                  cache:
                    memcached_client:
                      consistent_hash: true
                      max_idle_conns: 16
                      timeout: 500ms
                      update_interval: 1m