使用zabbix监控Redis代理twemproxy-白红宇

使用zabbix监控Redis代理twemproxy

阅读量：5765 次

发布时间：2019-06-18

本文共 13044 字，大约阅读时间需要 43 分钟。

一监控原理描述

twemproxy默认监听端口22222，可以使用nc等工具直接访问，以JSON格式输出

$ nc 127.0.0.1 22222{"service":"nutcracker", "source":"gintama-taiwan-redis2", "version":"0.4.0", "uptime":67135, "timestamp":1427100529, "total_connections":376, "curr_connections":7, "gintama": {"client_eof":0, "client_err":311, "client_connections":0, "server_ejects":58, "forward_error":53, "fragments":0, "192.168.100.68:6379": {"server_eof":1, "server_err":12, "server_timedout":0, "server_connections":1, "server_ejected_at":1427074216551863, "requests":553, "request_bytes":333854, "responses":541, "response_bytes":265633, "in_queue":0, "in_queue_bytes":0, "out_queue":0, "out_queue_bytes":0},"192.168.100.68:6380": {"server_eof":1, "server_err":9, "server_timedout":0, "server_connections":1, "server_ejected_at":1427074216552893, "requests":20, "request_bytes":1551, "responses":11, "response_bytes":207, "in_queue":0, "in_queue_bytes":0, "out_queue":0, "out_queue_bytes":0},"192.168.100.68:6381": {"server_eof":1, "server_err":1, "server_timedout":0, "server_connections":1, "server_ejected_at":1427037406770236, "requests":287, "request_bytes":10718, "responses":286, "response_bytes":23430, "in_queue":0, "in_queue_bytes":0, "out_queue":0, "out_queue_bytes":0},"192.168.100.69:6379": {"server_eof":1, "server_err":11, "server_timedout":0, "server_connections":1, "server_ejected_at":1427074216550647, "requests":55, "request_bytes":2633, "responses":44, "response_bytes":220, "in_queue":0, "in_queue_bytes":0, "out_queue":0, "out_queue_bytes":0},"192.168.100.69:6380": {"server_eof":1, "server_err":9, "server_timedout":0, "server_connections":1, "server_ejected_at":1427040015877803, "requests":291, "request_bytes":15563, "responses":282, "response_bytes":1402, "in_queue":0, "in_queue_bytes":0, "out_queue":0, "out_queue_bytes":0},"192.168.100.69:6381": {"server_eof":0, "server_err":11, "server_timedout":0, "server_connections":1, "server_ejected_at":1427059816411041, "requests":193, "request_bytes":17292, "responses":182, "response_bytes":1767, "in_queue":0, "in_queue_bytes":0, "out_queue":0, "out_queue_bytes":0}}}

可以使用工具整理一下输出

{    "service": "nutcracker",    "source": "gintama-taiwan-redis2",    "version": "0.4.0",    "uptime": 64944,    "timestamp": 1427098338,    "total_connections": 374,    "curr_connections": 7,    "gintama": {        "client_eof": 0,        "client_err": 309,        "client_connections": 0,        "server_ejects": 58,        "forward_error": 53,        "fragments": 0,        "192.168.100.68:6379": {            "server_eof": 1,            "server_err": 12,            "server_timedout": 0,            "server_connections": 1,            "server_ejected_at": 1427074216551863,            "requests": 552,            "request_bytes": 333808,            "responses": 540,            "response_bytes": 265628,            "in_queue": 0,            "in_queue_bytes": 0,            "out_queue": 0,            "out_queue_bytes": 0        },        "192.168.100.68:6380": {            "server_eof": 1,            "server_err": 9,            "server_timedout": 0,            "server_connections": 1,            "server_ejected_at": 1427074216552893,            "requests": 19,            "request_bytes": 1498,            "responses": 10,            "response_bytes": 202,            "in_queue": 0,            "in_queue_bytes": 0,            "out_queue": 0,            "out_queue_bytes": 0        },        "192.168.100.68:6381": {            "server_eof": 1,            "server_err": 1,            "server_timedout": 0,            "server_connections": 1,            "server_ejected_at": 1427037406770236,            "requests": 286,            "request_bytes": 10672,            "responses": 285,            "response_bytes": 23425,            "in_queue": 0,            "in_queue_bytes": 0,            "out_queue": 0,            "out_queue_bytes": 0        },        "192.168.100.69:6379": {            "server_eof": 1,            "server_err": 11,            "server_timedout": 0,            "server_connections": 1,            "server_ejected_at": 1427074216550647,            "requests": 54,            "request_bytes": 2580,            "responses": 43,            "response_bytes": 215,            "in_queue": 0,            "in_queue_bytes": 0,            "out_queue": 0,            "out_queue_bytes": 0        },        "192.168.100.69:6380": {            "server_eof": 1,            "server_err": 9,            "server_timedout": 0,            "server_connections": 1,            "server_ejected_at": 1427040015877803,            "requests": 291,            "request_bytes": 15563,            "responses": 282,            "response_bytes": 1402,            "in_queue": 0,            "in_queue_bytes": 0,            "out_queue": 0,            "out_queue_bytes": 0        },        "192.168.100.69:6381": {            "server_eof": 0,            "server_err": 11,            "server_timedout": 0,            "server_connections": 1,            "server_ejected_at": 1427059816411041,            "requests": 193,            "request_bytes": 17292,            "responses": 182,            "response_bytes": 1767,            "in_queue": 0,            "in_queue_bytes": 0,            "out_queue": 0,            "out_queue_bytes": 0        }    }}

由此可以知道zabbix需要监控service,total_connections几个规定字段。同时由于配置文件中各个pool是变更的，不固定的，各个pool下面的redis主机也是变更的，不固定的。所以，需要使用zabbix低级发现功能来发现由pool和redis主机构成的键值对。例如通过zabbix来发现

"{#REDIS_SERVER}":"gintamaXXXX192.168.100.69:6381"

这样的键值对。然后再根据键值对获取各个字段的值。

各个字段的含义可以使用nutcracker -D获取

pool stats:  client_eof          "# eof on client connections"  client_err          "# errors on client connections"  client_connections  "# active client connections"  server_ejects       "# times backend server was ejected"  forward_error       "# times we encountered a forwarding error"  fragments           "# fragments created from a multi-vector request"server stats:  server_eof          "# eof on server connections"  server_err          "# errors on server connections"  server_timedout     "# timeouts on server connections"  server_connections  "# active server connections"  server_ejected_at   "timestamp when server was ejected in usec since epoch"  requests            "# requests"  request_bytes       "total request bytes"  responses           "# responses"  response_bytes      "total response bytes"  in_queue            "# requests in incoming queue"  in_queue_bytes      "current request bytes in incoming queue"  out_queue           "# requests in outgoing queue"  out_queue_bytes     "current request bytes in outgoing queue"

二编写twemproxy的pool和redis主机发现脚本

twemproxy_pools_discovery.py

#/usr/bin/pythonimport yamlimport jsonconfig_file='/data/app_platform/twemproxy/conf/nutcracker.yml'pools=[]with open(config_file,'r') as f:  data=yaml.load(f)  for pool in  sorted(data.keys()):      pools.append({'{#REDIS_POOL}':pool})print json.dumps({'data':pools},indent=4,separators=(',',':'))

执行情况

$ python twemproxy_pools_discovery.py {    "data":[        {            "{#REDIS_POOL}":"gintama"        }    ]}

twemproxy_pools_servers_discovery.py

#/usr/bin/pythonimport yamlimport jsonconfig_file='/tmp/nutcracker.yml'pools_servers=[]with open(config_file,'r') as f:  data=yaml.load(f)  for pool in  sorted(data.keys()):      server_lists=data[pool]['servers']      for server in server_lists:           lists=(pool + 'XXXX' + server).split(':')           pools_servers.append({'{#REDIS_SERVERS}':(lists[0] + ':' + lists[1])})print json.dumps({'data':pools_servers},indent=4,separators=(',',':'))

在编写脚本的过程中需要边编写边调试才能达到自己想要的效果

执行情况如下：

$ python twemproxy_pools_servers_discovery.py {    "data":[        {            "{#REDIS_SERVER}":"gintamaXXXX192.168.100.68:6379"        },        {            "{#REDIS_SERVER}":"gintamaXXXX192.168.100.68:6380"        },        {            "{#REDIS_SERVER}":"gintamaXXXX192.168.100.68:6381"        },        {            "{#REDIS_SERVER}":"gintamaXXXX192.168.100.69:6379"        },        {            "{#REDIS_SERVER}":"gintamaXXXX192.168.100.69:6380"        },        {            "{#REDIS_SERVER}":"gintamaXXXX192.168.100.69:6381"        }    ]}

三编写twemproxy状态信息获取脚本

twemproxy_status.py

这个脚本需要安装argparse模块

Python2.6使用pip install argparse安装

Python2.7，argparse模块已经设为默认模块

#! /usr/bin/env pythonimport socketimport jsonimport argparse#{#    "service": "nutcracker",#    "source": "gintama-taiwan-redis1",#    "version": "0.4.0",#    "uptime": 136873,#    "timestamp": 1427168759,#    "total_connections": 489,#    "curr_connections": 7,#    "gintama": {#        "client_eof": 0,#        "client_err": 420,#        "client_connections": 0,#        "server_ejects": 62,#        "forward_error": 57,#        "fragments": 0,#        "192.168.100.68:6379": {#            "server_eof": 1,#            "server_err": 10,#            "server_timedout": 0,#            "server_connections": 1,#            "server_ejected_at": 1427074216548518,#            "requests": 627,#            "request_bytes": 329980,#            "responses": 617,#            "response_bytes": 280709,#            "in_queue": 0,#            "in_queue_bytes": 0,#            "out_queue": 0,#            "out_queue_bytes": 0#        },####"{#REDIS_SERVER}":"gintama@192.168.100.69:6381"##"{#REDIS_POOL}":"gintama"class NutcrackerServer(object):    def __init__(self):         self.server = '127.0.0.1'        self.port = '22222'    def nutcracker_status(self):                conn = socket.create_connection((self.server, self.port))        buf = True        content = ''        while buf:            buf = conn.recv(4096)            content += buf        conn.close()        self.data = json.loads(content)    def nutcracker_info(self,base_metric):        self.nutcracker_status()        print self.data[base_metric]    def nutcracker_pool_info(self,pool,pool_metric):        self.nutcracker_status()        print self.data[pool][pool_metric]    def nutcracker_server_info(self,pool,redis_server,server_metric):        self.nutcracker_status()        print self.data[pool][redis_server][server_metric]def parse_args():    parser=argparse.ArgumentParser(description='Twemproxy monitoring tool with Zabbix!', argument_default=False)    parser.add_argument('--metric',default='service',dest='metric',action='store',required=True,help='the twemproxy metric,such as uptime or version,etc')    parser.add_argument("--poolname",default='gintama',dest='poolname',action='store',required=False,help='the twemproxy pool and server discoveried by zabbix')    parser.add_argument("--poolserver",default='gintamaXXXX192.168.100.69:6380',dest='poolserver',action='store',required=False,help='the twemproxy pool and server discoveried by zabbix')    args=parser.parse_args()    return argsdef main():    args=parse_args()    metric=''    pool=''    redis_server=''    if args.metric:              metric=args.metric    else:       print "invalid metric"     if args.poolname:       pool=args.poolname    else:       print "invalid pool name"    if args.poolserver:       pool=args.poolserver.split('XXXXX')[0]       redis_server=args.poolserver.split('XXXX')[1]    else:       print "invalid pool server"    twemproxy = NutcrackerServer()        if not metric:           print "invalid metric"    elif metric in ["service","source","version","uptime","timestamp","total_connections","curr_connections"]:#       print "nutcracker base information"       twemproxy.nutcracker_info(metric)    elif pool!="" and metric in ["client_eof","client_err","client_connections","server_ejects","forward_error","fragments"]:#       print "nutcracker" + " " + pool + " " + "information"       twemproxy.nutcracker_pool_info(pool,metric)    elif pool and redis_server and metric in ["server_eof","server_err","server_timedout","server_connections","server_ejected_at","requests","request_bytes","responses","response_bytes","in_queue","in_queue_bytes","out_queue","out_queue_bytes"]:#       print "nutcracker" + " " + pool + " " + redis_server + " " + "information"       twemproxy.nutcracker_server_info(pool,redis_server,metric)    else:       print "invalid metric" if __name__ == '__main__':     main()

四添加zabbix子配置文件

twemproxy_status.conf

### Option: UserParameter#       User-defined parameter to monitor. There can be several user-defined parameters.#       Format: UserParameter=
     
      ,
      
       #       See 'zabbix_agentd' directory for examples.## Mandatory: no# Default:# UserParameter=UserParameter=redis_pool.discovery,/usr/bin/python /usr/local/zabbix/bin/twemproxy_pools_discovery.pyUserParameter=redis_server.discovery,/usr/bin/python /usr/local/zabbix/bin/twemproxy_pools_servers_discovery.pyUserParameter=nutcracker_info[*],/usr/bin/python /usr/local/zabbix/bin/twemproxy_status.py --metric $1UserParameter=nutcracker_pool_info[*],/usr/bin/python /usr/local/zabbix/bin/twemproxy_status.py --metric $1 --poolname $2UserParameter=nutcracker_server_info[*],/usr/bin/python /usr/local/zabbix/bin/twemproxy_status.py --metric $1 --poolserver $2

五添加zabbix模板

参见附件

参考文档：

转载于:https://blog.51cto.com/john88wang/1623309

你可能感兴趣的文章