OpenResty(Nginx Lua)统计网站访问信息

2018-03-31 00:27:14  阅读 66 次 评论 0 条

之前的一篇文章openresty(nginx lua)统计域名状态码、平均响应时间和流量实现了对域名状态码,平均响应时间和流量的统计。但之前的统计方法没有实现当某一域名404或500等状态码超过一定数量后发送具体的url来快速定位位置。这个功能我们其实是通过统计网站日志来实现了。为了摆脱对网站日志的依赖以及提高统计性能,我们尝试把此功能也用nginx lua来实现。具体的使用方法与之前的文章一样,这里只是更新了两个lua脚本。

使用方法


1、获取域名www.centos.bz 404状态码数量

  1. curl -s "localhost/domain_status?count=status&host=www.centos.bz&status=404"

输出:
10 688
第一列为状态码数量,第二列为域名请求总数
2、获取当域名www.centos.bz 404状态码超过50个时,输出前10个url

  1. curl -s "localhost/domain_status?count=statusUrl&host=www.centos.bz&status=404&exceed=50&output=10"

输出:
/hello-world 90
/centos 10

第一列为url,第二列为url请求次数。
3、获取域名www.centos.bz upstream一分钟内平均耗时

  1. curl -s "localhost/domain_status?count=upT&host=www.centos.bz"

输出:
0.02 452
第一列为upstream平均耗时,第二列为域名总请求次数。
4、获取当域名www.centos.bz upstream平均耗时超过0.5秒时,输出其url

  1. curl -s "localhost/domain_status?count=upTUrl&host=www.centos.bz&exceed=0.5"

输出:
/hello.php 0.82 52
第一列为url,第二列为此url平均耗时,第三列为此url请求次数。监控此接口数据可以快速定位出具体哪些url慢了。
5、获取域名www.centos.bz request time平均耗时

  1. curl -s "localhost/domain_status?count=reqT&host=www.centos.bz"

输出:
1.82 52
第一列为平均耗时,第二列为域名请求数。request time是指完成整个请求所需要的时间(包括把数据传输到用户浏览器的时间)。对于php请求,upstream time指的是nginx把php请求传给fastcgi到完成数据接收所需时间。所以request time永远大于upstream time。
6、获取域名www.centos.bz占用的带宽(单位:字节/秒)

  1. curl -s "localhost/domain_status?count=flow&host=www.centos.bz"

输出:
1024 52
第一列为此域名一分钟内平均传输速率,单位为字节/秒,第二列为域名请求总数。

相关脚本

log_acesss.lua

  1. local access = ngx.shared.access

  2. local host = ngx.var.host or "unknow"

  3. local status = ngx.var.status

  4. local body_bytes_sent = ngx.var.body_bytes_sent

  5. local request_time = ngx.var.request_time

  6. local upstream_response_time = ngx.var.upstream_response_time or 0

  7. local request_uri = ngx.var.request_uri or "/unknow"

  8. local timestamp = ngx.time()

  9. local expire_time = 70

  10.  

  11. local status_key = table.concat({host,"-",status,"-",timestamp})

  12. local flow_key = table.concat({host,"-flow-",timestamp})

  13. local req_time_key = table.concat({host,"-reqt-",timestamp})

  14. local up_time_key = table.concat({host,"-upt-",timestamp})

  15. local total_key = table.concat({host,"-total-",timestamp})

  16.  

  17. -- 域名总请求数

  18. local n,e = access:incr(total_key,1)

  19. if not n then

  20. access:set(total_key, 1, expire_time)

  21. end

  22.  

  23. -- 域名状态码请求数

  24. local n,e = access:incr(status_key,1)

  25. if not n then

  26. access:set(status_key, 1, expire_time)

  27. end

  28.  

  29. -- 域名流量

  30. local n,e = access:incr(flow_key,body_bytes_sent)

  31. if not n then

  32. access:set(flow_key, body_bytes_sent, expire_time)

  33. end

  34.  

  35. -- 域名请求耗时

  36. local n,e = access:incr(req_time_key,request_time)

  37. if not n then

  38. access:set(req_time_key, request_time, expire_time)

  39. end

  40.  

  41. -- 域名upstream耗时

  42. local n,e = access:incr(up_time_key,upstream_response_time)

  43. if not n then

  44. access:set(up_time_key, upstream_response_time, expire_time)

  45. end

  46.  

  47. -- 获取不带参数的uri

  48. local m, err = ngx.re.match(request_uri, "(.*?)\\?")

  49. local request_without_args = m and m[1] or request_uri

  50.  

  51. -- 存储状态码大于400的url

  52. if tonumber(status) >= 400 then

  53. -- 拼接url,状态码,字节数等字段

  54. local request_log_t = {}

  55. table.insert(request_log_t,host)

  56. table.insert(request_log_t,request_without_args)

  57. table.insert(request_log_t,status)

  58. local request_log = table.concat(request_log_t," ")

  59.  

  60. -- 把拼接的字段储存在字典中

  61. local log_key = table.concat({"status-",timestamp})

  62. local request_log_dict = access:get(log_key) or ""

  63. if request_log_dict == "" then

  64. request_log_dict = request_log

  65. else

  66. request_log_dict = table.concat({request_log_dict,"\n",request_log})

  67. end

  68. access:set(log_key, request_log_dict, expire_time)

  69. end

  70.  

  71. -- 存储upstream time大于0.5的url

  72. if tonumber(upstream_response_time) > 0.5 then

  73. -- 拼接url,状态码,字节数等字段

  74. local request_log_t = {}

  75. table.insert(request_log_t,host)

  76. table.insert(request_log_t,request_without_args)

  77. table.insert(request_log_t,upstream_response_time)

  78. local request_log = table.concat(request_log_t," ")

  79.  

  80. -- 把拼接的字段储存在字典中

  81. local log_key = table.concat({"upt-",timestamp})

  82. local request_log_dict = access:get(log_key) or ""

  83. if request_log_dict == "" then

  84. request_log_dict = request_log

  85. else

  86. request_log_dict = table.concat({request_log_dict,"\n",request_log})

  87. end

  88. access:set(log_key, request_log_dict, expire_time)

  89. end

domain_status.lua

  1. -- 各参数用法:

  2. -- count=status,host=xxx.com,status=404 统计xxx.com域名一分钟内404状态码个数.

  3. -- count=statusUrl,host=xxx.com,status=404,exceed=50,output=30 当xxx.com域名404状态码一分钟内超过50个时,输出前30个url,否则返回空.

  4.  

  5. -- count=upT,host=xxx.com 统计xxx.com域名一分钟内平均upsteam耗时

  6. -- count=upTUrl,host=xxx.com,exceed=0.5 输出upstreamTime超过0.5秒的url,没有就返回空

  7.  

  8. -- count=reqT,host=xxx.com 统计xxx.com域名一分钟内平均请求耗时

  9. -- count=flow,host=xxx.com 统计xxx.com域名一分钟内流量(单位字节/秒)

  10.  

  11. -- 函数: 获取迭代器值

  12. local get_field = function(iterator)

  13.     local m,err = iterator

  14.     if err then

  15.         ngx.log(ngx.ERR, "get_field iterator error: ", err)

  16.         ngx.exit(ngx.HTTP_OK)

  17.     end

  18.     return m[0]

  19. end

  20.  

  21. -- 函数: 按值排序table

  22. local getKeysSortedByValue = function (tbl, sortFunction)

  23.   local keys = {}

  24.   for key in pairs(tbl) do

  25.     table.insert(keys, key)

  26.   end

  27.  

  28.   table.sort(keys, function(a, b)

  29.     return sortFunction(tbl[a], tbl[b])

  30.   end)

  31.  

  32.   return keys

  33. end

  34.  

  35. -- 函数: 判断table是否存在某元素

  36. local tbl_contain = function(table,element)

  37.     for k in pairs(table) do

  38.         if k == element then

  39.             return true

  40.         end

  41.     end

  42.     return false

  43. end

  44.  

  45. local access = ngx.shared.access

  46. local now = ngx.time()

  47. local one_minute_ago = now - 60

  48.  

  49. -- 获取参数

  50. local args = ngx.req.get_uri_args()

  51. local count_arg = args["count"]

  52. local host_arg = args["host"]

  53. local status_arg = args["status"]

  54. local exceed_arg = args["exceed"]

  55. local output_arg = args["output"]

  56. local count_t = {["status"]=0,["statusUrl"]=0,["upT"]=0,["upTUrl"]=0,["reqT"]=0,["flow"]=0}

  57.  

  58. -- 检查参数是否满足

  59. if not tbl_contain(count_t,count_arg) then

  60.     ngx.print("count arg invalid.")

  61.     ngx.exit(ngx.HTTP_OK)

  62. end

  63.  

  64. if not host_arg then

  65.     ngx.print("host arg not found.")

  66.     ngx.exit(ngx.HTTP_OK)

  67. end

  68.  

  69. if count_arg == "status" and not status_arg then

  70.     ngx.print("status arg not found.")

  71.     ngx.exit(ngx.HTTP_OK)

  72. end

  73.  

  74. if count_arg == "statusUrl" and not (status_arg and exceed_arg and output_arg)  then

  75.     ngx.print("status or exceed or output arg not found.")

  76.     ngx.exit(ngx.HTTP_OK)

  77. end

  78.  

  79. if count_arg == "upTUrl" and not exceed_arg then

  80.     ngx.print("exceed arg not found.")

  81.     ngx.exit(ngx.HTTP_OK)

  82. end

  83.  

  84. -- 检查参数是否合法

  85. if status_arg and ngx.re.find(status_arg, "^[0-9]{3}$") == nil then

  86.     ngx.print("status arg must be a valid httpd code.")

  87.     ngx.exit(ngx.HTTP_OK)

  88. end

  89.  

  90. if exceed_arg and ngx.re.find(exceed_arg, "^[0-9.]+$") == nil then

  91.     ngx.print("exceed arg must be a number.")

  92.     ngx.exit(ngx.HTTP_OK)

  93. end

  94.  

  95. if output_arg and ngx.re.find(output_arg, "^[0-9]+$") == nil then

  96.     ngx.print("output arg must be a number.")

  97.     ngx.exit(ngx.HTTP_OK)

  98. end

  99.  

  100. -- 开始统计

  101. local url

  102. local status_code

  103. local upstream_time

  104. local status_total = 0

  105. local host

  106. local req_total = 0

  107. local flow_total = 0

  108. local reqtime_total = 0

  109. local upstream_total = 0

  110. local status_url_t = {}

  111. local upstream_url_t = {}

  112. local upstream_url_count_t = {}

  113.  

  114. local status_log

  115. local upt_log

  116.  

  117. for second_num=one_minute_ago,now do

  118.     local flow_key = table.concat({host_arg,"-flow-",second_num})

  119.     local req_time_key = table.concat({host_arg,"-reqt-",second_num})

  120.     local up_time_key = table.concat({host_arg,"-upt-",second_num}) 

  121.     local total_req_key = table.concat({host_arg,"-total-",second_num})

  122.     local log_key

  123.     local log_line

  124.  

  125.     -- 合并状态码大于等于400的请求日志到变量status_log

  126.     log_key = table.concat({"status-",second_num})

  127.     log_line = access:get(log_key) or ""

  128.     if not (log_line == "") then

  129.         status_log = table.concat({log_line,"\n",status_log})

  130.     end

  131.  

  132.     -- 合并upstream time大于0.5秒的请求日志到变量upt_log

  133.     log_key = table.concat({"upt-",second_num})

  134.     log_line = access:get(log_key) or ""

  135.     if not (log_line == "") then

  136.         upt_log = table.concat({log_line,"\n",upt_log})

  137.     end

  138.  

  139.     -- 域名总请求数

  140.     local req_sum = access:get(total_req_key) or 0

  141.     req_total = req_total + req_sum

  142.  

  143.     if count_arg == "status" or count_arg == "statusUrl" then

  144.         local status_key = table.concat({host_arg,"-",status_arg,"-",second_num})

  145.         local status_sum = access:get(status_key) or 0

  146.         status_total = status_total + status_sum

  147.     end

  148.  

  149.     if count_arg == "flow" then

  150.         local flow_sum = access:get(flow_key) or 0

  151.         flow_total = flow_total + flow_sum

  152.     end

  153.  

  154.     if count_arg == "reqT" then

  155.         local req_time_sum = access:get(req_time_key) or 0

  156.         reqtime_total = reqtime_total + req_time_sum

  157.     end

  158.  

  159.     if count_arg == "upT" then

  160.         local up_time_sum = access:get(up_time_key) or 0

  161.         upstream_total = upstream_total + up_time_sum

  162.     end

  163. end

  164.  

  165. -- 统计状态码url

  166. if count_arg == "statusUrl" and status_log and not (status_log == "") then

  167.     local iterator, err = ngx.re.gmatch(status_log,".+\n")

  168.     if not iterator then

  169.         ngx.log(ngx.ERR, "status_log iterator error: ", err)

  170.         return

  171.     end

  172.     for line in iterator do

  173.         if not line[0] then

  174.             ngx.log(ngx.ERR, "line[0] is nil")

  175.             return

  176.         end

  177.         local iterator, err = ngx.re.gmatch(line[0],"[^ \n]+")

  178.         if not iterator then

  179.             ngx.log(ngx.ERR, "line[0] iterator error: ", err)

  180.             return

  181.         end

  182.  

  183.         host = get_field(iterator())

  184.         url = get_field(iterator())

  185.         status_code = get_field(iterator())

  186.  

  187.         if status_code == status_arg then

  188.             if status_url_t[url] then

  189.                 status_url_t[url] = status_url_t[url] + 1

  190.             else

  191.                 status_url_t[url] = 1

  192.             end

  193.         end

  194.  

  195.     end   

  196. end

  197.  

  198. -- 统计upstream time大于0.5秒url

  199. if count_arg == "upTUrl" and upt_log and not (upt_log == "") then

  200.     local iterator, err = ngx.re.gmatch(upt_log,".+\n")

  201.     if not iterator then

  202.         ngx.log(ngx.ERR, "upt_log iterator error: ", err)

  203.         return

  204.     end

  205.     for line in iterator do

  206.         if not line[0] then

  207.             ngx.log(ngx.ERR, "line[0] is nil")

  208.             return

  209.         end

  210.         local iterator, err = ngx.re.gmatch(line[0],"[^ \n]+")

  211.         if not iterator then

  212.             ngx.log(ngx.ERR, "line[0] iterator error: ", err)

  213.             return

  214.         end

  215.  

  216.         host = get_field(iterator())

  217.         url = get_field(iterator())

  218.         upstream_time = get_field(iterator())

  219.         upstream_time = tonumber(upstream_time) or 0

  220.  

  221.         -- 统计各url upstream平均耗时

  222.         if host == host_arg then

  223.             if upstream_url_t[url] then

  224.                 upstream_url_t[url] = upstream_url_t[url] + upstream_time

  225.             else

  226.                 upstream_url_t[url] = upstream_time

  227.             end

  228.  

  229.             if upstream_url_count_t[url] then

  230.                 upstream_url_count_t[url] = upstream_url_count_t[url] + 1

  231.             else

  232.                 upstream_url_count_t[url] = 1

  233.             end

  234.         end    

  235.     end   

  236. end

  237.  

  238. -- 输出结果

  239. if count_arg == "status" then

  240.     ngx.print(status_total," ",req_total)

  241.  

  242. elseif count_arg == "flow" then

  243.     ngx.print(flow_total," ",req_total)

  244.  

  245. elseif count_arg == "reqT" then

  246.     local reqt_avg = 0

  247.     if req_total == 0 then

  248.         reqt_avg = 0

  249.     else

  250.         reqt_avg = reqtime_total/req_total

  251.     end

  252.     ngx.print(reqt_avg," ",req_total)

  253.  

  254. elseif count_arg == "upT" then

  255.     local upt_avg = 0

  256.     if req_total == 0 then

  257.             upt_avg = 0

  258.     else

  259.             upt_avg = upstream_total/req_total

  260.     end

  261.     ngx.print(upt_avg," ",req_total)

  262.  

  263. elseif count_arg == "statusUrl" then

  264.     if status_total > tonumber(exceed_arg) then

  265.         -- 排序table

  266.         status_url_t_key = getKeysSortedByValue(status_url_t, function(a, b) return a > b end)

  267.         local output_body = ""

  268.         for i, uri in ipairs(status_url_t_key) do

  269.             if output_body == "" then

  270.                 output_body = table.concat({uri," ",status_url_t[uri]})

  271.             else    

  272.                 output_body = table.concat({output_body,"\n",uri," ",status_url_t[uri]})

  273.             end

  274.             if i >= tonumber(output_arg) then

  275.                 ngx.print(output_body)

  276.                 ngx.exit(ngx.HTTP_OK)

  277.             end                

  278.         end

  279.  

  280.         ngx.print(output_body)

  281.         ngx.exit(ngx.HTTP_OK)

  282.     end

  283.  

  284. elseif count_arg == "upTUrl" then

  285.     local max_output = 30

  286.     local total_time = 0

  287.     local total_count = 0

  288.     local output_body = ""

  289.     local i = 0

  290.     for url in pairs(upstream_url_t) do

  291.         i = i + 1

  292.         total_time = upstream_url_t[url]

  293.         total_count = upstream_url_count_t[url]

  294.         avg_time = upstream_url_t[url] / upstream_url_count_t[url]

  295.         if avg_time > tonumber(exceed_arg) then

  296.             output_body = table.concat({url," ",avg_time," ",total_count,"\n",output_body})

  297.         end

  298.  

  299.         if i >= max_output then

  300.             ngx.print(output_body)

  301.             ngx.exit(ngx.HTTP_OK)

  302.         end            

  303.     end

  304.     ngx.print(output_body)

  305.     ngx.exit(ngx.HTTP_OK)

  306.  

  307. end


本文地址:http://blog.ailinux.net/post/57.html
版权声明:本文为原创文章,版权归 admin 所有,欢迎分享本文,转载请保留出处!

发表评论


表情

还没有留言,还不快点抢沙发?