From c93955bdac45c5862d1f0faf4a6ba863986c6c68 Mon Sep 17 00:00:00 2001 From: Bryce Walter Date: Tue, 15 Nov 2016 12:56:23 -0700 Subject: [PATCH 1/8] Fix up Dockerfile; add Prometheus Module w/ port --- mainline/alpine/Dockerfile | 261 +++++++++++++++++++------------------ 1 file changed, 133 insertions(+), 128 deletions(-) diff --git a/mainline/alpine/Dockerfile b/mainline/alpine/Dockerfile index 4b5f78ce..e612429a 100644 --- a/mainline/alpine/Dockerfile +++ b/mainline/alpine/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:3.4 -MAINTAINER NGINX Docker Maintainers "docker-maint@nginx.com", Dmitrii Zolotov +MAINTAINER NGINX Docker Maintainers "docker-maint@nginx.com", Dmitrii Zolotov , Bryce (Krylon360) Walter "b.t.walter@gmail.com" ENV NGINX_VERSION 1.11.5 @@ -13,143 +13,148 @@ ENV LUA_NGINX_MODULE_VERSION 0.10.6 ENV LUAJIT_LIB=/usr/local/lib/lua/5.1 ENV LUAJIT_INC=/usr/local/include/luajit-2.0 -#Original nginx build (with modules ngx_devel_kit, lua-nginx-module, nginx-auth-ldap) +#Original nginx build (with modules ngx_devel_kit, lua-nginx-module, nginx-auth-ldap, nginx-lua-vstat, nginx-lua-prometheus, nginx_stub_status_prometheus) RUN apk update && apk add git && mkdir -p /opt && cd /opt && git clone https://github.com/simpl/ngx_devel_kit && cd ngx_devel_kit && git checkout -b v$NGX_DEVEL_KIT_VERSION && \ apk add make g++ && cd /opt && git clone https://github.com/luajit/luajit && cd luajit && git checkout -b v$LUAJIT_VERSION && make && make install && cd .. && \ - cd /opt && git clone https://github.com/openresty/lua-nginx-module && cd lua-nginx-module && git checkout -b v$LUA_NGINX_MODULE_VERSION && \ - cd /opt && git clone https://github.com/kvspb/nginx-auth-ldap.git && apk del git && \ + cd /opt && git clone https://github.com/openresty/lua-nginx-module && cd lua-nginx-module && git checkout -b v$LUA_NGINX_MODULE_VERSION && cd .. \ + cd /opt && git clone https://github.com/kvspb/nginx-auth-ldap.git && cd .. && \ + cd /opt && git clone https://github.com/knyar/nginx-lua-prometheus.git && cd .. && \ + cd /opt && git clone git://github.com/vozlt/nginx-module-vts.git && cd .. && \ + apk del git && \ GPG_KEYS=B0F4253373F8F6F510D42178520A9993A1C052F8 \ - && CONFIG="\ - --prefix=/etc/nginx \ - --sbin-path=/usr/sbin/nginx \ - --modules-path=/usr/lib/nginx/modules \ - --conf-path=/etc/nginx/nginx.conf \ - --error-log-path=/var/log/nginx/error.log \ - --http-log-path=/var/log/nginx/access.log \ - --pid-path=/var/run/nginx.pid \ - --lock-path=/var/run/nginx.lock \ - --http-client-body-temp-path=/var/cache/nginx/client_temp \ - --http-proxy-temp-path=/var/cache/nginx/proxy_temp \ - --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp \ - --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp \ - --http-scgi-temp-path=/var/cache/nginx/scgi_temp \ - --user=nginx \ - --group=nginx \ + && CONFIG="\ + --prefix=/etc/nginx \ + --sbin-path=/usr/sbin/nginx \ + --modules-path=/usr/lib/nginx/modules \ + --conf-path=/etc/nginx/nginx.conf \ + --error-log-path=/var/log/nginx/error.log \ + --http-log-path=/var/log/nginx/access.log \ + --pid-path=/var/run/nginx.pid \ + --lock-path=/var/run/nginx.lock \ + --http-client-body-temp-path=/var/cache/nginx/client_temp \ + --http-proxy-temp-path=/var/cache/nginx/proxy_temp \ + --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp \ + --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp \ + --http-scgi-temp-path=/var/cache/nginx/scgi_temp \ + --user=nginx \ + --group=nginx \ --with-ld-opt="-Wl,-rpath,/usr/local/lib/lua/5.1" \ --add-module=/opt/ngx_devel_kit \ --add-module=/opt/lua-nginx-module \ --add-module=/opt/nginx-auth-ldap \ - --with-http_ssl_module \ - --with-http_realip_module \ - --with-http_addition_module \ - --with-http_sub_module \ - --with-http_dav_module \ - --with-http_flv_module \ - --with-http_mp4_module \ - --with-http_gunzip_module \ - --with-http_gzip_static_module \ - --with-http_random_index_module \ - --with-http_secure_link_module \ - --with-http_stub_status_module \ - --with-http_auth_request_module \ - --with-http_xslt_module=dynamic \ - --with-http_image_filter_module=dynamic \ - --with-http_geoip_module=dynamic \ - --with-http_perl_module=dynamic \ - --with-threads \ - --with-stream \ - --with-stream_ssl_module \ - --with-stream_ssl_preread_module \ - --with-stream_realip_module \ - --with-stream_geoip_module=dynamic \ - --with-http_slice_module \ - --with-mail \ - --with-mail_ssl_module \ - --with-compat \ - --with-file-aio \ - --with-http_v2_module \ - " \ - && addgroup -S nginx \ - && adduser -D -S -h /var/cache/nginx -s /sbin/nologin -G nginx nginx \ - && apk add --no-cache --virtual .build-deps \ - gcc \ - libc-dev \ - make \ - openssl-dev \ - pcre-dev \ - zlib-dev \ - linux-headers \ - curl \ - gnupg \ - libxslt-dev \ - gd-dev \ - geoip-dev \ - perl-dev \ - openldap-dev \ - && curl -fSL http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz -o nginx.tar.gz \ - && curl -fSL http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz.asc -o nginx.tar.gz.asc \ - && export GNUPGHOME="$(mktemp -d)" \ - && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEYS" \ - && gpg --batch --verify nginx.tar.gz.asc nginx.tar.gz \ - && rm -r "$GNUPGHOME" nginx.tar.gz.asc \ - && mkdir -p /usr/src \ - && tar -zxC /usr/src -f nginx.tar.gz \ - && rm nginx.tar.gz \ - && cd /usr/src/nginx-$NGINX_VERSION \ - && ./configure $CONFIG --with-debug \ - && make -j$(getconf _NPROCESSORS_ONLN) \ - && mv objs/nginx objs/nginx-debug \ - && mv objs/ngx_http_xslt_filter_module.so objs/ngx_http_xslt_filter_module-debug.so \ - && mv objs/ngx_http_image_filter_module.so objs/ngx_http_image_filter_module-debug.so \ - && mv objs/ngx_http_geoip_module.so objs/ngx_http_geoip_module-debug.so \ - && mv objs/ngx_http_perl_module.so objs/ngx_http_perl_module-debug.so \ - && mv objs/ngx_stream_geoip_module.so objs/ngx_stream_geoip_module-debug.so \ - && ./configure $CONFIG \ - && make -j$(getconf _NPROCESSORS_ONLN) \ - && make install \ - && rm -rf /etc/nginx/html/ \ - && mkdir /etc/nginx/conf.d/ \ - && mkdir -p /usr/share/nginx/html/ \ - && install -m644 html/index.html /usr/share/nginx/html/ \ - && install -m644 html/50x.html /usr/share/nginx/html/ \ - && install -m755 objs/nginx-debug /usr/sbin/nginx-debug \ - && install -m755 objs/ngx_http_xslt_filter_module-debug.so /usr/lib/nginx/modules/ngx_http_xslt_filter_module-debug.so \ - && install -m755 objs/ngx_http_image_filter_module-debug.so /usr/lib/nginx/modules/ngx_http_image_filter_module-debug.so \ - && install -m755 objs/ngx_http_geoip_module-debug.so /usr/lib/nginx/modules/ngx_http_geoip_module-debug.so \ - && install -m755 objs/ngx_http_perl_module-debug.so /usr/lib/nginx/modules/ngx_http_perl_module-debug.so \ - && install -m755 objs/ngx_stream_geoip_module-debug.so /usr/lib/nginx/modules/ngx_stream_geoip_module-debug.so \ - && ln -s ../../usr/lib/nginx/modules /etc/nginx/modules \ - && strip /usr/sbin/nginx* \ - && strip /usr/lib/nginx/modules/*.so \ - && rm -rf /usr/src/nginx-$NGINX_VERSION \ - && rm -rf /opt \ - \ - # Bring in gettext so we can get `envsubst`, then throw - # the rest away. To do this, we need to install `gettext` - # then move `envsubst` out of the way so `gettext` can - # be deleted completely, then move `envsubst` back. - && apk add --no-cache --virtual .gettext gettext \ - && mv /usr/bin/envsubst /tmp/ \ - \ - && runDeps="$( \ - scanelf --needed --nobanner /usr/sbin/nginx /usr/lib/nginx/modules/*.so /tmp/envsubst \ - | awk '{ gsub(/,/, "\nso:", $2); print "so:" $2 }' \ - | sort -u \ - | xargs -r apk info --installed \ - | sort -u \ - )" \ - && apk add --no-cache --virtual .nginx-rundeps $runDeps \ - && apk del .build-deps \ - && apk del .gettext \ - && mv /tmp/envsubst /usr/local/bin/ \ - \ - # forward request and error logs to docker log collector - && ln -sf /dev/stdout /var/log/nginx/access.log \ - && ln -sf /dev/stderr /var/log/nginx/error.log + --add-module=/opt/nginx-module-vts \ + --with-http_ssl_module \ + --with-http_realip_module \ + --with-http_addition_module \ + --with-http_sub_module \ + --with-http_dav_module \ + --with-http_flv_module \ + --with-http_mp4_module \ + --with-http_gunzip_module \ + --with-http_gzip_static_module \ + --with-http_random_index_module \ + --with-http_secure_link_module \ + --with-http_stub_status_module \ + --with-http_auth_request_module \ + --with-http_xslt_module=dynamic \ + --with-http_image_filter_module=dynamic \ + --with-http_geoip_module=dynamic \ + --with-http_perl_module=dynamic \ + --with-threads \ + --with-stream \ + --with-stream_ssl_module \ + --with-stream_ssl_preread_module \ + --with-stream_realip_module \ + --with-stream_geoip_module=dynamic \ + --with-http_slice_module \ + --with-mail \ + --with-mail_ssl_module \ + --with-compat \ + --with-file-aio \ + --with-http_v2_module \ + " \ + && addgroup -S nginx \ + && adduser -D -S -h /var/cache/nginx -s /sbin/nologin -G nginx nginx \ + && apk add --no-cache --virtual .build-deps \ + gcc \ + libc-dev \ + make \ + openssl-dev \ + pcre-dev \ + zlib-dev \ + linux-headers \ + curl \ + gnupg \ + libxslt-dev \ + gd-dev \ + geoip-dev \ + perl-dev \ + openldap-dev \ + && curl -fSL http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz -o nginx.tar.gz \ + && curl -fSL http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz.asc -o nginx.tar.gz.asc \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEYS" \ + && gpg --batch --verify nginx.tar.gz.asc nginx.tar.gz \ + && rm -r "$GNUPGHOME" nginx.tar.gz.asc \ + && mkdir -p /usr/src \ + && tar -zxC /usr/src -f nginx.tar.gz \ + && rm nginx.tar.gz \ + && cd /usr/src/nginx-$NGINX_VERSION \ + && ./configure $CONFIG --with-debug \ + && make -j$(getconf _NPROCESSORS_ONLN) \ + && mv objs/nginx objs/nginx-debug \ + && mv objs/ngx_http_xslt_filter_module.so objs/ngx_http_xslt_filter_module-debug.so \ + && mv objs/ngx_http_image_filter_module.so objs/ngx_http_image_filter_module-debug.so \ + && mv objs/ngx_http_geoip_module.so objs/ngx_http_geoip_module-debug.so \ + && mv objs/ngx_http_perl_module.so objs/ngx_http_perl_module-debug.so \ + && mv objs/ngx_stream_geoip_module.so objs/ngx_stream_geoip_module-debug.so \ + && ./configure $CONFIG \ + && make -j$(getconf _NPROCESSORS_ONLN) \ + && make install \ + && rm -rf /etc/nginx/html/ \ + && mkdir /etc/nginx/conf.d/ \ + && mkdir -p /usr/share/nginx/html/ \ + && install -m644 html/index.html /usr/share/nginx/html/ \ + && install -m644 html/50x.html /usr/share/nginx/html/ \ + && install -m755 objs/nginx-debug /usr/sbin/nginx-debug \ + && install -m755 objs/ngx_http_xslt_filter_module-debug.so /usr/lib/nginx/modules/ngx_http_xslt_filter_module-debug.so \ + && install -m755 objs/ngx_http_image_filter_module-debug.so /usr/lib/nginx/modules/ngx_http_image_filter_module-debug.so \ + && install -m755 objs/ngx_http_geoip_module-debug.so /usr/lib/nginx/modules/ngx_http_geoip_module-debug.so \ + && install -m755 objs/ngx_http_perl_module-debug.so /usr/lib/nginx/modules/ngx_http_perl_module-debug.so \ + && install -m755 objs/ngx_stream_geoip_module-debug.so /usr/lib/nginx/modules/ngx_stream_geoip_module-debug.so \ + && install -m755 /opt/nginx-lua-prometheus/prometheus.lua /etc/nginx/modules/prometheus.lua \ + && ln -s ../../usr/lib/nginx/modules /etc/nginx/modules \ + && strip /usr/sbin/nginx* \ + && strip /usr/lib/nginx/modules/*.so \ + && rm -rf /usr/src/nginx-$NGINX_VERSION \ + && rm -rf /opt \ + \ + # Bring in gettext so we can get `envsubst`, then throw + # the rest away. To do this, we need to install `gettext` + # then move `envsubst` out of the way so `gettext` can + # be deleted completely, then move `envsubst` back. + && apk add --no-cache --virtual .gettext gettext \ + && mv /usr/bin/envsubst /tmp/ \ + \ + && runDeps="$( \ + scanelf --needed --nobanner /usr/sbin/nginx /usr/lib/nginx/modules/*.so /tmp/envsubst \ + | awk '{ gsub(/,/, "\nso:", $2); print "so:" $2 }' \ + | sort -u \ + | xargs -r apk info --installed \ + | sort -u \ + )" \ + && apk add --no-cache --virtual .nginx-rundeps $runDeps \ + && apk del .build-deps \ + && apk del .gettext \ + && mv /tmp/envsubst /usr/local/bin/ \ + \ + # forward request and error logs to docker log collector + && ln -sf /dev/stdout /var/log/nginx/access.log \ + && ln -sf /dev/stderr /var/log/nginx/error.log COPY nginx.conf /etc/nginx/nginx.conf COPY nginx.vh.default.conf /etc/nginx/conf.d/default.conf -EXPOSE 80 443 +EXPOSE 8080 443 9145 CMD ["nginx", "-g", "daemon off;"] From 37ce3e26cda535d671902ff875784ddfa767e25f Mon Sep 17 00:00:00 2001 From: Bryce Walter Date: Tue, 15 Nov 2016 12:57:35 -0700 Subject: [PATCH 2/8] Add prometheus lua config; add VST + GeoIP Config --- mainline/alpine/nginx.conf | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/mainline/alpine/nginx.conf b/mainline/alpine/nginx.conf index e4bad8db..b639cd99 100644 --- a/mainline/alpine/nginx.conf +++ b/mainline/alpine/nginx.conf @@ -14,6 +14,11 @@ events { http { include /etc/nginx/mime.types; default_type application/octet-stream; + # VTS Stub with geoip-dev + geoip_country /usr/share/GeoIP/GeoIP.dat; + + vhost_traffic_status_zone; + vhost_traffic_status_filter_by_set_key $geoip_country_code country::*; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' @@ -21,6 +26,22 @@ http { access_log /var/log/nginx/access.log main; + # Lua Module for Prometheus + lua_shared_dict prometheus_metrics 10M; + lua_package_path "/etc/nginx/modules/prometheus.lua"; + init_by_lua ' + prometheus = require("prometheus").init("prometheus_metrics") + metric_requests = prometheus:counter( + "nginx_http_requests_total", "Number of HTTP requests", {"host", "status"}) + metric_latency = prometheus:histogram( + "nginx_http_request_duration_seconds", "HTTP request latency", {"host"}) + '; + log_by_lua ' + local host = ngx.var.host:gsub("^www.", "") + metric_requests:inc(1, {host, ngx.var.status}) + metric_latency:observe(ngx.now() - ngx.req.start_time(), {host}) + '; + sendfile on; #tcp_nopush on; From 860165f8d8a3d34e974b7a304b78c6810562f031 Mon Sep 17 00:00:00 2001 From: Bryce Walter Date: Tue, 15 Nov 2016 12:58:56 -0700 Subject: [PATCH 3/8] Add VST config to /status, add Prometheus Scrape to /prom_metrics --- mainline/alpine/nginx.vh.default.conf | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mainline/alpine/nginx.vh.default.conf b/mainline/alpine/nginx.vh.default.conf index 34aeb9ac..f5f6c3f2 100644 --- a/mainline/alpine/nginx.vh.default.conf +++ b/mainline/alpine/nginx.vh.default.conf @@ -1,10 +1,13 @@ server { - listen 80; - server_name localhost; - + listen 8080; + server_name STGK8SANGINX.acomcloud.int; + vhost_traffic_status_filter_by_set_key $geoip_country_code country::$server_name; #charset koi8-r; #access_log /var/log/nginx/log/host.access.log main; - + location /status { + vhost_traffic_status_display; + vhost_traffic_status_display_format html; + } location / { root /usr/share/nginx/html; index index.html index.htm; @@ -42,4 +45,9 @@ server { # deny all; #} } - +server { + listen 9145; + location /prom_metrics { + content_by_lua 'prometheus:collect()'; + } +} From 31a189695043099e9bfe915323606b83f5733ba3 Mon Sep 17 00:00:00 2001 From: Bryce Walter Date: Tue, 15 Nov 2016 13:15:10 -0700 Subject: [PATCH 4/8] copy lua file vs clone and move --- mainline/alpine/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mainline/alpine/Dockerfile b/mainline/alpine/Dockerfile index e612429a..ec6887e0 100644 --- a/mainline/alpine/Dockerfile +++ b/mainline/alpine/Dockerfile @@ -122,7 +122,6 @@ RUN apk update && apk add git && mkdir -p /opt && cd /opt && git clone https://g && install -m755 objs/ngx_http_geoip_module-debug.so /usr/lib/nginx/modules/ngx_http_geoip_module-debug.so \ && install -m755 objs/ngx_http_perl_module-debug.so /usr/lib/nginx/modules/ngx_http_perl_module-debug.so \ && install -m755 objs/ngx_stream_geoip_module-debug.so /usr/lib/nginx/modules/ngx_stream_geoip_module-debug.so \ - && install -m755 /opt/nginx-lua-prometheus/prometheus.lua /etc/nginx/modules/prometheus.lua \ && ln -s ../../usr/lib/nginx/modules /etc/nginx/modules \ && strip /usr/sbin/nginx* \ && strip /usr/lib/nginx/modules/*.so \ @@ -154,6 +153,7 @@ RUN apk update && apk add git && mkdir -p /opt && cd /opt && git clone https://g COPY nginx.conf /etc/nginx/nginx.conf COPY nginx.vh.default.conf /etc/nginx/conf.d/default.conf +COPY prometheus.lua /etc/nginx/prometheus.lua EXPOSE 8080 443 9145 From 0c6516f63b049567a83a95c52d45b00a8aa33fd7 Mon Sep 17 00:00:00 2001 From: Bryce Walter Date: Tue, 15 Nov 2016 13:15:59 -0700 Subject: [PATCH 5/8] remove clone --- mainline/alpine/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/mainline/alpine/Dockerfile b/mainline/alpine/Dockerfile index ec6887e0..8e476bb7 100644 --- a/mainline/alpine/Dockerfile +++ b/mainline/alpine/Dockerfile @@ -18,7 +18,6 @@ RUN apk update && apk add git && mkdir -p /opt && cd /opt && git clone https://g apk add make g++ && cd /opt && git clone https://github.com/luajit/luajit && cd luajit && git checkout -b v$LUAJIT_VERSION && make && make install && cd .. && \ cd /opt && git clone https://github.com/openresty/lua-nginx-module && cd lua-nginx-module && git checkout -b v$LUA_NGINX_MODULE_VERSION && cd .. \ cd /opt && git clone https://github.com/kvspb/nginx-auth-ldap.git && cd .. && \ - cd /opt && git clone https://github.com/knyar/nginx-lua-prometheus.git && cd .. && \ cd /opt && git clone git://github.com/vozlt/nginx-module-vts.git && cd .. && \ apk del git && \ GPG_KEYS=B0F4253373F8F6F510D42178520A9993A1C052F8 \ From 9db6db4300f8c1282839909e66431d1d2cf30202 Mon Sep 17 00:00:00 2001 From: Bryce Walter Date: Tue, 15 Nov 2016 13:16:17 -0700 Subject: [PATCH 6/8] change path of lua --- mainline/alpine/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mainline/alpine/nginx.conf b/mainline/alpine/nginx.conf index b639cd99..6bca2bf8 100644 --- a/mainline/alpine/nginx.conf +++ b/mainline/alpine/nginx.conf @@ -28,7 +28,7 @@ http { # Lua Module for Prometheus lua_shared_dict prometheus_metrics 10M; - lua_package_path "/etc/nginx/modules/prometheus.lua"; + lua_package_path "/etc/nginx/prometheus.lua"; init_by_lua ' prometheus = require("prometheus").init("prometheus_metrics") metric_requests = prometheus:counter( From 143fbe2039bbc3f14cb471c5ccfa27bad9c3da39 Mon Sep 17 00:00:00 2001 From: Bryce Walter Date: Tue, 15 Nov 2016 13:16:44 -0700 Subject: [PATCH 7/8] prometheus lua module --- mainline/alpine/prometheus.lua | 424 +++++++++++++++++++++++++++++++++ 1 file changed, 424 insertions(+) create mode 100644 mainline/alpine/prometheus.lua diff --git a/mainline/alpine/prometheus.lua b/mainline/alpine/prometheus.lua new file mode 100644 index 00000000..94fc7888 --- /dev/null +++ b/mainline/alpine/prometheus.lua @@ -0,0 +1,424 @@ +-- vim: ts=2:sw=2:sts=2:expandtab +-- +-- This module uses a single dictionary shared between Nginx workers to keep +-- all metrics. Each counter is stored as a separate entry in that dictionary, +-- which allows us to increment them using built-in `incr` method. +-- +-- Prometheus requires that (a) all samples for a given metric are presented +-- as one uninterrupted group, and (b) buckets of a histogram appear in +-- increasing numerical order. We satisfy that by carefully constructing full +-- metric names (i.e. metric name along with all labels) so that they meet +-- those requirements while being sorted alphabetically. In particular: +-- +-- * all labels for a given metric are presented in reproducible order (the one +-- used when labels were declared). "le" label for histogram metrics always +-- goes last; +-- * bucket boundaries (which are exposed as values of the "le" label) are +-- presented as floating point numbers with leading and trailing zeroes. +-- Number of of zeroes is determined for each bucketer automatically based on +-- bucket boundaries; +-- * internally "+Inf" bucket is stored as "Inf" (to make it appear after +-- all numeric buckets), and gets replaced by "+Inf" just before we +-- expose the metrics. +-- +-- For example, if you define your bucket boundaries as {0.00005, 10, 1000} +-- then we will keep the following samples for a metric `m1` with label +-- `site` set to `site1`: +-- +-- m1_bucket{site="site1",le="0000.00005"} +-- m1_bucket{site="site1",le="0010.00000"} +-- m1_bucket{site="site1",le="1000.00000"} +-- m1_bucket{site="site1",le="Inf"} +-- m1_count{site="site1"} +-- m1_sum{site="site1"} +-- +-- "Inf" will be replaced by "+Inf" while publishing metrics. +-- +-- You can find the latest version and documentation at +-- https://github.com/knyar/nginx-lua-prometheus +-- Released under MIT license. + + +-- Default set of latency buckets, 5ms to 10s: +local DEFAULT_BUCKETS = {0.005, 0.01, 0.02, 0.03, 0.05, 0.075, 0.1, 0.2, 0.3, + 0.4, 0.5, 0.75, 1, 1.5, 2, 3, 4, 5, 10} + +-- Metric is a "parent class" for all metrics. +local Metric = {} +function Metric:new(o) + o = o or {} + setmetatable(o, self) + self.__index = self + return o +end + +-- Checks that the right number of labels values have been passed. +-- +-- Args: +-- label_values: an array of label values. +-- +-- Returns: +-- an error message or nil +function Metric:check_labels(label_values) + if self.label_names == nil and label_values == nil then + return + elseif self.label_names == nil and label_values ~= nil then + return "Expected no labels for " .. self.name .. ", got " .. #label_values + elseif label_values == nil and self.label_names ~= nil then + return "Expected " .. #self.label_names .. " labels for " .. + self.name .. ", got none" + elseif #self.label_names ~= #label_values then + return "Wrong number of labels for " .. self.name .. ". Expected " .. + #self.label_names .. ", got " .. #label_values + end +end + +local Counter = Metric:new() +-- Increase a given counter by `value` +-- +-- Args: +-- value: (number) a value to add to the counter. Defaults to 1 if skipped. +-- label_values: an array of label values. Can be nil (i.e. not defined) for +-- metrics that have no labels. +function Counter:inc(value, label_values) + local err = self:check_labels(label_values) + if err ~= nil then + self.prometheus:log_error(err) + return + end + self.prometheus:inc(self.name, self.label_names, label_values, value or 1) +end + +local Histogram = Metric:new() +-- Record a given value in a histogram. +-- +-- Args: +-- value: (number) a value to record. Should be defined. +-- label_values: an array of label values. Can be nil (i.e. not defined) for +-- metrics that have no labels. +function Histogram:observe(value, label_values) + if value == nil then + self.prometheus:log_error("No value passed for " .. self.name) + return + end + local err = self:check_labels(label_values) + if err ~= nil then + self.prometheus:log_error(err) + return + end + self.prometheus:histogram_observe(self.name, self.label_names, label_values, value) +end + +local Prometheus = {} +Prometheus.__index = Prometheus +Prometheus.initialized = false + +-- Generate full metric name that includes all labels. +-- +-- Args: +-- name: string +-- label_names: (array) a list of label keys. +-- label_values: (array) a list of label values. +-- Returns: +-- (string) full metric name. +local function full_metric_name(name, label_names, label_values) + if not label_names then + return name + end + local label_parts = {} + for idx, key in ipairs(label_names) do + local label_value = (string.format("%s", label_values[idx]) + :gsub("\\", "\\\\") + :gsub("\n", "\\n") + :gsub('"', '\\"')) + table.insert(label_parts, key .. '="' .. label_value .. '"') + end + return name .. "{" .. table.concat(label_parts, ",") .. "}" +end + +-- Construct bucket format for a list of buckets. +-- +-- This receives a list of buckets and returns a sprintf template that should +-- be used for bucket boundaries to make them come in increasing order when +-- sorted alphabetically. +-- +-- To re-phrase, this is where we detect how many leading and trailing zeros we +-- need. +-- +-- Args: +-- buckets: a list of buckets +-- +-- Returns: +-- (string) a sprintf template. +local function construct_bucket_format(buckets) + local max_order = 1 + local max_precision = 1 + for _, bucket in ipairs(buckets) do + assert(type(bucket) == "number", "bucket boundaries should be numeric") + -- floating point number with all trailing zeros removed + local as_string = string.format("%f", bucket):gsub("0*$", "") + local dot_idx = as_string:find(".", 1, true) + max_order = math.max(max_order, dot_idx - 1) + max_precision = math.max(max_precision, as_string:len() - dot_idx) + end + return "%0" .. (max_order + max_precision + 1) .. "." .. max_precision .. "f" +end + +-- Extract short metric name from the full one. +-- +-- Args: +-- full_name: (string) full metric name that can include labels. +-- +-- Returns: +-- (string) short metric name with no labels. For a `*_bucket` metric of +-- histogram the _bucket suffix will be removed. +local function short_metric_name(full_name) + local labels_start, _ = full_name:find("{") + if not labels_start then + -- no labels + return full_name + end + local suffix_idx, _ = full_name:find("_bucket{") + if suffix_idx and full_name:find("le=") then + -- this is a histogram metric + return full_name:sub(1, suffix_idx - 1) + end + -- this is not a histogram metric + return full_name:sub(1, labels_start - 1) +end + +-- Makes a shallow copy of a table +local function copy_table(table) + local new = {} + if table ~= nil then + for k, v in ipairs(table) do + new[k] = v + end + end + return new +end + +-- Initialize the module. +-- +-- This should be called once from the `init_by_lua` section in nginx +-- configuration. +-- +-- Args: +-- dict_name: (string) name of the nginx shared dictionary which will be +-- used to store all metrics +-- prefix: (optional string) if supplied, prefix is added to all +-- metric names on output +-- +-- Returns: +-- an object that should be used to register metrics. +function Prometheus.init(dict_name, prefix) + local self = setmetatable({}, Prometheus) + self.dict = ngx.shared[dict_name or "prometheus_metrics"] + self.help = {} + if prefix then + self.prefix = prefix + else + self.prefix = '' + end + self.type = {} + self.registered = {} + self.buckets = {} + self.bucket_format = {} + self.initialized = true + + self:counter("nginx_metric_errors_total", + "Number of nginx-lua-prometheus errors") + self.dict:set("nginx_metric_errors_total", 0) + return self +end + +function Prometheus:log_error(...) + ngx.log(ngx.ERR, ...) + self.dict:incr("nginx_metric_errors_total", 1) +end + +function Prometheus:log_error_kv(key, value, err) + self:log_error( + "Error while setting '", key, "' to '", value, "': '", err, "'") +end + +-- Register a counter. +-- +-- Args: +-- name: (string) name of the metric. Required. +-- description: (string) description of the metric. Will be used for the HELP +-- comment on the metrics page. Optional. +-- label_names: array of strings, defining a list of metrics. Optional. +-- +-- Returns: +-- a Counter object. +function Prometheus:counter(name, description, label_names) + if not self.initialized then + ngx.log(ngx.ERR, "Prometheus module has not been initialized") + return + end + + if self.registered[name] then + self:log_error("Duplicate metric " .. name) + return + end + self.registered[name] = true + self.help[name] = description + self.type[name] = "counter" + + return Counter:new{name=name, label_names=label_names, prometheus=self} +end + +-- Register a histogram. +-- +-- Args: +-- name: (string) name of the metric. Required. +-- description: (string) description of the metric. Will be used for the HELP +-- comment on the metrics page. Optional. +-- label_names: array of strings, defining a list of metrics. Optional. +-- buckets: array if numbers, defining bucket boundaries. Optional. +-- +-- Returns: +-- a Counter object. +function Prometheus:histogram(name, description, label_names, buckets) + if not self.initialized then + ngx.log(ngx.ERR, "Prometheus module has not been initialized") + return + end + + for _, label_name in ipairs(label_names or {}) do + if label_name == "le" then + self:log_error("Invalid label name 'le' in " .. name) + return + end + end + + for _, suffix in ipairs({"", "_bucket", "_count", "_sum"}) do + if self.registered[name .. suffix] then + self:log_error("Duplicate metric " .. name .. suffix) + return + end + self.registered[name .. suffix] = true + end + self.help[name] = description + self.type[name] = "histogram" + + self.buckets[name] = buckets or DEFAULT_BUCKETS + self.bucket_format[name] = construct_bucket_format(self.buckets[name]) + + return Histogram:new{name=name, label_names=label_names, prometheus=self} +end + +-- Set a given dictionary key. +-- This overwrites existing values, so we use it only to initialize metrics. +function Prometheus:set(key, value) + local ok, err = self.dict:safe_set(key, value) + if not ok then + self:log_error_kv(key, value, err) + end +end + +-- Increment a given counter by `value`. +-- +-- Args: +-- name: (string) short metric name without any labels. +-- label_names: (array) a list of label keys. +-- label_values: (array) a list of label values. +-- value: (number) value to add. Optional, defaults to 1. +function Prometheus:inc(name, label_names, label_values, value) + local key = full_metric_name(name, label_names, label_values) + if value == nil then value = 1 end + if value < 0 then + self:log_error_kv(key, value, "Value should not be negative") + return + end + + local newval, err = self.dict:incr(key, value) + if newval then + return + end + -- Yes, this looks like a race, so I guess we might under-report some values + -- when multiple workers simultaneously try to create the same metric. + -- Hopefully this does not happen too often (shared dictionary does not get + -- reset during configuation reload). + if err == "not found" then + self:set(key, value) + return + end + -- Unexpected error + self:log_error_kv(key, value, err) +end + +-- Record a given value into a histogram metric. +-- +-- Args: +-- name: (string) short metric name without any labels. +-- label_names: (array) a list of label keys. +-- label_values: (array) a list of label values. +-- value: (number) value to observe. +function Prometheus:histogram_observe(name, label_names, label_values, value) + self:inc(name .. "_count", label_names, label_values, 1) + self:inc(name .. "_sum", label_names, label_values, value) + + -- we are going to mutate arrays of label names and values, so create a copy. + local l_names = copy_table(label_names) + local l_values = copy_table(label_values) + + -- Last bucket. Note, that the label value is "Inf" rather than "+Inf" + -- required by Prometheus. This is necessary for this bucket to be the last + -- one when all metrics are lexicographically sorted. "Inf" will get replaced + -- by "+Inf" in Prometheus:collect(). + table.insert(l_names, "le") + table.insert(l_values, "Inf") + self:inc(name .. "_bucket", l_names, l_values, 1) + + local label_count = #l_names + for _, bucket in ipairs(self.buckets[name]) do + if value <= bucket then + -- last label is now "le" + l_values[label_count] = self.bucket_format[name]:format(bucket) + self:inc(name .. "_bucket", l_names, l_values, 1) + end + end +end + +-- Present all metrics in a text format compatible with Prometheus. +-- +-- This function should be used to expose the metrics on a separate HTTP page. +-- It will get the metrics from the dictionary, sort them, and expose them +-- aling with TYPE and HELP comments. +function Prometheus:collect() + ngx.header.content_type = "text/plain" + if not self.initialized then + ngx.log(ngx.ERR, "Prometheus module has not been initialized") + return + end + + local keys = self.dict:get_keys(0) + -- Prometheus server expects buckets of a histogram to appear in increasing + -- numerical order of their label values. + table.sort(keys) + + local seen_metrics = {} + for _, key in ipairs(keys) do + local value, err = self.dict:get(key) + if value then + local short_name = short_metric_name(key) + if not seen_metrics[short_name] then + if self.help[short_name] then + ngx.say("# HELP " .. self.prefix .. short_name .. " " .. self.help[short_name]) + end + if self.type[short_name] then + ngx.say("# TYPE " .. self.prefix .. short_name .. " " .. self.type[short_name]) + end + seen_metrics[short_name] = true + end + -- Replace "Inf" with "+Inf" in each metric's last bucket 'le' label. + ngx.say(self.prefix .. key:gsub('le="Inf"', 'le="+Inf"'), " ", value) + else + self:log_error("Error getting '", key, "': ", err) + end + end +end + +return Prometheus From adc2a2fa5e493d4610fed2d7f57ddbd7256a7b4a Mon Sep 17 00:00:00 2001 From: Bryce Walter Date: Tue, 15 Nov 2016 13:57:54 -0700 Subject: [PATCH 8/8] remove geo_country for now due to error --- mainline/alpine/nginx.conf | 4 ++-- mainline/alpine/nginx.vh.default.conf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mainline/alpine/nginx.conf b/mainline/alpine/nginx.conf index 6bca2bf8..ed5e9755 100644 --- a/mainline/alpine/nginx.conf +++ b/mainline/alpine/nginx.conf @@ -15,10 +15,10 @@ http { include /etc/nginx/mime.types; default_type application/octet-stream; # VTS Stub with geoip-dev - geoip_country /usr/share/GeoIP/GeoIP.dat; + #geoip_country /usr/share/GeoIP/GeoIP.dat; vhost_traffic_status_zone; - vhost_traffic_status_filter_by_set_key $geoip_country_code country::*; + #vhost_traffic_status_filter_by_set_key $geoip_country_code country::*; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' diff --git a/mainline/alpine/nginx.vh.default.conf b/mainline/alpine/nginx.vh.default.conf index f5f6c3f2..71b72169 100644 --- a/mainline/alpine/nginx.vh.default.conf +++ b/mainline/alpine/nginx.vh.default.conf @@ -1,7 +1,7 @@ server { listen 8080; server_name STGK8SANGINX.acomcloud.int; - vhost_traffic_status_filter_by_set_key $geoip_country_code country::$server_name; + #vhost_traffic_status_filter_by_set_key $geoip_country_code country::$server_name; #charset koi8-r; #access_log /var/log/nginx/log/host.access.log main; location /status {