在配置为不带参数分析时，支持固定的URL参数，以区分部分的请求

Author: JeffXue

bin/config.py

@@ -15,6 +15,7 @@ def __init__(self, config_file):
 
         self.support_method = all_config.get('filter', 'support_method').split(',')
         self.is_with_parameters = int(all_config.get('filter', 'is_with_parameters'))
+        self.always_parameter_keys = all_config.get('filter', 'always_parameter_keys').split(',')
         self.urls_most_number = int(all_config.get('filter', 'urls_most_number'))
         self.urls_pv_threshold = int(all_config.get('filter', 'urls_pv_threshold'))
         self.fixed_parameter_keys = all_config.get('filter', 'fixed_parameter_keys').split(',')

bin/start.py

@@ -90,6 +90,24 @@ def get_new_url(origin_url):
     return new_url
 
 
+def get_new_url_for_always_parameters(origin_url):
+    if len(origin_url.split('?')) == 1:
+        return origin_url
+
+    url_front = origin_url.split('?')[0]
+    url_parameters = sorted(origin_url.split('?')[1].split('&'))
+    new_url_parameters = []
+    for parameter in url_parameters:
+        key = parameter.split('=')[0]
+        if key in config.always_parameter_keys:
+            new_url_parameters.append(parameter)
+    if new_url_parameters:
+        new_url = url_front + '?' + '&'.join(new_url_parameters)
+    else:
+        new_url = origin_url
+    return new_url
+
+
 def parse_log_file(target_file, log_format):
     # 用户IP
     hosts = []
@@ -125,7 +143,10 @@ def parse_log_file(target_file, log_format):
             if config.is_with_parameters:
                 url = get_new_url(match.group(log_format.get('url_index')))
             else:
-                url = match.group(log_format.get('url_index')).split('?')[0]
+                if config.always_parameter_keys:
+                    url = get_new_url_for_always_parameters(match.group(log_format.get('url_index')))
+                else:
+                    url = match.group(log_format.get('url_index')).split('?')[0]
             if is_ignore_url(url):
                 continue
             if match.group(log_format.get('method_index')) not in config.support_method:
@@ -187,7 +208,10 @@ def parse_log_file(target_file, log_format):
             if config.is_with_parameters:
                 url = get_new_url(match.group(log_format.get('url_index')))
             else:
-                url = match.group(log_format.get('url_index')).split('?')[0]
+                if config.always_parameter_keys:
+                    url = get_new_url_for_always_parameters(match.group(log_format.get('url_index')))
+                else:
+                    url = match.group(log_format.get('url_index')).split('?')[0]
             for url_data in url_data_list:
                 if url_data.url == ' '.join([method, url]):
                     url_data.time.append(match.group(log_format.get('time_index')))

conf/config.ini

@@ -6,13 +6,22 @@ log-pattern=(\S+)\s\S+\s(\S+)\s(\S+)\s(\d+)\s(\S+)\s(\S+)\s(\S+)\s(\S+)\s(\S+)\s
 log-format=datetime method url status protocol business_status cost host hostname real_ip
 
 [filter]
+# 支持的方法
 support_method=POST,GET
+# 是否带参数进行分析（但会包括awalys_parameter_keys所指定的参数）
 is_with_parameters=0
+always_parameter_keys=action
+# 访问量排行最大请求数量
 urls_most_number=200
+# 访问量排行的最低PV阀值，低于该阀值的不会进入访问量排行
 urls_pv_threshold=1000
+# 固定的参数，但is_with_parameters=1时，不会替换一下key的值
 fixed_parameter_keys=action,submitType,reportType
+# 自定义的参数转换
 custom_parameters=t={timeStamp},v={timeStamp},_={timeStamp}
+# 忽略的URL
 ignore_urls=/slb.html,/server-status,/httpstatus.html,/server-status-dinghuo/,/server-status-dinghuo
+# 忽略的请求类型
 static-file=css,CSS,dae,DAE,eot,EOT,gif,GIF,ico,ICO,jpeg,JPEG,jpg,JPG,js,JS,map,MAP,mp3,MP3,pdf,PDF,png,PNG,svg,SVG,swf,SWF,ttf,TTF,txt,TXT,woff,WOFF
 
 [report]