Merge branch 'master' into master

Author: AutomatedTester

.travis.yml

@@ -1,4 +1,15 @@
 language: python
+
+jdk:
+  - oraclejdk8
+
+addons:
+  apt:
+    packages:
+      - oracle-java8-installer
+      - oracle-java8-set-default
+      - google-chrome-stable
+
 sudo: false
 python:
   - 2.6
@@ -7,10 +18,12 @@ python:
   - 3.4
   - 3.5
 before_script:
-  - curl -l -L -O https://github.com/lightbody/browsermob-proxy/releases/download/browsermob-proxy-2.1.0-beta-6/browsermob-proxy-2.1.0-beta-6-bin.zip
-  - unzip browsermob-proxy-2.1.0-beta-6-bin.zip
-  - sh browsermob-proxy-2.1.0-beta-6/bin/browsermob-proxy --port=9090 &
+  - export DISPLAY=:99.0
+  - sh -e /etc/init.d/xvfb start
+  - ./bootstrap.sh
+  - ./start-servers.sh
   - sleep 5
 install: pip install pytest selenium
-script: python setup.py develop && py.test -m "not human" test
+script: python setup.py develop && py.test test
 sudo: false
+dist: trusty

History.md

@@ -1,4 +1,45 @@
 
+0.8.0 / 2017-02-07
+==================
+
+  * Adds rewrite rules clear api. Fixes content type for request/response intercept and adds doc links for writing intercept code. Adds comments clarifying intercept client test coverage. Augments human test for url rewrite and adds clear portion. Adds test for response intercept. Adds bootstrap script to download tools to test with. Refactors human tests so browser session are cleaned up when tests fail and tests run with chromium. Adds server start script for travis. Changes .travis.yml to use provided scripts and also run headless chrome.
+  * If the JSON doesn't load, it's likely because the user has another server running on the same port. Improve the error message here so can figure out what's going on more easily.
+  * Inherit from Exception as StandardError was removed in Python 3
+  * Add ProxyServerError and raise it instead of Exception
+  * Fix docstring for limits()
+  * Fix path join issue: log_path_name would be /server.log
+  * Make default option values more robust
+  * Add retry sleep and count in options
+  * Reenabling timeout tests as they have been fixed in B6 of the server
+  * Removing duplicate method. Fixes #41
+  * docstrings cleanup
+  * configurable log file
+  * added title options for new har and new har page
+  * replaced mutable default values with None
+  * Update Requests dependency version
+  * Updating supported Python versions
+  * Disabling a test due to BMP Server bug
+  * Bumping version number of BMP server used in testing
+  * Add bmp.log to .gitignore
+  * Add details on how to contribute
+  * Updating response URL endpoints to use filter
+  * Ignoring .cache directory
+  * Change remap_hosts to accept dictionary
+  * Make sure the process is still running while waiting for the server to start.
+  * Changed hostmap to be a named argument for compatibility
+  * Merge remote-tracking branch 'upstream/master' into patch-1
+  * Add the ability to empty the DNS cache
+  * Correct function name
+  * Add the ability to return all active proxy ports from Browsermob
+  * switch to Travis container setup
+  * Update to latest browsermob proxy release for testing
+  * Correct exception typo
+  * Update remap_hosts test
+  * Fix whitespace
+  * Let remap_hosts accept a dictionary
+  * Upgrading Travis to it's container system
+  * bumping to 0.7.1
+
 0.7.1 / 2015-07-09
 ==================
 

bootstrap.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+if [ "$(uname)" == "Darwin" ]; then
+    echo "We're on a MAC!"
+   chromeDriver="chromedriver_mac64.zip"
+   geckoDriverVersion="v0.13.0"
+   geckoDriver="geckodriver-$geckoDriverVersion-macos.tar.gz"
+else
+    echo "We're not on a MAC!"
+   chromeDriver="chromedriver_linux64.zip"
+   geckoDriverVersion="v0.14.0"
+   geckoDriver="geckodriver-$geckoDriverVersion-linux64.tar.gz"
+fi
+rm -rf tools
+mkdir -p tools && \
+cd tools && \
+wget https://github.com/lightbody/browsermob-proxy/releases/download/browsermob-proxy-2.1.4/browsermob-proxy-2.1.4-bin.zip && \
+unzip -o browsermob-proxy-2.1.4-bin.zip && \
+rm -rf browsermob-proxy*.zip* && \
+wget http://selenium-release.storage.googleapis.com/3.0/selenium-server-standalone-3.0.1.jar && \
+wget https://github.com/mozilla/geckodriver/releases/download/${geckoDriverVersion}/${geckoDriver} && \
+tar zxf ${geckoDriver} && \
+rm -rf ${geckoDriver}* && \
+wget https://chromedriver.storage.googleapis.com/2.27/${chromeDriver} && \
+unzip ${chromeDriver} && \
+rm -rf ${chromeDriver}* && \
+cd .. 

browsermobproxy/client.py

@@ -34,7 +34,8 @@ def __init__(self, url, params=None, options=None):
                 jcontent = json.loads(content)
             except Exception as e:
                 raise Exception("Could not read Browsermob-Proxy json\n"
-                    "Another server running on this port?\n%s..."%content[:512])
+                 "Another server running on this port?\n%s..."%content[:512])
+
             self.port = jcontent['port']
         url_parts = self.host.split(":")
         self.proxy = url_parts[1][2:] + ":" + str(self.port)
@@ -103,7 +104,6 @@ def har(self):
 
         return r.json()
 
-
     def new_har(self, ref=None, options=None, title=None):
         """
         This sets a new HAR to be recorded
@@ -199,24 +199,30 @@ def headers(self, headers):
 
     def response_interceptor(self, js):
         """
-        Executes the javascript against each response
-
-        :param str js: the javascript to execute
+        Executes the java/js code against each response
+        `HttpRequest request <https://netty.io/4.1/api/io/netty/handler/codec/http/HttpRequest.html>`_,
+        `HttpMessageContents contents <https://raw.githubusercontent.com/lightbody/browsermob-proxy/master/browsermob-core/src/main/java/net/lightbody/bmp/util/HttpMessageContents.java>`_,
+        `HttpMessageInfo messageInfo <https://raw.githubusercontent.com/lightbody/browsermob-proxy/master/browsermob-core/src/main/java/net/lightbody/bmp/util/HttpMessageInfo.java>`_
+        are available objects to interact with.
+        :param str js: the js/java code to execute
         """
         r = requests.post(url='%s/proxy/%s/filter/response' % (self.host, self.port),
-                  data=js,
-                  headers={'content-type': 'application/json'})
+                          data=js,
+                          headers={'content-type': 'text/plain'})
         return r.status_code
 
     def request_interceptor(self, js):
         """
-        Executes the javascript against each request
-
-        :param str js: the javascript to execute
+        Executes the java/js code against each response
+        `HttpRequest request <https://netty.io/4.1/api/io/netty/handler/codec/http/HttpRequest.html>`_,
+        `HttpMessageContents contents <https://raw.githubusercontent.com/lightbody/browsermob-proxy/master/browsermob-core/src/main/java/net/lightbody/bmp/util/HttpMessageContents.java>`_,
+        `HttpMessageInfo messageInfo <https://raw.githubusercontent.com/lightbody/browsermob-proxy/master/browsermob-core/src/main/java/net/lightbody/bmp/util/HttpMessageInfo.java>`_
+        are available objects to interact with.
+        :param str js: the js/java code to execute
         """
         r = requests.post(url='%s/proxy/%s/filter/request' % (self.host, self.port),
-                  data=js,
-                  headers={'content-type': 'application/json'})
+                          data=js,
+                          headers={'content-type': 'text/plain'})
         return r.status_code
 
     LIMITS = {
@@ -295,7 +301,7 @@ def remap_hosts(self, address=None, ip_address=None, hostmap=None):
             hostmap[address] = ip_address
 
         r = requests.post('%s/proxy/%s/hosts' % (self.host, self.port),
-                         json.dumps(hostmap),
+                          json.dumps(hostmap),
                           headers={'content-type': 'application/json'})
         return r.status_code
 
@@ -308,7 +314,7 @@ def wait_for_traffic_to_stop(self, quiet_period, timeout):
         :param int timeout: max number of milliseconds to wait
         """
         r = requests.put('%s/proxy/%s/wait' % (self.host, self.port),
-                 {'quietPeriodInMs': quiet_period, 'timeoutInMs': timeout})
+                         {'quietPeriodInMs': quiet_period, 'timeoutInMs': timeout})
         return r.status_code
 
     def clear_dns_cache(self):
@@ -334,12 +340,21 @@ def rewrite_url(self, match, replace):
                          params)
         return r.status_code
 
+    def clear_all_rewrite_url_rules(self):
+        """
+        Clears all URL rewrite rules
+        :return: status code
+        """
+
+        r = requests.delete('%s/proxy/%s/rewrite' % (self.host, self.port))
+        return r.status_code
+
     def retry(self, retry_count):
         """
         Retries. No idea what its used for, but its in the API...
 
         :param int retry_count: the number of retries
         """
         r = requests.put('%s/proxy/%s/retry' % (self.host, self.port),
-                 {'retrycount': retry_count})
+                         {'retrycount': retry_count})
         return r.status_code

browsermobproxy/server.py

@@ -1,10 +1,13 @@
 import os
 import platform
+import signal
 import socket
 import subprocess
 import time
 import logging
 
+import sys
+
 from .client import Client
 from .exceptions import ProxyServerError
 
@@ -63,6 +66,8 @@ def __init__(self, path='browsermob-proxy', options=None):
             More items will be added in the future.
             This defaults to an empty dictionary
         """
+        self.win_env = sys.platform == "win32"
+
         options = options if options is not None else {}
 
         path_var_sep = ':'
@@ -82,7 +87,7 @@ def __init__(self, path='browsermob-proxy', options=None):
                                    "in path provided: %s" % path)
 
         self.path = path
-        self.host = 'localhost'
+        self.host = options.get('host', 'localhost')
         self.port = options.get('port', 8080)
         self.process = None
 
@@ -109,13 +114,16 @@ def start(self, options=None):
         log_path_name = os.path.join(log_path, log_file)
         self.log_file = open(log_path_name, 'w')
 
+
         if self._is_listening(.1):
             logging.info("BrowserMob proxy already running. Won't start again.")
             return
-        
-        self.process = subprocess.Popen(self.command,
-                                        stdout=self.log_file,
-                                        stderr=subprocess.STDOUT)
+       
+        if self.win_env:
+            self.process = self._start_on_windows()
+        else:
+            self.process = self._start_on_unix()
+
         count = 0
         while not self._is_listening():
             if self.process.poll():
@@ -131,16 +139,30 @@ def start(self, options=None):
                 self.stop()
                 raise ProxyServerError("Can't connect to Browsermob-Proxy")
 
+    def _start_on_windows(self):
+        return subprocess.Popen(self.command,
+                                creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
+                                stdout=self.log_file,
+                                stderr=subprocess.STDOUT)
+
+    def _start_on_unix(self):
+        return subprocess.Popen(self.command,
+                                preexec_fn=os.setsid,
+                                stdout=self.log_file,
+                                stderr=subprocess.STDOUT)
+
     def stop(self):
         """
         This will stop the process running the proxy
         """
         if self.process.poll() is not None:
             return
 
+        group_pid = os.getpgid(self.process.pid) if not self.win_env else self.process.pid
         try:
             self.process.kill()
             self.process.wait()
+            os.killpg(group_pid, signal.SIGINT)
         except AttributeError:
             # kill may not be available under windows environment
             pass

setup.py

@@ -1,10 +1,10 @@
 from setuptools import setup, find_packages
 
 setup(name='browsermob-proxy',
-      version='0.7.1',
+      version='0.8.0',
       description='A library for interacting with the Browsermob Proxy',
       author='David Burns',
-      author_email='david.burns at theautomatedtester dot co dot uk',
+      author_email='david.burns@theautomatedtester.co.uk',
       url='http://oss.theautomatedtester.co.uk/browsermob-proxy-py',
       classifiers=['Development Status :: 3 - Alpha',
                   'Intended Audience :: Developers',

start-servers.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+TOOLS=$(pwd)/tools
+cd $TOOLS/browsermob-proxy-2.1.4/bin/ && \
+./browsermob-proxy --port 9090 & \
+cd $TOOLS && \
+java -Dwebdriver.chrome.driver=chromedriver -Dwebdriver.gecko.driver=geckodriver -jar selenium-server-standalone-3.0.1.jar &

test/test_client.py

@@ -167,6 +167,8 @@ def test_close(self):
     def test_response_interceptor_with_parsing_js(self):
         """
         /proxy/:port/interceptor/response
+        This test is only checking very basic syntax rules. The snippet needs to be JAVA/JS
+        Code only gets validated if the filter/interceptor is used.
         """
         js = 'alert("foo")'
         status_code = self.client.response_interceptor(js)
@@ -175,6 +177,8 @@ def test_response_interceptor_with_parsing_js(self):
     def test_response_interceptor_with_invalid_js(self):
         """
         /proxy/:port/interceptor/response
+        This test is only checking very basic syntax rules. The snippet needs to be JAVA/JS
+        Code only gets validated if the filter/interceptor is used.
         """
         js = 'alert("foo"'
         status_code = self.client.response_interceptor(js)
@@ -183,6 +187,8 @@ def test_response_interceptor_with_invalid_js(self):
     def test_request_interceptor_with_parsing_js(self):
         """
         /proxy/:port/interceptor/request
+        This test is only checking very basic syntax rules. The snippet needs to be JAVA/JS
+        Code only gets validated if the filter/interceptor is used.
         """
         js = 'alert("foo")'
         status_code = self.client.request_interceptor(js)
@@ -191,6 +197,8 @@ def test_request_interceptor_with_parsing_js(self):
     def test_request_interceptor_with_invalid_js(self):
         """
         /proxy/:port/interceptor/request
+        This test is only checking very basic syntax rules. The snippet needs to be JAVA/JS
+        Code only gets validated if the filter/interceptor is used.
         """
         js = 'alert("foo"'
         status_code = self.client.request_interceptor(js)