NDEV-814 update core libraries to upstream

Makefile

@@ -13,7 +13,7 @@ CA_KEY_FILE_PATH := ca-key.pem
 CA_CERT_FILE_PATH := ca-cert.pem
 CA_SIGNING_KEY_FILE_PATH := ca-signing-key.pem
 
-.PHONY: all https-certificates ca-certificates autopep8 devtools
+.PHONY: all https-certificates ca-certificates autopep8
 .PHONY: lib-version lib-clean lib-test lib-package lib-coverage lib-lint
 .PHONY: lib-release-test lib-release lib-profile
 .PHONY: container container-run container-release

README.md

@@ -85,7 +85,6 @@ Table of Contents
         * [Man-In-The-Middle Plugin](#maninthemiddleplugin)
         * [Proxy Pool Plugin](#proxypoolplugin)
     * [HTTP Web Server Plugins](#http-web-server-plugins)
-        * [Reverse Proxy](#reverse-proxy)
         * [Web Server Route](#web-server-route)
     * [Plugin Ordering](#plugin-ordering)
 * [End-to-End Encryption](#end-to-end-encryption)
@@ -1610,9 +1609,8 @@ usage: proxy [-h] [--backlog BACKLOG] [--basic-auth BASIC_AUTH]
              [--ca-signing-key-file CA_SIGNING_KEY_FILE]
              [--cert-file CERT_FILE]
              [--client-recvbuf-size CLIENT_RECVBUF_SIZE]
-             [--devtools-ws-path DEVTOOLS_WS_PATH]
              [--disable-headers DISABLE_HEADERS] [--disable-http-proxy]
-             [--enable-devtools] [--enable-events]
+             [--enable-events]
              [--enable-static-server] [--enable-web-server]
              [--hostname HOSTNAME] [--key-file KEY_FILE]
              [--log-level LOG_LEVEL] [--log-file LOG_FILE]
@@ -1660,9 +1658,6 @@ optional arguments:
                         the client in a single recv() operation. Bump this
                         value for faster uploads at the expense of increased
                         RAM.
-  --devtools-ws-path DEVTOOLS_WS_PATH
-                        Default: /devtools. Only applicable if --enable-
-                        devtools is used.
   --disable-headers DISABLE_HEADERS
                         Default: None. Comma separated list of headers to
                         remove before dispatching client request to upstream

proxy/__init__.py

@@ -8,16 +8,19 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-from .proxy import entry_point
-from .proxy import main, start
-from .proxy import Proxy
+from .proxy import Proxy, main, sleep_loop, entry_point
+
 
 __all__ = [
     # PyPi package entry_point. See
     # https://github.com/abhinavsingh/proxy.py#from-command-line-when-installed-using-pip
     'entry_point',
     # Embed proxy.py. See
     # https://github.com/abhinavsingh/proxy.py#embed-proxypy
-    'main', 'start',
+    'main',
+    # Unit testing with proxy.py. See
+    # https://github.com/abhinavsingh/proxy.py#unit-testing-with-proxypy
     'Proxy',
+    # Utility exposed for demos
+    'sleep_loop',
 ]

proxy/common/_version.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+
+    Version definition.
+"""
+from typing import Tuple, Union
+
+
+__version__ = '2.4.3'
+_ver_tup = 2, 4, 3
+
+
+def _to_int_or_str(inp: str) -> Union[int, str]:    # pragma: no cover
+    try:
+        return int(inp)
+    except ValueError:
+        return inp
+
+
+def _split_version_parts(inp: str) -> Tuple[str, ...]:  # pragma: no cover
+    public_version, _plus, local_version = inp.partition('+')
+    return *public_version.split('.'), local_version
+
+
+try:
+    VERSION = _ver_tup
+except NameError:   # pragma: no cover
+    VERSION = tuple(
+        map(_to_int_or_str, _split_version_parts(__version__)),
+    )
+
+
+__all__ = '__version__', 'VERSION'

proxy/common/backports.py

@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import time
+import threading
+from queue import Empty
+from typing import Any, Deque
+from collections import deque
+
+
+class cached_property:      # pragma: no cover
+    """Decorator for read-only properties evaluated only once within TTL period.
+    It can be used to create a cached property like this::
+
+        import random
+
+        # the class containing the property must be a new-style class
+        class MyClass:
+            # create property whose value is cached for ten minutes
+            @cached_property(ttl=600)
+            def randint(self):
+                # will only be evaluated every 10 min. at maximum.
+                return random.randint(0, 100)
+
+    The value is cached  in the '_cached_properties' attribute of the object instance that
+    has the property getter method wrapped by this decorator. The '_cached_properties'
+    attribute value is a dictionary which has a key for every property of the
+    object which is wrapped by this decorator. Each entry in the cache is
+    created only when the property is accessed for the first time and is a
+    two-element tuple with the last computed property value and the last time
+    it was updated in seconds since the epoch.
+
+    The default time-to-live (TTL) is 0 seconds i.e. cached value will never expire.
+
+    To expire a cached property value manually just do::
+        del instance._cached_properties[<property name>]
+
+    Adopted from https://wiki.python.org/moin/PythonDecoratorLibrary#Cached_Properties
+    © 2011 Christopher Arndt, MIT License.
+
+    NOTE: We need this function only because Python in-built are only available
+    for 3.8+.  Hence, we must get rid of this function once proxy.py no longer
+    support version older than 3.8.
+
+    .. spelling::
+
+       backports
+       getter
+       Arndt
+       del
+    """
+
+    def __init__(self, ttl: float = 0):
+        self.ttl = ttl
+
+    def __call__(self, fget: Any, doc: Any = None) -> 'cached_property':
+        self.fget = fget
+        self.__doc__ = doc or fget.__doc__
+        self.__name__ = fget.__name__
+        self.__module__ = fget.__module__
+        return self
+
+    def __get__(self, inst: Any, owner: Any) -> Any:
+        now = time.time()
+        try:
+            value, last_update = inst._cached_properties[self.__name__]
+            if self.ttl > 0 and now - last_update > self.ttl:   # noqa: WPS333
+                raise AttributeError
+        except (KeyError, AttributeError):
+            value = self.fget(inst)
+            try:
+                cache = inst._cached_properties
+            except AttributeError:
+                cache, inst._cached_properties = {}, {}
+            finally:
+                cache[self.__name__] = (value, now)  # pylint: disable=E0601
+        return value
+
+
+class NonBlockingQueue:
+    '''Simple, unbounded, non-blocking FIFO queue.
+
+    Supports only a single consumer.
+
+    NOTE: This is available in Python since 3.7 as SimpleQueue.
+    Here because proxy.py still supports 3.6
+    '''
+
+    def __init__(self) -> None:
+        self._queue: Deque[Any] = deque()
+        self._count: threading.Semaphore = threading.Semaphore(0)
+
+    def put(self, item: Any) -> None:
+        '''Put the item on the queue.'''
+        self._queue.append(item)
+        self._count.release()
+
+    def get(self) -> Any:
+        '''Remove and return an item from the queue.'''
+        if not self._count.acquire(False, None):
+            raise Empty
+        return self._queue.popleft()
+
+    def empty(self) -> bool:
+        '''Return True if the queue is empty, False otherwise (not reliable!).'''
+        return len(self._queue) == 0    # pragma: no cover
+
+    def qsize(self) -> int:
+        '''Return the approximate size of the queue (not reliable!).'''
+        return len(self._queue)     # pragma: no cover

proxy/common/constants.py

@@ -9,26 +9,61 @@
     :license: BSD, see LICENSE for more details.
 """
 import os
+import sys
 import time
 import pathlib
+import secrets
+import platform
 import ipaddress
-
-from typing import List
+import sysconfig
+from typing import Any, List
 
 from .version import __version__
 
+
+SYS_PLATFORM = platform.system()
+IS_WINDOWS = SYS_PLATFORM.lower() in ('windows', 'cygwin')
+
+
+def _env_threadless_compliant() -> bool:
+    """Returns true for Python 3.8+ across all platforms
+    except Windows."""
+    return not IS_WINDOWS and sys.version_info >= (3, 8)
+
+
 PROXY_PY_START_TIME = time.time()
 
 # /path/to/proxy.py/proxy folder
 PROXY_PY_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
+# Path to virtualenv/lib/python3.X/site-packages
+PROXY_PY_SITE_PACKAGES = sysconfig.get_path('purelib')
+assert PROXY_PY_SITE_PACKAGES
+
 CRLF = b'\r\n'
 COLON = b':'
 WHITESPACE = b' '
 COMMA = b','
 DOT = b'.'
 SLASH = b'/'
-HTTP_1_1 = b'HTTP/1.1'
+AT = b'@'
+HTTP_PROTO = b'http'
+HTTPS_PROTO = HTTP_PROTO + b's'
+HTTP_1_0 = HTTP_PROTO.upper() + SLASH + b'1.0'
+HTTP_1_1 = HTTP_PROTO.upper() + SLASH + b'1.1'
+HTTP_URL_PREFIX = HTTP_PROTO + COLON + SLASH + SLASH
+HTTPS_URL_PREFIX = HTTPS_PROTO + COLON + SLASH + SLASH
+
+LOCAL_INTERFACE_HOSTNAMES = (
+    b'localhost',
+    b'127.0.0.1',
+    b'::1',
+)
+
+ANY_INTERFACE_HOSTNAMES = (
+    b'0.0.0.0',
+    b'::',
+)
 
 PROXY_AGENT_HEADER_KEY = b'Proxy-agent'
 PROXY_AGENT_HEADER_VALUE = b'proxy.py v' + \
@@ -39,44 +74,84 @@
 # Defaults
 DEFAULT_BACKLOG = 100
 DEFAULT_BASIC_AUTH = None
-DEFAULT_BUFFER_SIZE = 1024 * 1024
+DEFAULT_MAX_SEND_SIZE = 64 * 1024
+DEFAULT_BUFFER_SIZE = 128 * 1024
 DEFAULT_CA_CERT_DIR = None
 DEFAULT_CA_CERT_FILE = None
 DEFAULT_CA_KEY_FILE = None
 DEFAULT_CA_SIGNING_KEY_FILE = None
 DEFAULT_CERT_FILE = None
-DEFAULT_CA_FILE = None
+DEFAULT_CA_FILE = pathlib.Path(
+    PROXY_PY_SITE_PACKAGES,
+) / 'certifi' / 'cacert.pem'
 DEFAULT_CLIENT_RECVBUF_SIZE = DEFAULT_BUFFER_SIZE
-DEFAULT_DEVTOOLS_WS_PATH = b'/devtools'
 DEFAULT_DISABLE_HEADERS: List[bytes] = []
-DEFAULT_DISABLE_HTTP_PROXY = False
+DEFAULT_ENABLE_SSH_TUNNEL = False
 DEFAULT_ENABLE_EVENTS = False
 DEFAULT_EVENTS_QUEUE = None
 DEFAULT_ENABLE_STATIC_SERVER = False
 DEFAULT_ENABLE_WEB_SERVER = False
+DEFAULT_ALLOWED_URL_SCHEMES = [HTTP_PROTO, HTTPS_PROTO]
 DEFAULT_IPV4_HOSTNAME = ipaddress.IPv4Address('127.0.0.1')
 DEFAULT_IPV6_HOSTNAME = ipaddress.IPv6Address('::1')
 DEFAULT_KEY_FILE = None
 DEFAULT_LOG_FILE = None
-DEFAULT_LOG_FORMAT = '%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(funcName)s:%(lineno)d - %(message)s'
+DEFAULT_LOG_FORMAT = '%(asctime)s - pid:%(process)d [%(levelname)-.1s] %(module)s.%(funcName)s:%(lineno)d - %(message)s'
 DEFAULT_LOG_LEVEL = 'INFO'
+DEFAULT_WEB_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' \
+    '{request_method} {request_path} - {request_ua} - {connection_time_ms}ms'
+DEFAULT_HTTP_PROXY_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \
+    '{request_method} {server_host}:{server_port}{request_path} - ' + \
+    '{response_code} {response_reason} - {response_bytes} bytes - ' + \
+    '{connection_time_ms}ms'
+DEFAULT_HTTPS_PROXY_ACCESS_LOG_FORMAT = '{client_ip}:{client_port} - ' + \
+    '{request_method} {server_host}:{server_port} - ' + \
+    '{response_bytes} bytes - {connection_time_ms}ms'
+DEFAULT_NUM_ACCEPTORS = 0
 DEFAULT_NUM_WORKERS = 0
 DEFAULT_OPEN_FILE_LIMIT = 1024
 DEFAULT_PAC_FILE = None
 DEFAULT_PAC_FILE_URL_PATH = b'/'
 DEFAULT_PID_FILE = None
-DEFAULT_PLUGINS = ''
+DEFAULT_PORT_FILE = None
+DEFAULT_PLUGINS: List[Any] = []
 DEFAULT_PORT = 8899
 DEFAULT_SERVER_RECVBUF_SIZE = DEFAULT_BUFFER_SIZE
 DEFAULT_STATIC_SERVER_DIR = os.path.join(PROXY_PY_DIR, "public")
-DEFAULT_THREADLESS = False
-DEFAULT_TIMEOUT = 10
+DEFAULT_MIN_COMPRESSION_LENGTH = 20  # In bytes
+DEFAULT_THREADLESS = _env_threadless_compliant()
+DEFAULT_LOCAL_EXECUTOR = True
+DEFAULT_TIMEOUT = 10.0
 DEFAULT_VERSION = False
 DEFAULT_HTTP_PORT = 80
-DEFAULT_MAX_SEND_SIZE = 16 * 1024
+DEFAULT_HTTPS_PORT = 443
+DEFAULT_WORK_KLASS = 'proxy.http.HttpProtocolHandler'
+DEFAULT_ENABLE_PROXY_PROTOCOL = False
+# 25 milliseconds to keep the loops hot
+# Will consume ~0.3-0.6% CPU when idle.
+DEFAULT_SELECTOR_SELECT_TIMEOUT = 25 / 1000
+DEFAULT_WAIT_FOR_TASKS_TIMEOUT = 1 / 1000
+DEFAULT_INACTIVE_CONN_CLEANUP_TIMEOUT = 1   # in seconds
 
 DEFAULT_DATA_DIRECTORY_PATH = os.path.join(str(pathlib.Path.home()), '.proxy')
+DEFAULT_CACHE_DIRECTORY_PATH = os.path.join(
+    DEFAULT_DATA_DIRECTORY_PATH, 'cache',
+)
+DEFAULT_CACHE_REQUESTS = False
+DEFAULT_CACHE_BY_CONTENT_TYPE = False
 
 # Cor plugins enabled by default or via flags
+DEFAULT_ABC_PLUGINS = [
+    'HttpProtocolHandlerPlugin',
+    'HttpProxyBasePlugin',
+    'HttpWebServerBasePlugin',
+    'WebSocketTransportBasePlugin',
+]
 PLUGIN_HTTP_PROXY = 'proxy.http.proxy.HttpProxyPlugin'
 PLUGIN_WEB_SERVER = 'proxy.http.server.HttpWebServerPlugin'
+PLUGIN_PAC_FILE = 'proxy.http.server.HttpWebServerPacFilePlugin'
+PLUGIN_WEBSOCKET_TRANSPORT = 'proxy.http.websocket.transport.WebSocketTransport'
+
+PY2_DEPRECATION_MESSAGE = '''DEPRECATION: proxy.py no longer supports Python 2.7.  Kindly upgrade to Python 3+. '
+                'If for some reasons you cannot upgrade, use'
+                '"pip install proxy.py==0.3".'''