From 1eb9ec6a1dd7a3b11a101c15907196c1cf841251 Mon Sep 17 00:00:00 2001 From: halfak Date: Mon, 2 Nov 2015 15:35:11 -0600 Subject: [PATCH 01/26] Makes requests.Session easier to configure by passing 'session_params' to constructor. --- mwapi/session.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mwapi/session.py b/mwapi/session.py index c6b1836..a9e9fbe 100644 --- a/mwapi/session.py +++ b/mwapi/session.py @@ -52,12 +52,15 @@ class Session: """ def __init__(self, host, user_agent=None, api_path=None, - timeout=None, session=None): + timeout=None, session=None, **session_params): self.host = str(host) self.api_path = str(api_path or "/w/api.php") self.api_url = self.host + self.api_path self.timeout = float(timeout) if timeout is not None else None self.session = session or requests.Session() + for key, value in session_params.items(): + setattr(self.session, key, value) + self.headers = {} if user_agent is None: @@ -83,6 +86,7 @@ def _request(self, method, params=None, files=None): data=data, files=files, timeout=self.timeout, headers=self.headers, + verify=True, stream=True) except requests.exceptions.Timeout as e: raise TimeoutError(str(e)) from e From d6e9ff92844dcfa2cc540dde6fc67b0197880fd6 Mon Sep 17 00:00:00 2001 From: halfak Date: Mon, 2 Nov 2015 15:54:47 -0600 Subject: [PATCH 02/26] Adds support for passing an auth object around -- critical for use of OAuth (mwoauth) --- demo_mwoauth.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ mwapi/session.py | 38 +++++++++++++++++++++++++------------- 2 files changed, 73 insertions(+), 13 deletions(-) create mode 100644 demo_mwoauth.py diff --git a/demo_mwoauth.py b/demo_mwoauth.py new file mode 100644 index 0000000..0261de8 --- /dev/null +++ b/demo_mwoauth.py @@ -0,0 +1,48 @@ +import mwapi +import mwoauth +from requests_oauthlib import OAuth1 + +# Consruct a "consumer" from the key/secret provided by MediaWiki +import config # You'll need to provide this +from six.moves import input # For compatibility between python 2 and 3 + +consumer_token = mwoauth.ConsumerToken(config.consumer_key, + config.consumer_secret) + +# Construct handshaker with wiki URI and consumer +handshaker = mwoauth.Handshaker("https://en.wikipedia.org/w/index.php", + consumer_token) + +# Step 1: Initialize -- ask MediaWiki for a temporary key/secret for user +redirect, request_token = handshaker.initiate() + +# Step 2: Authorize -- send user to MediaWiki to confirm authorization +print("Point your browser to: %s" % redirect) # +response_qs = input("Response query string: ") + +# Step 3: Complete -- obtain authorized key/secret for "resource owner" +access_token = handshaker.complete(request_token, response_qs) + +# Construct an auth object with the consumer and access tokens +auth1 = OAuth1(consumer_token.key, + client_secret=consumer_token.secret, + resource_owner_key=access_token.key, + resource_owner_secret=access_token.secret) + +# Construct an mwapi session. Nothing special here. +session = mwapi.Session( + host="https://en.wikipedia.org", + user_agent="mwoauth demo script -- ahalfaker@wikimedia.org") + +# Now, accessing the API on behalf of a user +print("Reading top 10 watchlist items") +response = session.get( + action="query", + list="watchlist", + wllimit=10, + wlprop="title|comment", + format="json", + auth=auth1 +) +for item in response.json()['query']['watchlist']: + print("{title}\t{comment}".format(**item)) diff --git a/mwapi/session.py b/mwapi/session.py index a9e9fbe..fa36355 100644 --- a/mwapi/session.py +++ b/mwapi/session.py @@ -71,7 +71,7 @@ def __init__(self, host, user_agent=None, api_path=None, else: self.headers['User-Agent'] = user_agent - def _request(self, method, params=None, files=None): + def _request(self, method, params=None, files=None, auth=None): if method.lower() == "post": data = params data['format'] = "json" @@ -87,7 +87,8 @@ def _request(self, method, params=None, files=None): timeout=self.timeout, headers=self.headers, verify=True, - stream=True) + stream=True, + auth=auth) except requests.exceptions.Timeout as e: raise TimeoutError(str(e)) from e except requests.exceptions.ConnectionError as e: @@ -123,7 +124,7 @@ def _request(self, method, params=None, files=None): return doc def request(self, method, params=None, query_continue=None, - files=None, continuation=False): + files=None, auth=None, continuation=False): """ Sends an HTTP request to the API. @@ -141,6 +142,8 @@ def request(self, method, params=None, query_continue=None, files : `dict` A dictionary of (filename : `str`, data : `bytes`) pairs to send with the request. + auth : mixed + Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. continuation : `bool` If true, a continuation will be attempted and a generator of JSON response documents will be returned. @@ -151,12 +154,14 @@ def request(self, method, params=None, query_continue=None, """ normal_params = _normalize_params(params, query_continue) if continuation: - return self._continuation(method, params=normal_params, files=files) + return self._continuation(method, params=normal_params, auth=auth, + files=files) else: - return self._request(method, params=normal_params, files=files) + return self._request(method, params=normal_params, auth=auth, + files=files) def continuation(self, method, params=None, query_continue=None, - files=None): + auth=None, files=None): """ Makes a request and, if the response calls for a continuation, performs that continuation. @@ -172,6 +177,8 @@ def continuation(self, method, params=None, query_continue=None, files : `dict` A dictionary of (filename : `str`, data : `bytes`) pairs to send with the initial request. + auth : mixed + Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. query_continue : `dict` A 'continue' field from a past request. This field represents the point from which a query should be continued. @@ -180,12 +187,12 @@ def continuation(self, method, params=None, query_continue=None, A generator of response JSON documents. """ - def _continuation(self, method, params=None, files=None): + def _continuation(self, method, params=None, files=None, auth=None): if 'continue' not in params: params['continue'] = '' while True: - doc = self._request(method, params=params, files=files) + doc = self._request(method, params=params, files=files, auth=None) yield doc if 'continue' not in doc: break @@ -233,12 +240,15 @@ def logout(self): """ self.post(action='logout') - def get(self, query_continue=None, continuation=False, **params): + def get(self, query_continue=None, auth=None, continuation=False, + **params): """Makes an API request with the GET method :Parameters: query_continue : `dict` Optionally, the value of a query continuation 'continue' field. + auth : mixed + Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. continuation : `bool` If true, a continuation will be attempted and a generator of JSON response documents will be returned. @@ -253,12 +263,12 @@ def get(self, query_continue=None, continuation=False, **params): :class:`mwapi.errors.APIError` : if the API responds with an error """ - return self.request('GET', params=params, + return self.request('GET', params=params, auth=auth, query_continue=query_continue, continuation=continuation) - def post(self, query_continue=None, upload_file=None, continuation=False, - **params): + def post(self, query_continue=None, upload_file=None, auth=None, + continuation=False, **params): """Makes an API request with the POST method :Parameters: @@ -266,6 +276,8 @@ def post(self, query_continue=None, upload_file=None, continuation=False, Optionally, the value of a query continuation 'continue' field. upload_file : `bytes` The bytes of a file to upload. + auth : mixed + Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. continuation : `bool` If true, a continuation will be attempted and a generator of JSON response documents will be returned. @@ -284,7 +296,7 @@ def post(self, query_continue=None, upload_file=None, continuation=False, else: files = None - return self.request('POST', params=params, + return self.request('POST', params=params, auth=auth, query_continue=query_continue, files=files, continuation=continuation) From a3fd6763e7a83efc5d97c21821403643e4e3285b Mon Sep 17 00:00:00 2001 From: halfak Date: Fri, 18 Dec 2015 09:06:40 -0600 Subject: [PATCH 03/26] Increments version to 0.4.0 --- mwapi/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mwapi/__init__.py b/mwapi/__init__.py index f8fee03..b796d2e 100644 --- a/mwapi/__init__.py +++ b/mwapi/__init__.py @@ -19,4 +19,4 @@ __all__ = [MWApi, Session] -__version__ = "0.3.1" +__version__ = "0.4.0" diff --git a/setup.py b/setup.py index 72bc2c2..d279580 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="mwapi", - version="0.3.1", # Change in mwapi/__init__.py + version="0.4.0", # Change in mwapi/__init__.py author="Yuvi Panda", author_email="yuvipanda@gmail.com", url="http://github.com/yuvipanda/python-mwapi", From 8f14fd72461f28e32264c68321ca3a0a3fb2a7e2 Mon Sep 17 00:00:00 2001 From: halfak Date: Thu, 14 Jan 2016 18:53:26 -0600 Subject: [PATCH 04/26] Hides config.py in gitignore (because it contains secret keys) --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 5dbc07e..c2e9910 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,10 @@ __pycache__/ *.py[cod] +# Secret data +config.py + + # Temporary text editor files *~ From 12d4998a3d583e3403a5254b2611a53326438ae7 Mon Sep 17 00:00:00 2001 From: halfak Date: Thu, 14 Jan 2016 19:00:09 -0600 Subject: [PATCH 05/26] Adds support for formatversion in Session constructor --- demo_queries.py | 8 +++++--- mwapi/session.py | 11 ++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/demo_queries.py b/demo_queries.py index f1d9873..040a997 100644 --- a/demo_queries.py +++ b/demo_queries.py @@ -21,7 +21,9 @@ import mwapi.errors my_agent = 'mwapi demo script ' -session = mwapi.Session('https://en.wikipedia.org', user_agent=my_agent) +session = mwapi.Session('https://en.wikipedia.org', + formatversion=2, + user_agent=my_agent) print("Logging into English Wikipedia") session.login(input("Username: "), getpass.getpass("Password: ")) @@ -41,7 +43,7 @@ def query_revisions_by_revids(revids, batch=50, **params): doc = session.post(action='query', prop='revisions', revids=batch_ids, **params) - for page_doc in doc['query'].get('pages', {}).values(): + for page_doc in doc['query']['pages']: page_meta = {k: v for k, v in page_doc.items() if k != 'revisions'} if 'revisions' in page_doc: @@ -66,7 +68,7 @@ def query_revisions(title=None, pageid=None, batch=50, limit=50, continuation=True, **params) for doc in response_docs: - for page_doc in doc['query'].get('pages', {}).values(): + for page_doc in doc['query']['pages']: page_meta = {k: v for k, v in page_doc.items() if k != 'revisions'} if 'revisions' in page_doc: for revision_doc in page_doc['revisions']: diff --git a/mwapi/session.py b/mwapi/session.py index fa36355..30684a9 100644 --- a/mwapi/session.py +++ b/mwapi/session.py @@ -39,6 +39,8 @@ class Session: The User-Agent header to include with all requests. Use this field to identify your script/bot/application to system admins of the MediaWiki API you are using. + formatversion : int + The formatversion to supply to the API for all requests. api_path : `str` The path to "api.php" on the server -- must begin with "/". timeout : `float` @@ -51,9 +53,12 @@ class Session: (optional) a `requests` session object to use """ - def __init__(self, host, user_agent=None, api_path=None, + def __init__(self, host, user_agent=None, formatversion=None, + api_path=None, timeout=None, session=None, **session_params): self.host = str(host) + self.formatversion = int(formatversion) if formatversion is not None \ + else None self.api_path = str(api_path or "/w/api.php") self.api_url = self.host + self.api_path self.timeout = float(timeout) if timeout is not None else None @@ -72,6 +77,10 @@ def __init__(self, host, user_agent=None, api_path=None, self.headers['User-Agent'] = user_agent def _request(self, method, params=None, files=None, auth=None): + params = params or {} + if self.formatversion is not None: + params['formatversion'] = self.formatversion + if method.lower() == "post": data = params data['format'] = "json" From 691d16aea311aabfa9189ebc939c6dd5d1fa879d Mon Sep 17 00:00:00 2001 From: halfak Date: Fri, 6 May 2016 17:23:55 -0500 Subject: [PATCH 06/26] Increments version to 0.4.1 --- mwapi/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mwapi/__init__.py b/mwapi/__init__.py index b796d2e..3ab3c5f 100644 --- a/mwapi/__init__.py +++ b/mwapi/__init__.py @@ -19,4 +19,4 @@ __all__ = [MWApi, Session] -__version__ = "0.4.0" +__version__ = "0.4.1" diff --git a/setup.py b/setup.py index d279580..a442e9e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="mwapi", - version="0.4.0", # Change in mwapi/__init__.py + version="0.4.1", # Change in mwapi/__init__.py author="Yuvi Panda", author_email="yuvipanda@gmail.com", url="http://github.com/yuvipanda/python-mwapi", From 2877251fbc3df6d389ae2ca1ebcfddb1844af3de Mon Sep 17 00:00:00 2001 From: halfak Date: Sat, 26 Nov 2016 14:24:34 -0600 Subject: [PATCH 07/26] adds support for OATH two factor --- demo_queries.py | 4 ++-- mwapi/cli.py | 38 ++++++++++++++++++++++++++++++++++++++ mwapi/errors.py | 18 +++++++++++++++++- mwapi/session.py | 48 +++++++++++++++++++++++++++++++++--------------- 4 files changed, 90 insertions(+), 18 deletions(-) create mode 100644 mwapi/cli.py diff --git a/demo_queries.py b/demo_queries.py index 040a997..f19d25b 100644 --- a/demo_queries.py +++ b/demo_queries.py @@ -18,6 +18,7 @@ from itertools import islice import mwapi +import mwapi.cli import mwapi.errors my_agent = 'mwapi demo script ' @@ -25,8 +26,7 @@ formatversion=2, user_agent=my_agent) -print("Logging into English Wikipedia") -session.login(input("Username: "), getpass.getpass("Password: ")) +mwapi.cli.do_login(session, 'https://en.wikipedia.org') print("whoami?") print("\t", session.get(action='query', meta='userinfo'), "\n") diff --git a/mwapi/cli.py b/mwapi/cli.py new file mode 100644 index 0000000..1dea7b9 --- /dev/null +++ b/mwapi/cli.py @@ -0,0 +1,38 @@ +import getpass +import sys + +from .errors import ClientInteractionRequest + + +def do_login(session, for_what): + username, password = request_username_password(for_what) + try: + session.login(username, password) + except ClientInteractionRequest as cir: + params = request_interaction(cir) + session.continue_login(cir.login_token, **params) + +def request_interaction(cir): + sys.stderr.write("{0}\n".format(cir.message)) + + params = {} + for req_doc in cir.requests: + # sys.stderr.write("id: {0}\n".format(req_doc['id'])) + for name, field in req_doc['fields'].items(): + prefix = "{0}({1}): ".format(field['label'], name) + if field['sensitive']: + value = getpass.getpass(prefix) + else: + sys.stderr.write(prefix) + sys.stderr.flush() + value = open('/dev/tty').readline().strip() + + params[name] = value + + return params + +def request_username_password(for_what): + sys.stderr.write("Log into " + for_what + "\n") + sys.stderr.write("Username: ") + sys.stderr.flush() + return open('/dev/tty').readline().strip(), getpass.getpass("Password: ") diff --git a/mwapi/errors.py b/mwapi/errors.py index abea912..81a096b 100644 --- a/mwapi/errors.py +++ b/mwapi/errors.py @@ -47,7 +47,23 @@ class LoginError(RuntimeError): @classmethod def from_doc(cls, doc): - return cls(doc.get('result')) + return cls(doc.get('status')) + + +class ClientInteractionRequest(RuntimeError): + """ + Thrown when user input is needed to log in. + """ + + def __init__(self, login_token, message, requests): + super().__init__((login_token, message, requests)) + self.login_token = login_token + self.message = message + self.requests = requests + + @classmethod + def from_doc(cls, login_token, doc): + return cls(login_token, doc.get('message'), doc.get('requests', [])) class RequestError(requests.exceptions.RequestException): diff --git a/mwapi/session.py b/mwapi/session.py index 30684a9..0936ae2 100644 --- a/mwapi/session.py +++ b/mwapi/session.py @@ -19,8 +19,9 @@ import requests import requests.exceptions -from .errors import (APIError, ConnectionError, HTTPError, LoginError, - RequestError, TimeoutError, TooManyRedirectsError) +from .errors import (APIError, ClientInteractionRequest, ConnectionError, + HTTPError, LoginError, RequestError, TimeoutError, + TooManyRedirectsError) DEFAULT_USERAGENT = "mwapi (python) -- default user-agent" @@ -57,8 +58,8 @@ def __init__(self, host, user_agent=None, formatversion=None, api_path=None, timeout=None, session=None, **session_params): self.host = str(host) - self.formatversion = int(formatversion) if formatversion is not None \ - else None + self.formatversion = int(formatversion) \ + if formatversion is not None else None self.api_path = str(api_path or "/w/api.php") self.api_url = self.host + self.api_path self.timeout = float(timeout) if timeout is not None else None @@ -209,7 +210,7 @@ def _continuation(self, method, params=None, files=None, auth=None): params.update(doc['continue']) files = None # Don't send files again - def login(self, username, password): + def login(self, username, password, login_token=None): """ Authenticate with the given credentials. If authentication is successful, all further requests sent will be signed the authenticated @@ -228,17 +229,34 @@ def login(self, username, password): :class:`mwapi.errors.LoginError` : if authentication fails :class:`mwapi.errors.APIError` : if the API responds with an error """ - token_doc = self.post(action="login", lgname=username, - lgpassword=password) + if login_token is None: + token_doc = self.post(action='query', meta='tokens', type='login') + login_token = token_doc['query']['tokens']['logintoken'] + + login_doc = self.post( + action="clientlogin", username=username, password=password, + logintoken=login_token, loginreturnurl="http://example.org/") + + if login_doc['clientlogin']['status'] == "UI": + raise ClientInteractionRequest.from_doc( + login_token, login_doc['clientlogin']) + elif login_doc['clientlogin']['status'] != 'PASS': + raise LoginError.from_doc(login_doc['clientlogin']) + return login_doc['clientlogin'] + + def continue_login(self, login_token=None, **params): + + login_params = { + 'action': "clientlogin", + 'logintoken': login_token, + 'logincontinue': 1 + } + login_params.update(params) + login_doc = self.post(**login_params) + if login_doc['clientlogin']['status'] != 'PASS': + raise LoginError.from_doc(login_doc['clientlogin']) + return login_doc['clientlogin'] - login_doc = self.post(action="login", lgname=username, - lgpassword=password, - lgtoken=token_doc['login']['token']) - - result = login_doc['login']['result'] - if result != 'Success': - raise LoginError.from_doc(login_doc['login']) - return result def logout(self): """ From 8ab013c6776ae8a0c2d7522d7eebb1964580bb1c Mon Sep 17 00:00:00 2001 From: halfak Date: Sat, 26 Nov 2016 14:27:25 -0600 Subject: [PATCH 08/26] Minor flake8 fixes. --- mwapi/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mwapi/cli.py b/mwapi/cli.py index 1dea7b9..3b04a4b 100644 --- a/mwapi/cli.py +++ b/mwapi/cli.py @@ -12,6 +12,7 @@ def do_login(session, for_what): params = request_interaction(cir) session.continue_login(cir.login_token, **params) + def request_interaction(cir): sys.stderr.write("{0}\n".format(cir.message)) @@ -31,6 +32,7 @@ def request_interaction(cir): return params + def request_username_password(for_what): sys.stderr.write("Log into " + for_what + "\n") sys.stderr.write("Username: ") From 3ea0d8f05989ea3950ab3d2cf315a28ab83a29f6 Mon Sep 17 00:00:00 2001 From: halfak Date: Sat, 26 Nov 2016 14:34:21 -0600 Subject: [PATCH 09/26] More verbose login error message. --- mwapi/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mwapi/errors.py b/mwapi/errors.py index 81a096b..e0fadd3 100644 --- a/mwapi/errors.py +++ b/mwapi/errors.py @@ -47,7 +47,7 @@ class LoginError(RuntimeError): @classmethod def from_doc(cls, doc): - return cls(doc.get('status')) + return cls(doc.get('status') + " -- " + doc.get('message')) class ClientInteractionRequest(RuntimeError): From 7a60d34b3195341cb4474fee6905197df3c77e84 Mon Sep 17 00:00:00 2001 From: halfak Date: Sat, 26 Nov 2016 14:41:52 -0600 Subject: [PATCH 10/26] Centralizes metadata pattern with about.py --- mwapi/__init__.py | 8 +++++--- mwapi/about.py | 10 ++++++++++ setup.py | 17 +++++++++++------ 3 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 mwapi/about.py diff --git a/mwapi/__init__.py b/mwapi/__init__.py index 3ab3c5f..64807bd 100644 --- a/mwapi/__init__.py +++ b/mwapi/__init__.py @@ -13,10 +13,12 @@ :License: MIT """ from .session import Session +from .about import (__name__, __version__, __author__, __author_email__, + __description__, __license__, __url__) MWApi = Session -__all__ = [MWApi, Session] - -__version__ = "0.4.1" +__all__ = [MWApi, Session, + __name__, __version__, __author__, __author_email__, + __description__, __license__, __url__] diff --git a/mwapi/about.py b/mwapi/about.py new file mode 100644 index 0000000..1baaf9e --- /dev/null +++ b/mwapi/about.py @@ -0,0 +1,10 @@ +__name__ = "mwapi" +__version__ = "0.4.1" +__author__ = "Aaron Halfaker" +__author_email__ = "aaron.halfaker@gmail.com" +__description__ = "Simple wrapper for the Mediawiki API" +__license__ = "MIT" +__url__ = "https://github.com/mediawiki-utilities/python-mwapi" + +all = [__name__, __version__, __author__, __author_email__, __description__, + __license__, __url__] diff --git a/setup.py b/setup.py index a442e9e..d799175 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,19 @@ +import os.path + from setuptools import setup +about_path = os.path.join(os.path.dirname(__file__), "mwapi/about.py") +exec(compile(open(about_path).read(), about_path, "exec")) + setup( - name="mwapi", - version="0.4.1", # Change in mwapi/__init__.py - author="Yuvi Panda", - author_email="yuvipanda@gmail.com", - url="http://github.com/yuvipanda/python-mwapi", + name=__name__, # noqa + version=__version__, # noqa + author=__author__, # noqa + author_email=__author_email__, # noqa + description=__description__, # noqa + url=__url__, # noqa packages=["mwapi"], license=open("LICENSE").read(), - description="Simple wrapper for the Mediawiki API", long_description=open("README.md").read(), install_requires=["requests"] ) From a5570a31b42530869481696a8fa59e0a550af294 Mon Sep 17 00:00:00 2001 From: halfak Date: Sat, 26 Nov 2016 14:42:28 -0600 Subject: [PATCH 11/26] Increments version to 0.5.0 --- mwapi/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mwapi/about.py b/mwapi/about.py index 1baaf9e..42029c6 100644 --- a/mwapi/about.py +++ b/mwapi/about.py @@ -1,5 +1,5 @@ __name__ = "mwapi" -__version__ = "0.4.1" +__version__ = "0.5.0" __author__ = "Aaron Halfaker" __author_email__ = "aaron.halfaker@gmail.com" __description__ = "Simple wrapper for the Mediawiki API" From 9522c8c9374d402a718b63d74ecb11391aac1094 Mon Sep 17 00:00:00 2001 From: halfak Date: Mon, 28 Nov 2016 17:30:17 -0600 Subject: [PATCH 12/26] Updates docs. --- doc/cli.rst | 1 + doc/index.rst | 1 + 2 files changed, 2 insertions(+) create mode 100644 doc/cli.rst diff --git a/doc/cli.rst b/doc/cli.rst new file mode 100644 index 0000000..958baf4 --- /dev/null +++ b/doc/cli.rst @@ -0,0 +1 @@ +.. automodule:: mwapi.cli diff --git a/doc/index.rst b/doc/index.rst index 5460620..9faa772 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -21,6 +21,7 @@ Contents session errors + cli Authors ------- From 8afe1276ad11343801d5221d80d2d913c7760fe2 Mon Sep 17 00:00:00 2001 From: halfak Date: Wed, 14 Dec 2016 13:24:01 -0600 Subject: [PATCH 13/26] Improvements to docs --- mwapi/cli.py | 31 ++++++++++++++++++++++++++++++- mwapi/session.py | 18 ++++++++++++++++-- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/mwapi/cli.py b/mwapi/cli.py index 3b04a4b..61ecdd0 100644 --- a/mwapi/cli.py +++ b/mwapi/cli.py @@ -1,3 +1,12 @@ +""" +Command-line Interface (cli) +============================ + +This module provides utilities for interacting with a user from the +command-line. + +.. autofunction:: mwapi.cli.do_login +""" import getpass import sys @@ -5,6 +14,26 @@ def do_login(session, for_what): + """ + Performs a login handshake with a user on the command-line. This method + will handle all of the follow-up requests (e.g. capcha or two-factor). A + login that requires two-factor looks like this:: + + >>> import mwapi.cli + >>> import mwapi + >>> mwapi.cli.do_login(mwapi.Session("https://en.wikipedia.org"), "English Wikipedia") + Log into English Wikipedia + Username: Halfak (WMF) + Passord: + Please enter verification code from your mobile app + Token(OATHToken): 234567 + + :Parameters: + session : :class:`mwapi.Session` + A session object to use for login + for_what : `str` + A name to display to the use (for what they are logging into) + """ # noqa username, password = request_username_password(for_what) try: session.login(username, password) @@ -21,7 +50,7 @@ def request_interaction(cir): # sys.stderr.write("id: {0}\n".format(req_doc['id'])) for name, field in req_doc['fields'].items(): prefix = "{0}({1}): ".format(field['label'], name) - if field['sensitive']: + if field.get('sensitive', False): value = getpass.getpass(prefix) else: sys.stderr.write(prefix) diff --git a/mwapi/session.py b/mwapi/session.py index 0936ae2..39385f3 100644 --- a/mwapi/session.py +++ b/mwapi/session.py @@ -227,6 +227,7 @@ def login(self, username, password, login_token=None): :Raises: :class:`mwapi.errors.LoginError` : if authentication fails + :class:`mwapi.errors.ClientInteractionRequest` : if authentication requires a continue_login() call :class:`mwapi.errors.APIError` : if the API responds with an error """ if login_token is None: @@ -244,7 +245,21 @@ def login(self, username, password, login_token=None): raise LoginError.from_doc(login_doc['clientlogin']) return login_doc['clientlogin'] - def continue_login(self, login_token=None, **params): + def continue_login(self, login_token, **params): + """ + Continues a login that requires an additional step. This is common + for when login requires completing a captcha or supplying a two-factor + authentication token. + + :Parameters: + login_token : `str` + A login token generated by the MediaWiki API (and used in a + previous call to login()) + params : `mixed` + A set of parameters to include with the request. This depends + on what "requests" for additional information were made by the + MediaWiki API. + """ login_params = { 'action': "clientlogin", @@ -257,7 +272,6 @@ def continue_login(self, login_token=None, **params): raise LoginError.from_doc(login_doc['clientlogin']) return login_doc['clientlogin'] - def logout(self): """ Logs out of the session with MediaWiki From f78c1baae64412c852f720fa5944615f60b835e4 Mon Sep 17 00:00:00 2001 From: halfak Date: Wed, 14 Dec 2016 13:24:24 -0600 Subject: [PATCH 14/26] Increments version to 0.5.1 --- mwapi/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mwapi/about.py b/mwapi/about.py index 42029c6..e570142 100644 --- a/mwapi/about.py +++ b/mwapi/about.py @@ -1,5 +1,5 @@ __name__ = "mwapi" -__version__ = "0.5.0" +__version__ = "0.5.1" __author__ = "Aaron Halfaker" __author_email__ = "aaron.halfaker@gmail.com" __description__ = "Simple wrapper for the Mediawiki API" From 756b815cceeb77eca4128ca642e8046a6dee1f0d Mon Sep 17 00:00:00 2001 From: halfak Date: Fri, 17 Mar 2017 10:05:38 -0500 Subject: [PATCH 15/26] Uses __license__ in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d799175..f39a476 100644 --- a/setup.py +++ b/setup.py @@ -12,8 +12,8 @@ author_email=__author_email__, # noqa description=__description__, # noqa url=__url__, # noqa + license=__license__, # noqa packages=["mwapi"], - license=open("LICENSE").read(), long_description=open("README.md").read(), install_requires=["requests"] ) From 98ef12347321c431dddee31c9249087c14411cd9 Mon Sep 17 00:00:00 2001 From: Lucas Werkmeister Date: Sun, 28 Apr 2019 23:01:24 +0200 Subject: [PATCH 16/26] Add support for boolean and optional parameters True parameters are turned into the empty string, and False and None parameters are completely removed from the params dict. Fixes #37. --- mwapi/session.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mwapi/session.py b/mwapi/session.py index 39385f3..9879ebf 100644 --- a/mwapi/session.py +++ b/mwapi/session.py @@ -345,6 +345,8 @@ def post(self, query_continue=None, upload_file=None, auth=None, def _normalize_value(value): if isinstance(value, str): return value + elif isinstance(value, bool): + return "" if value else None elif hasattr(value, "__iter__"): return "|".join(str(v) for v in value) else: @@ -353,6 +355,7 @@ def _normalize_value(value): def _normalize_params(params, query_continue=None): normal_params = {k: _normalize_value(v) for k, v in params.items()} + normal_params = {k: v for k, v in normal_params.items() if v is not None} if query_continue is not None: normal_params.update(query_continue) From 92f90356c5a0a934f6f5e61480a9011ce25c91ac Mon Sep 17 00:00:00 2001 From: Amir Sarabadani Date: Mon, 29 Apr 2019 05:26:00 +0200 Subject: [PATCH 17/26] Fix flake8 --- demo_queries.py | 2 +- mwapi/session.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/demo_queries.py b/demo_queries.py index f19d25b..1197d12 100644 --- a/demo_queries.py +++ b/demo_queries.py @@ -13,7 +13,6 @@ 5. Cause the API to throw an error and catch it. """ -import getpass import sys from itertools import islice @@ -82,6 +81,7 @@ def query_revisions(title=None, pageid=None, batch=50, limit=50, if yielded >= limit: break + print("Querying by title") rev_ids = [] sys.stdout.write("\t ") diff --git a/mwapi/session.py b/mwapi/session.py index 9879ebf..763c008 100644 --- a/mwapi/session.py +++ b/mwapi/session.py @@ -59,7 +59,7 @@ def __init__(self, host, user_agent=None, formatversion=None, timeout=None, session=None, **session_params): self.host = str(host) self.formatversion = int(formatversion) \ - if formatversion is not None else None + if formatversion is not None else None self.api_path = str(api_path or "/w/api.php") self.api_url = self.host + self.api_path self.timeout = float(timeout) if timeout is not None else None From eba8eb09d353b3f7755a13824aa6e3f9de761d3d Mon Sep 17 00:00:00 2001 From: Yury Bulka Date: Sat, 18 May 2019 11:01:27 +0200 Subject: [PATCH 18/26] Add a continuation example to README --- README.md | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f7bfd1..8ab76b4 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,9 @@ and that your wiki is using MediaWiki 1.15.3 or greater. * **Repositiory:** https://github.com/mediawiki-utilities/python-mwapi * **License:** MIT -## Example +## Examples + +### Single query >>> import mwapi >>> @@ -27,6 +29,39 @@ and that your wiki is using MediaWiki 1.15.3 or greater. 'timestamp': '2005-12-23T00:07:17Z'}], 'title': 'Grigol Ordzhonikidze', 'pageid': 1429626}}}, 'batchcomplete': ''} +### Query with continuation + +```python +import mwapi +from mwapi.errors import APIError + +session = mwapi.Session('https://en.wikipedia.org/') + +# If passed a `continuation` parameter, returns an iterable over a continued query. +# On each iteration, a new request is made for the next portion of the results. +continued = session.get( + formatversion=2, + action='query', + generator='categorymembers', + gcmtitle='Category:17th-century classical composers', + gcmlimit=100, # 100 results per request + continuation=True) + +pages = [] +for portion in continued: + try: + if 'query' in portion: + for page in portion['query']['pages']: + pages.append(page['title']) + else: + print("Mediwiki returned empty result batch.") + except APIError: + raise ValueError( + "MediaWiki returned an error:", str(APIError) + ) + +print("Fetched {} pages".format(len(pages))) +``` ## Authors * YuviPanda -- https://github.com/yuvipanda From 8d31bb3e2ed997d4a701e6b1fa544b2830a766f0 Mon Sep 17 00:00:00 2001 From: Lucas Werkmeister Date: Sun, 3 Nov 2019 13:53:32 +0100 Subject: [PATCH 19/26] Clean up README.md Fix a typo, update a link to HTTPS and lightly rephrase the first paragraph. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8ab76b4..9a2675e 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # MediaWiki API -This MIT Licensed library provides a very simple convenience wrapper -around the [MediaWiki API](http://www.mediawiki.org/wiki/API). and -includes support for authenticated sessions. It requires Python 3 +This MIT-licensed library provides a very simple convenience wrapper +around the [MediaWiki API](https://www.mediawiki.org/wiki/API), +including support for authenticated sessions. It requires Python 3 and that your wiki is using MediaWiki 1.15.3 or greater. * **Installation:** ``pip install mwapi`` * **Documentation:** https://pythonhosted.org/mwapi -* **Repositiory:** https://github.com/mediawiki-utilities/python-mwapi +* **Repository:** https://github.com/mediawiki-utilities/python-mwapi * **License:** MIT ## Examples From 19ef12d6538185cc3d91a0676f2f2543f0a436d0 Mon Sep 17 00:00:00 2001 From: Lucas Werkmeister Date: Sun, 3 Nov 2019 13:56:45 +0100 Subject: [PATCH 20/26] Fix another typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a2675e..5393371 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ for portion in continued: for page in portion['query']['pages']: pages.append(page['title']) else: - print("Mediwiki returned empty result batch.") + print("MediaWiki returned empty result batch.") except APIError: raise ValueError( "MediaWiki returned an error:", str(APIError) From d969ea72125b4e9fdfa54cedb30372319c5ff98c Mon Sep 17 00:00:00 2001 From: Yury Bulka Date: Sun, 3 Nov 2019 19:24:10 +0200 Subject: [PATCH 21/26] README: fix exception handling in continuation example Thanks @lucaswerkmeister --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8ab76b4..b17dc71 100644 --- a/README.md +++ b/README.md @@ -48,17 +48,17 @@ continued = session.get( continuation=True) pages = [] -for portion in continued: - try: +try: + for portion in continued: if 'query' in portion: for page in portion['query']['pages']: pages.append(page['title']) else: print("Mediwiki returned empty result batch.") - except APIError: - raise ValueError( - "MediaWiki returned an error:", str(APIError) - ) +except APIError as error: + raise ValueError( + "MediaWiki returned an error:", str(error) + ) print("Fetched {} pages".format(len(pages))) ``` From 7db6d8cec7228fab81e10018b6ed542f367614cc Mon Sep 17 00:00:00 2001 From: AikoChou Date: Wed, 6 Jul 2022 17:12:38 +0200 Subject: [PATCH 22/26] Add support for async session --- README.md | 63 ++++++++++++ mwapi/__init__.py | 3 +- mwapi/async_session.py | 213 +++++++++++++++++++++++++++++++++++++++++ mwapi/errors.py | 27 ++++-- mwapi/session.py | 22 +---- mwapi/util.py | 19 ++++ setup.py | 2 +- 7 files changed, 318 insertions(+), 31 deletions(-) create mode 100644 mwapi/async_session.py create mode 100644 mwapi/util.py diff --git a/README.md b/README.md index 0e3fb19..c0c14c4 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,69 @@ except APIError as error: print("Fetched {} pages".format(len(pages))) ``` +### Asynchronous single query + +```python +import asyncio +import aiohttp +import mwapi + +async def query(): + async with aiohttp.ClientSession() as s: + session = mwapi.AsyncSession( + 'https://en.wikipedia.org', + user_agent='mwapi async demo', + session=s) + response = await asyncio.create_task( + session.get(action='query', prop='revisions', revids=32423425) + ) + print(response) + +asyncio.run(query()) +``` + +### Asynchronous query with continuation + +```python +import asyncio +import aiohttp + +import mwapi +from mwapi.errors import APIError + +async def query(): + async with aiohttp.ClientSession() as s: + session = mwapi.AsyncSession( + 'https://en.wikipedia.org', + user_agent='mwapi async demo', + session=s) + + continued = await asyncio.create_task( + session.get( + formatversion=2, + action='query', + generator='categorymembers', + gcmtitle='Category:17th-century classical composers', + gcmlimit=100, # 100 results per request + continuation=True) + ) + pages = [] + try: + async for portion in continued: + if 'query' in portion: + for page in portion['query']['pages']: + pages.append(page['title']) + else: + print("MediaWiki returned empty result batch.") + except APIError as error: + raise ValueError( + "MediaWiki returned an error:", str(error) + ) + print("Fetched {} pages".format(len(pages))) + +asyncio.run(query()) +``` + ## Authors * YuviPanda -- https://github.com/yuvipanda * Aaron Halfaker -- https://github.com/halfak diff --git a/mwapi/__init__.py b/mwapi/__init__.py index 64807bd..8a945c3 100644 --- a/mwapi/__init__.py +++ b/mwapi/__init__.py @@ -13,12 +13,13 @@ :License: MIT """ from .session import Session +from .async_session import AsyncSession from .about import (__name__, __version__, __author__, __author_email__, __description__, __license__, __url__) MWApi = Session -__all__ = [MWApi, Session, +__all__ = [MWApi, Session, AsyncSession, __name__, __version__, __author__, __author_email__, __description__, __license__, __url__] diff --git a/mwapi/async_session.py b/mwapi/async_session.py new file mode 100644 index 0000000..7b22cd9 --- /dev/null +++ b/mwapi/async_session.py @@ -0,0 +1,213 @@ +import logging + +import asyncio +import aiohttp + +from .errors import (APIError, ConnectionError, RequestError, TimeoutError, + TooManyRedirectsError) +from .util import _normalize_params + +DEFAULT_USERAGENT = "mwapi (python) -- default user-agent" + +logger = logging.getLogger(__name__) + + +class AsyncSession: + """ + Constructs a new API asynchronous session. + + :Parameters: + host : `str` + Host to which to connect to. Must include http:// or https:// and + no trailing "/". + user_agent : `str` + The User-Agent header to include with all requests. Use this field + to identify your script/bot/application to system admins of the + MediaWiki API you are using. + formatversion : int + The formatversion to supply to the API for all requests. + api_path : `str` + The path to "api.php" on the server -- must begin with "/". + timeout : `float` + How long to wait for the server to send data before giving up + and raising an error ( + :class:`aiohttp.client_exceptions.ServerTimeoutError` or + :class:`asyncio.exceptions.TimeoutError`). + By default aiohttp uses a total 300 seconds (5min) timeout. + session : `aiohttp.ClientSession` + (optional) an `aiohttp` session object to use + """ + + def __init__(self, host, user_agent=None, formatversion=None, + api_path=None, + timeout=None, session=None, **session_params): + self.host = str(host) + self.formatversion = int(formatversion) \ + if formatversion is not None else None + self.api_path = str(api_path or "/w/api.php") + self.api_url = self.host + self.api_path + self.timeout = float(timeout) \ + if timeout is not None else aiohttp.ClientTimeout(total=300) + self.session = session or aiohttp.ClientSession() + for key, value in session_params.items(): + setattr(self.session, key, value) + + self.headers = {} + + if user_agent is None: + logger.warning("Sending requests with default User-Agent. " + + "Set 'user_agent' on mwapi.Session to quiet this " + + "message.") + self.headers['User-Agent'] = DEFAULT_USERAGENT + else: + self.headers['User-Agent'] = user_agent + + async def _request(self, method, params=None, auth=None): + params = params or {} + if self.formatversion is not None: + params['formatversion'] = self.formatversion + + if method.lower() == "post": + data = params + data['format'] = "json" + params = None + + else: + data = None + params = params or {} + params['format'] = "json" + + try: + async with self.session.request(method=method, url=self.api_url, + params=params, data=data, + timeout=self.timeout, + headers=self.headers, + verify_ssl=True, + auth=auth) as resp: + + doc = await resp.json() + + if 'error' in doc: + raise APIError.from_doc(doc['error']) + + if 'warnings' in doc: + logger.warning("The following query raised warnings: {0}" + .format(params or data)) + for module, warning in doc['warnings'].items(): + logger.warning("\t- {0} -- {1}" + .format(module, warning)) + return doc + + except (ValueError, aiohttp.ContentTypeError): + if resp is None: + prefix = "No response data" + else: + prefix = (await resp.text())[:350] + raise ValueError("Could not decode as JSON:\n{0}" + .format(prefix)) + except (aiohttp.ServerTimeoutError, + asyncio.exceptions.TimeoutError) as e: + raise TimeoutError(str(e)) from e + except aiohttp.ClientConnectionError as e: + raise ConnectionError(str(e)) from e + except aiohttp.TooManyRedirects as e: + raise TooManyRedirectsError(str(e)) from e + except Exception as e: + raise RequestError(str(e)) from e + + + async def request(self, method, params=None, query_continue=None, + auth=None, continuation=False): + """ + Sends an HTTP request to the API. + + :Parameters: + method : `str` + Which HTTP method to use for the request? + (Usually "POST" or "GET") + params : `dict` + A set of parameters to send with the request. These parameters + will be included in the POST body for post requests or a query + string otherwise. + query_continue : `dict` + A 'continue' field from a past request. This field represents + the point from which a query should be continued. + auth : mixed + Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. + continuation : `bool` + If true, a continuation will be attempted and a generator of + JSON response documents will be returned. + + :Returns: + A response JSON documents (or a generator of documents if + `continuation == True`) + """ + normal_params = _normalize_params(params, query_continue) + if continuation: + return self._continuation(method, params=normal_params, auth=auth) + else: + return await self._request(method, params=normal_params, auth=auth) + + async def _continuation(self, method, params=None, auth=None): + if "continue" not in params: + params["continue"] = "" + + while True: + doc = await self._request(method, params=params, auth=auth) + yield doc + if "continue" not in doc: + break + # re-send all continue values in the next call + params.update(doc["continue"]) + + async def get(self, query_continue=None, auth=None, continuation=False, + **params): + """Makes an API request with the GET method + + :Parameters: + query_continue : `dict` + Optionally, the value of a query continuation 'continue' field. + auth : mixed + Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. + continuation : `bool` + If true, a continuation will be attempted and a generator of + JSON response documents will be returned. + params : + Keyword parameters to be sent in the query string. + + :Returns: + A response JSON documents (or a generator of documents if + `continuation == True`) + + :Raises: + :class:`mwapi.errors.APIError` : if the API responds with an error + """ + return await self.request("GET", params=params, auth=auth, + query_continue=query_continue, + continuation=continuation) + + async def post(self, query_continue=None, auth=None, continuation=False, + **params): + """Makes an API request with the POST method + + :Parameters: + query_continue : `dict` + Optionally, the value of a query continuation 'continue' field. + auth : mixed + Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. + continuation : `bool` + If true, a continuation will be attempted and a generator of + JSON response documents will be returned. + params : + Keyword parameters to be sent in the POST message body. + + :Returns: + A response JSON documents (or a generator of documents if + `continuation == True`) + + :Raises: + :class:`mwapi.errors.APIError` : if the API responds with an error + """ + return await self.request("POST", params=params, auth=auth, + query_continue=query_continue, + continuation=continuation) diff --git a/mwapi/errors.py b/mwapi/errors.py index e0fadd3..3c21d76 100644 --- a/mwapi/errors.py +++ b/mwapi/errors.py @@ -17,6 +17,8 @@ .. autoclass:: TimeoutError """ import requests.exceptions +import aiohttp +import asyncio class APIError(RuntimeError): @@ -66,16 +68,19 @@ def from_doc(cls, login_token, doc): return cls(login_token, doc.get('message'), doc.get('requests', [])) -class RequestError(requests.exceptions.RequestException): +class RequestError(requests.exceptions.RequestException, + aiohttp.ClientError): """ - A generic error thrown by :mod:`requests`. + A generic error thrown by :mod:`requests` or `aiohttp`. """ pass -class ConnectionError(requests.exceptions.ConnectionError): +class ConnectionError(requests.exceptions.ConnectionError, + aiohttp.ClientConnectionError): """ - Handles a :class:`requests.exceptions.ConnectionError` + Handles a :class:`requests.exceptions.ConnectionError` or + :class:`aiohttp.ClientConnectionError`. """ pass @@ -87,15 +92,21 @@ class HTTPError(requests.exceptions.HTTPError): pass -class TooManyRedirectsError(requests.exceptions.TooManyRedirects): +class TooManyRedirectsError(requests.exceptions.TooManyRedirects, + aiohttp.TooManyRedirects): """ - Handles a :class:`requests.exceptions.TooManyRedirects` + Handles a :class:`requests.exceptions.TooManyRedirects` or + :class:`aiohttp.TooManyRedirects`. """ pass -class TimeoutError(requests.exceptions.Timeout): +class TimeoutError(requests.exceptions.Timeout, + aiohttp.ServerTimeoutError, + asyncio.exceptions.TimeoutError): """ - Handles a :class:`requests.exceptions.TimeoutError` + Handles a :class:`requests.exceptions.TimeoutError` or + :class:`aiohttp.ServerTimeoutError` or + :class:`asyncio.exceptions.TimeoutError`. """ pass diff --git a/mwapi/session.py b/mwapi/session.py index 763c008..8b5707b 100644 --- a/mwapi/session.py +++ b/mwapi/session.py @@ -22,6 +22,7 @@ from .errors import (APIError, ClientInteractionRequest, ConnectionError, HTTPError, LoginError, RequestError, TimeoutError, TooManyRedirectsError) +from .util import _normalize_params DEFAULT_USERAGENT = "mwapi (python) -- default user-agent" @@ -340,24 +341,3 @@ def post(self, query_continue=None, upload_file=None, auth=None, return self.request('POST', params=params, auth=auth, query_continue=query_continue, files=files, continuation=continuation) - - -def _normalize_value(value): - if isinstance(value, str): - return value - elif isinstance(value, bool): - return "" if value else None - elif hasattr(value, "__iter__"): - return "|".join(str(v) for v in value) - else: - return value - - -def _normalize_params(params, query_continue=None): - normal_params = {k: _normalize_value(v) for k, v in params.items()} - normal_params = {k: v for k, v in normal_params.items() if v is not None} - - if query_continue is not None: - normal_params.update(query_continue) - - return normal_params diff --git a/mwapi/util.py b/mwapi/util.py new file mode 100644 index 0000000..48321ca --- /dev/null +++ b/mwapi/util.py @@ -0,0 +1,19 @@ +def _normalize_value(value): + if isinstance(value, str): + return value + elif isinstance(value, bool): + return "" if value else None + elif hasattr(value, "__iter__"): + return "|".join(str(v) for v in value) + else: + return value + + +def _normalize_params(params, query_continue=None): + normal_params = {k: _normalize_value(v) for k, v in params.items()} + normal_params = {k: v for k, v in normal_params.items() if v is not None} + + if query_continue is not None: + normal_params.update(query_continue) + + return normal_params diff --git a/setup.py b/setup.py index f39a476..ed37f12 100644 --- a/setup.py +++ b/setup.py @@ -15,5 +15,5 @@ license=__license__, # noqa packages=["mwapi"], long_description=open("README.md").read(), - install_requires=["requests"] + install_requires=["requests", "aiohttp"] ) From 6fd422dc5c169735efb1bf7dba45cf2dd032d7ee Mon Sep 17 00:00:00 2001 From: halfak Date: Fri, 22 Jul 2022 08:45:48 -0700 Subject: [PATCH 23/26] Increments version to 0.6.0 --- mwapi/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mwapi/about.py b/mwapi/about.py index e570142..cf183d8 100644 --- a/mwapi/about.py +++ b/mwapi/about.py @@ -1,5 +1,5 @@ __name__ = "mwapi" -__version__ = "0.5.1" +__version__ = "0.6.0" __author__ = "Aaron Halfaker" __author_email__ = "aaron.halfaker@gmail.com" __description__ = "Simple wrapper for the Mediawiki API" From ba9fb89c9766a551cbad76eff8efb76d86aeac25 Mon Sep 17 00:00:00 2001 From: halfak Date: Fri, 22 Jul 2022 08:48:26 -0700 Subject: [PATCH 24/26] Fixed long_description_content_type in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index ed37f12..3e37fb2 100644 --- a/setup.py +++ b/setup.py @@ -15,5 +15,6 @@ license=__license__, # noqa packages=["mwapi"], long_description=open("README.md").read(), + long_description_content_type="text/markdown", install_requires=["requests", "aiohttp"] ) From 90d2f5d40a9aaaf9822b8e59540b30c85965621a Mon Sep 17 00:00:00 2001 From: AikoChou Date: Fri, 22 Jul 2022 21:13:00 +0200 Subject: [PATCH 25/26] Change 'asyncio.exceptions.TimeoutError' to 'asyncio.TimeoutError' In Python 3.7, 'asyncio.exceptions.TimeoutError' raises AttributeError : module 'asyncio' has no attribute 'exceptions', since it's in class 'concurrent.futures._base.TimeoutError'. In Python 3.8 onward, TimeoutError is in class 'asyncio.exceptions.TimeoutError'. Written as 'asyncio.TimeoutError' will be acceptable for either version. --- mwapi/async_session.py | 4 ++-- mwapi/errors.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mwapi/async_session.py b/mwapi/async_session.py index 7b22cd9..c02f0ee 100644 --- a/mwapi/async_session.py +++ b/mwapi/async_session.py @@ -32,7 +32,7 @@ class AsyncSession: How long to wait for the server to send data before giving up and raising an error ( :class:`aiohttp.client_exceptions.ServerTimeoutError` or - :class:`asyncio.exceptions.TimeoutError`). + :class:`asyncio.TimeoutError`). By default aiohttp uses a total 300 seconds (5min) timeout. session : `aiohttp.ClientSession` (optional) an `aiohttp` session object to use @@ -106,7 +106,7 @@ async def _request(self, method, params=None, auth=None): raise ValueError("Could not decode as JSON:\n{0}" .format(prefix)) except (aiohttp.ServerTimeoutError, - asyncio.exceptions.TimeoutError) as e: + asyncio.TimeoutError) as e: raise TimeoutError(str(e)) from e except aiohttp.ClientConnectionError as e: raise ConnectionError(str(e)) from e diff --git a/mwapi/errors.py b/mwapi/errors.py index 3c21d76..5e8a24f 100644 --- a/mwapi/errors.py +++ b/mwapi/errors.py @@ -103,10 +103,10 @@ class TooManyRedirectsError(requests.exceptions.TooManyRedirects, class TimeoutError(requests.exceptions.Timeout, aiohttp.ServerTimeoutError, - asyncio.exceptions.TimeoutError): + asyncio.TimeoutError): """ Handles a :class:`requests.exceptions.TimeoutError` or :class:`aiohttp.ServerTimeoutError` or - :class:`asyncio.exceptions.TimeoutError`. + :class:`asyncio.TimeoutError`. """ pass From fc45d32d118655acc95ab634c9e069e3670531fd Mon Sep 17 00:00:00 2001 From: halfak Date: Tue, 26 Jul 2022 09:00:33 -0700 Subject: [PATCH 26/26] Increments version to 0.6.1 --- mwapi/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mwapi/about.py b/mwapi/about.py index cf183d8..89d2bfb 100644 --- a/mwapi/about.py +++ b/mwapi/about.py @@ -1,5 +1,5 @@ __name__ = "mwapi" -__version__ = "0.6.0" +__version__ = "0.6.1" __author__ = "Aaron Halfaker" __author_email__ = "aaron.halfaker@gmail.com" __description__ = "Simple wrapper for the Mediawiki API"