diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..70ca310 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,36 @@ +Welcome to the Keen IO Python Client GitHub repo! šŸ‘‹šŸŽ‰ + +- Please [**search for existing issues**](https://github.com/keenlabs/KeenClient-Python/issues) in order to ensure we don't have duplicate bugs/feature requests. +- Please be respectful and considerate of others when commenting on issues +- Found a bugšŸœ? + - If it's a security vulnerability, please email team@keen.io instead of filing an issue + - If not, please fill out the sections below... thank you šŸ˜Ž! + +### Issue Summary + +A summary of the issue and the browser/OS environment in which it occurs. + +Things you can include that are helpful + +- Code samples for what produces the error (or a link if the code is long) +- Error mesages / stack traces +- Screenshots can be very helpful here. + +```python +print("Hello world") +``` + +Note: DO NOT include your credentials in ANY code examples, descriptions, or media you make public. + +### Steps to Reproduce + +1. This is the first step +2. This is the second step, etc. + +Any other info e.g. Why do you consider this to be a bug? What did you expect to happen instead? + +### Technical details: + +* Python Version: +* Operating System (OSX Sierra 10.12.6, Windows 10, etc): +* Frameworks + Versions (if applicable): diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..bfec0e5 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## What does this PR do? How does it affect users? + +## How should this be tested? + +Step through the code line by line. Things to keep in mind as you review: + - Are there any edge cases not covered by this code? + - Does this code follow conventions (naming, formatting, modularization, etc) where applicable? + +Fetch the branch and/or deploy to staging to test the following: + +- [ ] Does the code compile without warnings (check shell, console)? +- [ ] Do all tests pass? +- [ ] If the feature sends data to Keen, is the data visible in the project if you run an extraction (include link to collection/query)? +- [ ] If the feature saves data to a database, can you confirm the data is indeed created in the database? + +## Related tickets? diff --git a/.gitignore b/.gitignore index 3971ce7..d0a087a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ # Packages *.egg +*.eggs *.egg-info dist build diff --git a/.travis.yml b/.travis.yml index d63dc32..0d2c80b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,23 @@ language: python python: - - "2.6" - "2.7" - - "3.2" - - "3.3" + - "3.5" + - "3.6" + - "3.7" + - "3.8" # command to install dependencies -install: "pip install -r requirements.txt --use-mirrors" +install: | + pip install -r requirements.txt + # command to run tests -script: nosetests \ No newline at end of file +script: "python setup.py test" + +# deploy master to PyPI +deploy: + provider: pypi + user: wiktor.olko + # encrypted with the Keen Travis project PublicKey. Only Travis has the PrivateKey and is able to decode it. + # travis encrypt PASSWORD_HERE --add deploy.password + password: + secure: nxwi3BYKikV+nz3+OLWv7BXakLM2I+EafvCRiBoYUNQ2BBzg2pyBqwKna/AeqID8ASaHYWfHha64+7rDvJD1Sh7/Xgcc0GzDhtqUAiKXyKsdlg+4Q0XHWT3jeRTA2+vkqYMuren3MlbcSgzyyNO/IKf3zItiX+cA8/dyxRqOe3g= + skip_existing: true diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..7850e0a --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,264 @@ +Changelog +--------- + +0.7.0 +`````` + ++ Do not force TLSv1 which is deprecated + + +0.6.1 +`````` + ++ Dropped support for Python 3.4 + + +0.6.0 +`````` + ++ Added the delete_access_key method ++ Renamed the argument names in Access Key API (breaking change) + + +0.5.2 +`````` + ++ Added support for Python 3.7 and 3.8. + + +0.5.1 +`````` + ++ Fixed various Access Key bugs. + + +0.5.0 +`````` ++ Added support for Access Keys. ++ Added support for order_by and limit, group_by options. ++ Deprecated python 2.6, 3.2 and 3.3. ++ Scoped Keys are now deprecated in favor of Access Keys. ++ Now permits more versions of the requests library. (issue #133) + + +0.4.0 +`````` + ++ SavedQueriesInterface.create() now accepts a dict as the query definition. ++ get_collection() and get_all_collections() now only require a Read Key instead of Master. ++ SavedQueriesInterface.update() now performs partial updates. update_full() exhibits old behavior. ++ Misc documentation updates. + +0.3.31 +`````` + ++ Fix broken releases. + +0.3.29 +`````` + ++ Add Keen-Sdk header to all requests, containing the SDK version. + +0.3.28 +`````` + ++ Fix incorrect README. + +0.3.27 +`````` + ++ Return JSON response when uploading events in a batch. + +0.3.26 +`````` + ++ Removed unused `Padding` from requirements.txt to make python 3.x installs cleaner. + +0.3.25 +`````` + ++ Replaced defunct `pycrypto` library with `cryptodome`. ++ Fixed UnicodeDecodeError under PY3 while installing in Windows. + +0.3.24 +`````` + ++ Updated documentation + +0.3.23 +`````` + ++ Added status code to JSON parse error response + +0.3.22 +`````` + ++ Added support for python 3.5 + +0.3.21 +`````` + ++ Fixed bug with scoped key generation not working with newer Keen projects. + +0.3.20 +`````` + ++ Added `saved_queries` support ++ Added Python 3.4 support + +0.3.19 +`````` + ++ Added `base_url` as a possible env variable + +0.3.18 +`````` + ++ Updated error handling to except `ValueError` + +0.3.17 +`````` + ++ Fixed timestamp overriding keen addons ++ Added `get_collection` and `get_all_collections` methods + +0.3.16 +`````` + ++ Added `all_keys` parameter which allows users to expose all keys in query response. ++ Added `delete_events` method. + +0.3.15 +`````` + ++ Added better error handling to surface all errors from HTTP API calls. + +0.3.14 +`````` + ++ Added compatibility for pip 1.0 + +0.3.13 +`````` + ++ Added compatibility for pip < 1.5.6 + +0.3.12 +`````` + ++ Made requirements more flexible. + +0.3.11 +`````` + ++ Added `requirements.txt` to pypi package. + +0.3.10 +`````` + ++ Fixed requirements in `setup.py` ++ Updated test inputs and documentation. + +0.3.9 +````` + ++ Added ```master_key``` parameter. + +0.3.8 +````` + ++ Mocked tests. ++ Added ```median``` query method. ++ Added support for `$python setup.py test`. + +0.3.7 +````` + ++ Upgraded to requests==2.5.1 + +0.3.6 +````` + ++ Added ```max_age``` parameter for caching. + +0.3.5 +````` + ++ Added client configurable timeout to gets. + +0.3.4 +````` + ++ Added ```percentile``` query method. + +0.3.3 +````` + ++ Support ```interval``` parameter for multi analyses on the keen module. + +0.3.2 +````` + ++ Reuse internal requests' session inside an instance of KeenApi. + +0.3.1 +````` + ++ Support ```property_names``` parameter for extractions. + +0.3.0 +````` + ++ Added client configurable timeout to posts. ++ Upgraded to requests==2.2.1. + +0.2.3 +````` + ++ Fixed sys.version_info issue with Python 2.6. + +0.2.2 +````` + ++ Added interval to multi_analysis. + +0.2.1 +````` + ++ Added stacktrace_id and unique_id to Keen API errors. + +0.2.0 +````` + ++ Added add_events method to keen/__init__.py so it can be used at a module level. ++ Added method to generate image beacon URLs. + +0.1.9 +````` + ++ Added support for publishing events in batches ++ Added support for configuring client automatically from environment ++ Added methods on keen module directly + +0.1.8 +````` + ++ Added querying support + +0.1.7 +````` + ++ Bugfix to use write key when sending events - do not use 0.1.6! + +0.1.6 +````` + ++ Changed project token -> project ID. ++ Added support for read and write scoped keys. ++ Added support for generating scoped keys yourself. ++ Added support for python 2.6, 3.2, and 3.3 + +0.1.5 +````` + ++ Added documentation. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..19d751d --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Keen IO Community Code of Conduct + +The Keen IO Community is dedicated to providing a safe, inclusive, welcoming, and harassment-free space and experience for all community participants, regardless of gender identity and expression, sexual orientation, disability, physical appearance, socioeconomic status, body size, ethnicity, nationality, level of experience, age, religion (or lack thereof), or other identity markers. Our Code of Conduct exists because of that dedication, and we do not tolerate harassment in any form. See our reporting guidelines [here](https://github.com/keen/community-code-of-conduct/blob/master/incident-reporting.md). Our full Code of Conduct can be found at this [link](https://github.com/keen/community-code-of-conduct/blob/master/long-form-code-of-conduct.md). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..3e5b6ab --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,93 @@ +# Keen IO Python Client + +First off, thank you for considering contributing to this official Keen IO client. It's people like you that make Keen IO such a great tool. + +We put these guidelines together to try and make working with our SDK as straight forward as possible, and hopefully help you understand how we communicate about potential changes and improvements. + +Improving documentation, bug triaging, building modules for various frameworks or writing tutorials are all examples of helpful contributions we really appreciate. + +Please, don't use the issue tracker for support questions. If you have a support question please come hang out in our Slack at http://keen.chat or send an email to team@keen.io. + +## Guidelines + +* Create issues for any major changes and enhancements that you wish to make. Discuss things transparently and get community feedback. +* Be welcoming to newcomers and encourage diverse new contributors from all backgrounds. +* Please take time to read our [Community Code of Conduct](#community-code-of-conduct) which includes Reporting Guidelines. + +## Community Code of Conduct + +The Keen IO Community is dedicated to providing a safe, inclusive, welcoming, and harassment-free space and experience for all community participants, regardless of gender identity and expression, sexual orientation, disability, physical appearance, socioeconomic status, body size, ethnicity, nationality, level of experience, age, religion (or lack thereof), or other identity markers. Our Code of Conduct exists because of that dedication, and we do not tolerate harassment in any form. See our reporting guidelines [here](https://github.com/keen/community-code-of-conduct/blob/master/incident-reporting.md). Our full Code of Conduct can be found at this [link](https://github.com/keen/community-code-of-conduct/blob/master/long-form-code-of-conduct.md). + +## Your First Contribution + +Here are a couple of friendly tutorials with more information about contributing to OSS projects: + +- http://makeapullrequest.com/, +- http://www.firsttimersonly.com/ +-[How to Contribute to an Open Source Project on GitHub](https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github) +- [Github's Open Source Guide](https://opensource.guide) + +At this point, you're ready to make your changes! Feel free to ask for help; everyone is a beginner at first :smile_cat: + +If a maintainer asks you to "rebase" your PR, they're saying that a lot of code has changed, and that you need to update your branch so it's easier to merge. + +### Run the following commands to get this project installed locally + +```ssh +$ git clone https://github.com/keenlabs/KeenClient-Python && cd KeenClient-Python +# Make sure you're in a python virtual environment before installing +$ python setup.py develop + +# Run tests locally +$ python setup.py test +``` + +### Submitting a Pull Request + +Use the template below. If certain testing steps are not relevant, specify that in the PR. If additional checks are needed, add 'em! Please run through all testing steps before asking for a review. + +``` +## What does this PR do? How does it affect users? + +## How should this be tested? + +Step through the code line by line. Things to keep in mind as you review: + - Are there any edge cases not covered by this code? + - Does this code follow conventions (naming, formatting, modularization, etc) where applicable? + +Fetch the branch and/or deploy to staging to test the following: + +- [ ] Does the code compile without warnings (check shell, console)? +- [ ] Do all tests pass? +- [ ] If the feature sends data to Keen, is the data visible in the project if you run an extraction (include link to collection/query)? +- [ ] If the feature saves data to a database, can you confirm the data is indeed created in the database? + +## Related tickets? +``` + +This PR template can be viewed rendered in Markdown [here](./.github/PULL_REQUEST_TEMPLATE.md). Github will auto-populate any new PRs filed with this template, so don't worry about copy-pasting it. + +## How to report a bug + +If you find a security vulnerability, do NOT open an issue. Email team@keen.io instead. + +If you find a bug that's not a security vulnerability please head over to the `issues <./issues>`_ page, and file a new report. + +We created these labels to help us organize issues: + +- `bugs` +- `docs` +- `enhancements` +- `feature-request` + +Please use them when creating an issue where it makes sense! + +## Suggesting features + +We welcome your feedback and requests. If you have a straight forward request please open up an issue that details the request. If you want to talk to someone on the Keen team head over to http://keen.chat or send a note to team@keen.io and we will make sure and get you in touch with the product team. + +# Code review process + +The core team looks at Pull Requests and issues on a regular basis and will typically respond within 5 business days. + + diff --git a/LICENSE.txt b/LICENSE.txt index c47a4c0..a656355 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2013 Keen Labs +Copyright (c) 2017 Keen Labs Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..ccf2a9b --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include requirements.txt +include README.md +include LICENSE.txt diff --git a/README.md b/README.md deleted file mode 100644 index 144f29c..0000000 --- a/README.md +++ /dev/null @@ -1,260 +0,0 @@ -Keen IO Official Python Client Library -====================================== - -[![Build Status](https://secure.travis-ci.org/keenlabs/KeenClient-Python.png)](http://travis-ci.org/keenlabs/KeenClient-Python) - -This is the official Python Client for the [Keen IO](https://keen.io/) API. The -Keen IO API lets developers build analytics features directly into their apps. - -This is still under active development. Stay tuned for improvements! - -### Installation - -Use pip to install! - - pip install keen - -This client is known to work on Python 2.6, 2.7, 3.2 and 3.3 - -### Usage - -To use this client with the Keen IO API, you have to configure your Keen IO Project ID and its access keys (if you need an account, [sign up here](https://keen.io/) - it's free). - -Setting a write key is required for publishing events. Setting a read key is required for running queries. The recommended way to set this configuration information is via the environment. The keys you can set are `KEEN_PROJECT_ID`, `KEEN_WRITE_KEY`, and `KEEN_READ_KEY`. - -If you don't want to use environment variables for some reason, you can directly set values as follows: - -```python - keen.project_id = "xxxx" - keen.write_key = "yyyy" - keen.read_key = "zzzz" -``` - -You can also configure unique client instances as follows: - -```python - from keen.client import KeenClient - - client = KeenClient( - project_id="xxxx", - write_key="yyyy", - read_key="zzzz" - ) -``` - -##### Send Events to Keen IO - -Once you've set `KEEN_PROJECT_ID` and `KEEN_WRITE_KEY`, sending events is simple: - -```python - keen.add_event("sign_ups", { - "username": "lloyd", - "referred_by": "harry" - }) -``` - -##### Send Batch Events to Keen IO - -You can upload Events in a batch, like so: - -```python - # uploads 4 events total - 2 to the "sign_ups" collection and 2 to the "purchases" collection - keen.add_events({ - "sign_ups": [ - { "username": "nameuser1" }, - { "username": "nameuser2" } - ], - "purchases": [ - { "price": 5 }, - { "price": 6 } - ] - }) -``` - - -That's it! After running your code, check your Keen IO Project to see the event/events has been added. - -##### Do analysis with Keen IO - -Here are some examples of querying. Let's assume you've added some events to the "purchases" collection. - -```python - keen.count("purchases") # => 100 - keen.sum("purchases", target_property="price") # => 10000 - keen.minimum("purchases", target_property="price") # => 20 - keen.maximum("purchases", target_property="price") # => 100 - keen.average("purchases", target_property="price") # => 49.2 - - keen.sum("purchases", target_property="price", group_by="item.id") # => [{ "item.id": 123, "result": 240 }, { ... }] - - keen.count_unique("purchases", target_property="user.id") # => 3 - keen.select_unique("purchases", target_property="user.email") # => ["bob@aol.com", "joe@yahoo.biz"] - - keen.extraction("purchases", timeframe="today") # => [{ "price" => 20, ... }, { ... }] - - keen.multi_analysis("purchases", analyses={"total":{"analysis_type":"sum", "target_property":"price"}, "average":{"analysis_type":"average", "target_property":"price"}) # => {"total":10329.03, "average":933.93} - - step1 = { - "event_collection": "signup", - "actor_property": "user.email" - } - step2 = { - "event_collection": "purchase", - "actor_property": "user.email" - } - keen.funnel([step1, step2], timeframe="today") # => [2039, 201] -``` - -#### Advanced Usage - -See below for more options. - -##### Overwriting event timestamps - -Two time-related properties are included in your event automatically. The properties ā€œkeen.timestampā€ and ā€œkeen.created_atā€ are set at the time your event is recorded. You have the ability to overwrite the keen.timestamp property. This could be useful, for example, if you are backfilling historical data. Be sure to use [ISO-8601 Format](https://keen.io/docs/event-data-modeling/event-data-intro/#iso-8601-format). - -Keen stores all date and time information in UTC! - -```python - keen.add_event("sign_ups", { - "keen": { - "timestamp": "2012-07-06T02:09:10.141Z" - }, - "username": "lloyd", - "referred_by": "harry" - }) -``` - -##### Get from Keen IO with a Timeout - -By default, GET requests will timeout after 305 seconds. If you want to manually override this, you can create a KeenClient with the "get_timeout" parameter. This client will fail GETs if no bytes have been returned by the server in the specified time. For example: - -```python - from keen.client import KeenClient - - client = KeenClient( - project_id="xxxx", - write_key="yyyy", - read_key="zzzz", - get_timeout=100 - - ) -``` - -This will cause queries such as count(), sum(), and average() to timeout after 100 seconds. If this timeout limit is hit, a requests.Timeout will be raised. Due to a bug in the requests library, you might also see an SSLError (https://github.com/kennethreitz/requests/issues/1294) - -##### Send to Keen IO with a Timeout - -By default, POST requests will timeout after 305 seconds. If you want to manually override this, you can create a KeenClient with the "post_timeout" parameter. This client will fail POSTs if no bytes have been returned by the server in the specified time. For example: - -```python - from keen.client import KeenClient - - client = KeenClient( - project_id="xxxx", - write_key="yyyy", - read_key="zzzz", - post_timeout=100 - - ) -``` - -This will cause both add_event() and add_events() to timeout after 100 seconds. If this timeout limit is hit, a requests.Timeout will be raised. Due to a bug in the requests library, you might also see an SSLError (https://github.com/kennethreitz/requests/issues/1294) - -##### Create Scoped Keys - -The Python client enables you to create [Scoped Keys](https://keen.io/docs/security/#scoped-key) easily. For example: - -```python - from keen.client import KeenClient - from keen import scoped_keys - - api_key = KEEN_MASTER_KEY - - write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}) - read_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["read"]}) -``` - -`write_key` and `read_key` now contain scoped keys based on your master API key. - -### Changelog - - -##### 0.3.5 -+ Added client configurable timeout to gets. - -##### 0.3.4 -+ Added ```percentile``` query method. - -##### 0.3.3 - -+ Support ```interval``` parameter for multi analyses on the keen module. - -##### 0.3.2 - -+ Reuse internal requests' session inside an instance of KeenApi. - -##### 0.3.1 - -+ Support ```property_names``` parameter for extractions. - -##### 0.3.0 - -+ Added client configurable timeout to posts. -+ Upgraded to requests==2.2.1. - -##### 0.2.3 - -+ Fixed sys.version_info issue with Python 2.6. - -##### 0.2.2 - -+ Added interval to multi_analysis. - -##### 0.2.1 - -+ Added stacktrace_id and unique_id to Keen API errors. - -##### 0.2.0 - -+ Added add_events method to keen/__init__.py so it can be used at a module level. -+ Added method to generate image beacon URLs. - -##### 0.1.9 - -+ Added support for publishing events in batches -+ Added support for configuring client automatically from environment -+ Added methods on keen module directly - -##### 0.1.8 - -+ Added querying support - -##### 0.1.7 - -+ Bugfix to use write key when sending events - do not use 0.1.6! - -##### 0.1.6 - -+ Changed project token -> project ID. -+ Added support for read and write scoped keys. -+ Added support for generating scoped keys yourself. -+ Added support for python 2.6, 3.2, and 3.3 - -##### 0.1.5 - -+ Added documentation. - -### To Do - -* Asynchronous insert -* Scoped keys - -### Questions & Support - -If you have any questions, bugs, or suggestions, please -report them via Github Issues. Or, come chat with us anytime -at [users.keen.io](http://users.keen.io). We'd love to hear your feedback and ideas! - -### Contributing -This is an open source project and we love involvement from the community! Hit us up with pull requests and issues. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..bf2eb28 --- /dev/null +++ b/README.rst @@ -0,0 +1,680 @@ +Keen IO Official Python Client Library +====================================== + +|build-status| |pypi-version| + +This is the official Python Client for the `Keen IO `_ API. The +Keen IO API lets developers build analytics features directly into their apps. + +This is still under active development. Stay tuned for improvements! + +Installation +------------ + +Use pip to install! + +:: + + pip install keen + +This client is known to work on Python 2.7, 3.5, 3.6, 3.7 and 3.8. + +For versions of Python < 2.7.9, youā€™ll need to install pyasn1, ndg-httpsclient, pyOpenSSL. + +Usage +----- + +To use this client with the Keen IO API, you have to configure your Keen IO Project ID and its access +keys (if you need an account, `sign up here `_ - it's free). + +Setting a write key is required for publishing events. Setting a read key is required for +running queries. The recommended way to set this configuration information is via the environment. +The keys you can set are `KEEN_PROJECT_ID`, `KEEN_WRITE_KEY`, `KEEN_READ_KEY`, and `KEEN_MASTER_KEY`. +As per the `Principle of Least Privilege `_, it's recommended that you not use the master_key if not +necessary. This SDK will expect and use the precise key for a given operation, and throw an +exception in cases of misuse. + +If you don't want to use environment variables for some reason, you can directly set values as follows: + +.. code-block:: python + + keen.project_id = "xxxx" + keen.write_key = "yyyy" + keen.read_key = "zzzz" + keen.master_key = "abcd" # not required for typical usage + + +For information on how to configure unique client instances, take a look at the +`Advanced Usage <#advanced-usage>`_ section below. + +Send Events to Keen IO +`````````````````````` + +Once you've set `KEEN_PROJECT_ID` and `KEEN_WRITE_KEY`, sending events is simple: + +.. code-block:: python + + keen.add_event("sign_ups", { + "username": "lloyd", + "referred_by": "harry" + }) + +Data Enrichment +``````````````` + +A data enrichment is a powerful add-on to enrich the data you're already streaming to Keen IO by pre-processing the data and adding helpful data properties. To activate add-ons, you simply add some new properties within the "keen" namespace in your events. Detailed documentation for the configuration of our add-ons is available `here `_. + +Here is an example of using the `URL parser `_: + +.. code-block:: python + + keen.add_event("requests", { + "page_url" : "http://my-website.com/cool/link?source=twitter&foo=bar/#title", + "keen" : { + "addons" : [ + { + "name" : "keen:url_parser", + "input" : { + "url" : "page_url" + }, + "output" : "parsed_page_url" + } + ] + } + }) + +Keen IO will parse the URL for you and that would equivalent to: + +.. code-block:: python + + keen.add_event("request", { + "page_url" : "http://my-website.com/cool/link?source=twitter&foo=bar/#title", + "parsed_page_url": { + "protocol" : "http", + "domain" : "my-website.com", + "path" : "/cool/link", + "anchor" : "title", + "query_string" : { + "source" : "twitter", + "foo" : "bar" + } + } + }) + +Here is another example of using the `Datetime parser `_. Let's assume you want to do a deeper analysis on the "purchases" event by day of the week (Monday, Tuesday, Wednesday, etc.) and other interesting Datetime components. You can use "keen.timestamp" property that is included in your event automatically. + +.. code-block:: python + + keen.add_event("purchases", { + "keen": { + "addons": [ + { + "name": "keen:date_time_parser", + "input": { + "date_time" : "keen.timestamp" + }, + "output": "timestamp_info" + } + ] + }, + "price": 500 + }) + +Other Data Enrichment add-ons are located in the `API reference docs `_. + +Send Batch Events to Keen IO +```````````````````````````` + +You can upload Events in a batch, like so: + +.. code-block:: python + + # uploads 4 events total - 2 to the "sign_ups" collection and 2 to the "purchases" collection + keen.add_events({ + "sign_ups": [ + { "username": "nameuser1" }, + { "username": "nameuser2" } + ], + "purchases": [ + { "price": 5 }, + { "price": 6 } + ] + }) + + +That's it! After running your code, check your Keen IO Project to see the event/events has been added. + +Do analysis with Keen IO +```````````````````````` + +Here are some examples of querying. Let's assume you've added some events to the "purchases" collection. +For more code samples, take a look at Keen's `docs `_ + +.. code-block:: python + + keen.count("purchases", timeframe="this_14_days") # => 100 + keen.sum("purchases", target_property="price", timeframe="this_14_days") # => 10000 + keen.minimum("purchases", target_property="price", timeframe="this_14_days") # => 20 + keen.maximum("purchases", target_property="price", timeframe="this_14_days") # => 100 + keen.average("purchases", target_property="price", timeframe="this_14_days") # => 49.2 + + keen.sum("purchases", target_property="price", group_by="item.id", timeframe="this_14_days") # => [{ "item.id": 123, "result": 240 }, { ... }] + + keen.count_unique("purchases", target_property="user.id", timeframe="this_14_days") # => 3 + keen.select_unique("purchases", target_property="user.email", timeframe="this_14_days") # => ["bob@aol.com", "joe@yahoo.biz"] + + # Alpha support for ordering your results and limiting what is returned is now supported in the Python SDK. + # Keep in mind that even if you limit your results with the "limit" keyword, you are still querying over the + # normal amount of data, and thus your compute costs will not change. Limit only changes what is displayed. + + # The keyword "limit" must be a positive integer. The keyword "order_by" must be a dictionary with a required + # "property_name" specified and optionally a "direction". The "property_name" must be one of the "group_by" + # properties or "result". The "direction" may be either keen.direction.DESCENDING or keen.direction.ASCENDING. + # Ascending is the default direction used if no "direction" is supplied. No other keywords may be used in the + # "order_by" dictionary. + + # You may only use "order_by" if you supply a "group_by". You may only use "limit" if you supply an "order_by". + + # This will run a count query with results grouped by zip code. + # It will display only the top ten zip code results based upon how many times users in those zip codes logged in. + keen.count("purchases", group_by="zip_code", timeframe="this_14_days", limit=10, + order_by={"property_name": "result", "direction": keen.direction.DESCENDING}) + + keen.extraction("purchases", timeframe="today") # => [{ "price" => 20, ... }, { ... }] + + keen.multi_analysis( + "purchases", + analyses={ + "total":{ + "analysis_type": "sum", + "target_property": "price" + }, + "average":{ + "analysis_type": "average", + "target_property": "price" + } + }, + timeframe='this_14_days' + ) # => {"total":10329.03, "average":933.93} + + step1 = { + "event_collection": "sign_ups", + "actor_property": "user.email" + } + step2 = { + "event_collection": "purchases", + "actor_property": "user.email" + } + keen.funnel([step1, step2], timeframe="today") # => [2039, 201] + + +To return the full API response from a funnel analysis (as opposed to the singular "result" key), set `all_keys=True`. + +For example, `keen.funnel([step1, step2], timeframe="today", all_keys=True)` would return "result", "actors" and "steps" keys. + +Delete Events +````````````` + +The Keen IO API allows you to `delete events `_ from event collections, optionally supplying filters, timeframe or timezone to narrow the scope of what you would like to delete. + +You'll need to set your master_key. + +.. code-block:: python + + keen.delete_events("event_collection", filters=[{"property_name": 'username', "operator": 'eq', "property_value": 'Bob'}]) + +Advanced Usage +`````````````` + +See below for more options. + +Check Batch Upload Response For Errors +'''''''''''''''''''''''''''''''''''''' + +When you upload events in a batch, some of them may succeed and some of them may have errors. The Keen API returns information on each. Here's an example: + +Upload code (remember, Keen IO doesn't allow periods in property names): + +.. code-block:: python + + response = keen.add_events({ + "sign_ups": [ + { "username": "nameuser1" }, + { "username": "nameuser2", "an.invalid.property.name": 1 } + ], + "purchases": [ + { "price": 5 }, + { "price": 6 } + ] + }) + +That code would result in the following API JSON response: + +.. code-block:: javascript + + { + "sign_ups": [ + {"success": true}, + {"success": false, "error": {"name": "some_error_name", "description": "some longer description"}} + ], + "purchases": [ + {"success": true}, + {"success": true} + ] + } + +So in python, to check on the results of your batch, you'd have code like so: + +.. code-block:: python + + batch = { + "sign_ups": [ + { "username": "nameuser1" }, + { "username": "nameuser2", "an.invalid.property.name": 1 } + ], + "purchases": [ + { "price": 5 }, + { "price": 6 } + ] + } + response = keen.add_events(batch) + + for collection in response: + collection_result = response[collection] + event_count = 0 + for individual_result in collection_result: + if not individual_result["success"]: + print("Event had error! Collection: '{}'. Event body: '{}'.".format(collection, batch[collection][event_count])) + event_count += 1 + +Configure Unique Client Instances +''''''''''''''''''''''''''''''''' + +If you intend to send events or query from different projects within the same python file, you'll need to set up +unique client instances (one per project). You can do this by assigning an instance of KeenClient to a variable like so: + +.. code-block:: python + + from keen.client import KeenClient + + client = KeenClient( + project_id="xxxx", # your project ID for collecting cycling data + write_key="yyyy", + read_key="zzzz", + master_key="abcd" # not required for typical usage + ) + + client_hike = KeenClient( + project_id="xxxx", # your project ID for collecting hiking data (different from the one above) + write_key="yyyy", + read_key="zzzz", + master_key="abcd" # not required for typical usage + ) + + +You can send events like this: + +.. code-block:: python + + # add an event to an event collection in your cycling project + client.add_event(...) + + # or add an event to an event collection in your hiking project + client_hike.add_event(...) + + +Similarly, you can query events like this: + +.. code-block:: python + + client.count(...) + + +Saved Queries +''''''''''''' + +You can manage your `saved queries `_ from the Keen python client. + +.. code-block:: python + + # Create your KeenClient + from keen.client import KeenClient + + client = KeenClient( + project_id="xxxx", # your project ID + read_key="zzzz", + master_key="abcd" # Most Saved Query functionality requires master_key + ) + + # Create a saved query + saved_query_attributes = { + # NOTE : For now, refresh_rate must explicitly be set to 0 unless you + # intend to create a Cached Query. + "refresh_rate": 0, + "query": { + "analysis_type": "count", + "event_collection": "purchases", + "timeframe": "this_2_weeks", + "filters": [{ + "property_name": "price", + "operator": "gte", + "property_value": 1.00 + }] + } + } + + client.saved_queries.create("saved-query-name", saved_query_attributes) + + # Get all saved queries + client.saved_queries.all() + + # Get one saved query + client.saved_queries.get("saved-query-name") + + # Get saved query with results + client.saved_queries.results("saved-query-name") + + # NOTE : Updating Saved Queries requires sending the entire query definition. Any attribute not + # sent is interpreted as being cleared/removed. This means that properties set via another + # client, including the Projects Explorer Web UI, will be lost this way. + # + # The update() function makes this easier by allowing client code to just specify the + # properties that need updating. To do this, it will retrieve the existing query definition + # first, which means there will be two HTTP requests. Use update_full() in code that already + # has a full query definition that can reasonably be expected to be current. + + # Update a saved query to now be a cached query with the minimum refresh rate of 4 hrs... + + # ...using partial update: + client.saved_queries.update("saved-query-name", { "refresh_rate": 14400 }) + + # ...using full update, if we've already fetched the query definition: + saved_query_attributes["refresh_rate"] = 14400 + client.saved_queries.update_full("saved-query-name", saved_query_attributes) + + # Update a saved query to a new resource name... + + # ...using partial update: + client.saved_queries.update("saved-query-name", { "query_name": "cached-query-name" }) + + # ...using full update, if we've already fetched the query definition or have it lying around + # for whatever reason. We send "refresh_rate" again, along with the entire definition, or else + # it would be reset: + saved_query_attributes["query_name"] = "cached-query-name" + client.saved_queries.update_full("saved-query-name", saved_query_attributes) + + # Delete a saved query (use the new resource name since we just changed it) + client.saved_queries.delete("cached-query-name") + +Cached Datasets +''''''''''''''' +.. code-block:: python + + # Create your KeenClient + from keen.client import KeenClient + + client = KeenClient( + project_id="xxxx", # your project ID + read_key="zzzz", + master_key="abcd" + ) + + # Create a Cached Dataset + dataset_name = "NEW_DATASET" + display_name = "My new dataset" + query = { + "project_id": "PROJECT ID", + "analysis_type": "count", + "event_collection": "purchases", + "filters": [ + { + "property_name": "price", + "operator": "gte", + "property_value": 100 + } + ], + "timeframe": "this_500_days", + "timezone": None, + "interval": "daily", + "group_by": ["ip_geo_info.country"] + } + index_by = "product.id" + client.cached_datasets.create( + dataset_name, + query, + index_by, + display_name + ) + + # Get all Cached Datasets + client.cached_datasets.all() + + # Get one Cached Dataset + client.cached_datasets.get(dataset_name) + + # Retrieve Cached Dataset results + index_by = "a_project_id" + timeframe ="this_2_hours" + client.cached_datasets.results( + dataset_name, + index_by, + timeframe + ) + + # Using an absolute timeframe + timeframe = { + "start": "2018-11-02T00:00:00.000Z", + "end": "2018-11-02T02:00:00.000Z" + } + client.cached_datasets.results( + dataset_name, + index_by, + timeframe + ) + + # Using multiple index_by values + index_by = { + "project.id": "a_project_id", + "project.foo": "bar" + } + client.cached_datasets.results( + "dataset_with_multiple_indexes", + index_by, + timeframe + ) + + # Delete a Cached Dataset + client.cached_datasets.delete(dataset_name) + +Overwriting event timestamps +'''''''''''''''''''''''''''' + +Two time-related properties are included in your event automatically. The properties ā€œkeen.timestampā€ +and ā€œkeen.created_atā€ are set at the time your event is recorded. You have the ability to overwrite the +keen.timestamp property. This could be useful, for example, if you are backfilling historical data. Be +sure to use `ISO-8601 Format `_. + +Keen stores all date and time information in UTC! + +.. code-block:: python + + keen.add_event("sign_ups", { + "keen": { + "timestamp": "2012-07-06T02:09:10.141Z" + }, + "username": "lloyd", + "referred_by": "harry" + }) + + +Get from Keen IO with a Timeout +''''''''''''''''''''''''''''''' + +By default, GET requests will timeout after 305 seconds. If you want to manually override this, you can +create a KeenClient with the "get_timeout" parameter. This client will fail GETs if no bytes have been +returned by the server in the specified time. For example: + +.. code-block:: python + + from keen.client import KeenClient + + client = KeenClient( + project_id="xxxx", + write_key="yyyy", + read_key="zzzz", + get_timeout=100 + ) + + +This will cause queries such as count(), sum(), and average() to timeout after 100 seconds. If this timeout +limit is hit, a requests.Timeout will be raised. Due to a bug in the requests library, you might also see an +SSLError (`#1294 `_) + +Send to Keen IO with a Timeout +'''''''''''''''''''''''''''''' + +By default, POST requests will timeout after 305 seconds. If you want to manually override this, you can +create a KeenClient with the "post_timeout" parameter. This client will fail POSTs if no bytes have been +returned by the server in the specified time. For example: + +.. code-block:: python + + from keen.client import KeenClient + + client = KeenClient( + project_id="xxxx", + write_key="yyyy", + post_timeout=100 + ) + + +This will cause both add_event() and add_events() to timeout after 100 seconds. If this timeout limit is hit, a requests.Timeout will be raised. Due to a bug in the requests library, you might also see an SSLError (https://github.com/kennethreitz/requests/issues/1294) + +Create Access Keys +'''''''''''''''''' + +The Python client enables the creation and manipulation of `Access Keys `_. Examples: + +.. code-block:: python + + from keen.client import KeenClient + # You could also simply use: import keen + # If you do this, you will need your project ID and master key set in environment variables. + + client = KeenClient( + project_id="xxxx", + master_key="zzzz" + ) + + # Create an access key. See: https://keen.io/docs/access/access-keys/#customizing-your-access-key + client.create_access_key(name="Dave_Barry_Key", is_active=True, permitted=["writes", "cached_queries"], + options={"cached_queries": {"allowed": ["dave_barry_in_cyberspace_sales"]}}) + + # Display all access keys associated with this client's project. + client.list_access_keys() + + # Get details on a particular access key. + client.get_access_key(key="ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + # Revoke (disable) an access key. + client.revoke_access_key(key="ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + # Unrevoke (re-enable) an access key. + client.unrevoke_access_key(key="ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + # Delete an access key + client.delete_access_key(key="ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + # Change just the name of an access key. + client.update_access_key_name(key="ABCDEFGHIJKLMNOPQRSTUVWXYZ", name="Some_New_Name") + + # Add new access key permissions to existing permissions on a given key. + # In this case the set of permissions currently contains "writes" and "cached_queries". + # This function call keeps the old permissions and adds "queries" to that set. + # ("writes", "cached_queries") + ("queries") = ("writes", "cached_queries", "queries") + client.add_access_key_permissions(key="ABCDEFGHIJKLMNOPQRSTUVWXYZ", permissions=["queries"]) + + # Remove one or more access key permissions from a given key. + # In this case the set of permissions currently contains "writes", "cached_queries", and "queries". + # This function call will keep the old permissions not explicitly removed here. + # So we will remove both "writes" and "queries" from the set, leaving only "cached_queries". + # ("writes", "cached_queries", "queries") - ("writes", "queries") = ("cached_queries") + client.remove_access_key_permissions(key="ABCDEFGHIJKLMNOPQRSTUVWXYZ", permissions=["writes", "queries"]) + + # We can also perform a full update on the permissions, replacing all existing permissions with a new list. + # In this case our existing permissions contains only "cached_queries". + # We will replace this set with the "writes" permission with this function call. + # ("cached_queries") REPLACE-WITH ("writes") = ("writes") + client.update_access_key_permissions(key="ABCDEFGHIJKLMNOPQRSTUVWXYZ", permissions=["writes"]) + + # Replace all existing key options with this new options object. + client.update_access_key_options(key="ABCDEFGHIJKLMNOPQRSTUVWXYZ", options={"writes": { + "autofill": { + "customer": { + "id": "93iskds39kd93id", + "name": "Ada Corp." + } + } + }}) + + # Replace everything but the key ID with what is supplied here. + # If a field is not supplied here, it will be set to a blank value. + # In this case, no options are supplied, so all options will be removed. + client.update_access_key_full(key="ABCDEFGHIJKLMNOPQRSTUVWXYZ", name="Strong_Bad", is_active=True, permitted=["queries"]) + + +Create Scoped Keys (**Deprecated**) +''''''''''''''''''''''''''''''''''' + +The Python client enables you to create `Scoped Keys `_ easily, but Access Keys are better! +If you need to use them anyway, for legacy reasons, here's how: + +.. code-block:: python + + from keen.client import KeenClient + from keen import scoped_keys + + api_key = KEEN_MASTER_KEY + + write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}) + read_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["read"]}) + + +`write_key` and `read_key` now contain scoped keys based on your master API key. + +Testing +------- + +To run tests: + +:: + + python setup.py test + + +Changelog +--------- + +This project is in alpha stage at version 0.7.0. See the full `CHANGELOG <./CHANGELOG.rst>`_. + + +Questions & Support +------------------- + +If you have any questions, bugs, or suggestions, please +report them via Github `Issues <./issues>`_. We'd love to hear your feedback and ideas! + +Contributing +------------ + +This is an open source project and we love involvement from the community! Hit us up with pull requests and issues. + +`Learn more about contributing to this project <./CONTRIBUTING.md>`_. + +.. |build-status| image:: https://img.shields.io/travis/keenlabs/KeenClient-Python.svg?maxAge=600 + :target: http://travis-ci.org/keenlabs/KeenClient-Python + :alt: Build status + +.. |pypi-version| image:: https://img.shields.io/pypi/v/keen.svg?maxAge=600 + :target: https://pypi.python.org/pypi/keen/ + :alt: Keen on PyPI diff --git a/keen/Padding.py b/keen/Padding.py index 5615340..b213370 100644 --- a/keen/Padding.py +++ b/keen/Padding.py @@ -103,7 +103,10 @@ def appendCMSPadding(str, blocksize=AES_blocksize): def removeCMSPadding(str, blocksize=AES_blocksize): '''CMS padding: Remove padding with bytes containing the number of padding bytes ''' - pad_len = ord(str[-1]) # last byte contains number of padding bytes + try: + pad_len = ord(str[-1]) # last byte contains number of padding bytes + except TypeError: + pad_len = str[-1] assert pad_len <= blocksize, 'padding error' assert pad_len <= len(str), 'padding error' @@ -152,7 +155,10 @@ def appendZeroLenPadding(str, blocksize=AES_blocksize): def removeZeroLenPadding(str, blocksize=AES_blocksize): 'Remove Padding with zeroes + last byte equal to the number of padding bytes' - pad_len = ord(str[-1]) # last byte contains number of padding bytes + try: + pad_len = ord(str[-1]) # last byte contains number of padding bytes + except TypeError: + pad_len = str[-1] assert pad_len < blocksize, 'padding error' assert pad_len < len(str), 'padding error' diff --git a/keen/__init__.py b/keen/__init__.py index 2a1be01..647e0d5 100644 --- a/keen/__init__.py +++ b/keen/__init__.py @@ -8,41 +8,74 @@ project_id = None write_key = None read_key = None - +master_key = None +base_url = None def _initialize_client_from_environment(): - global _client, project_id, write_key, read_key + ''' Initialize a KeenClient instance using environment variables. ''' + global _client, project_id, write_key, read_key, master_key, base_url if _client is None: # check environment for project ID and keys project_id = project_id or os.environ.get("KEEN_PROJECT_ID") write_key = write_key or os.environ.get("KEEN_WRITE_KEY") read_key = read_key or os.environ.get("KEEN_READ_KEY") + master_key = master_key or os.environ.get("KEEN_MASTER_KEY") + base_url = base_url or os.environ.get("KEEN_BASE_URL") if not project_id: raise InvalidEnvironmentError("Please set the KEEN_PROJECT_ID environment variable or set keen.project_id!") _client = KeenClient(project_id, write_key=write_key, - read_key=read_key) + read_key=read_key, + master_key=master_key, + base_url=base_url) def add_event(event_collection, body, timestamp=None): + """ Adds an event. + + Depending on the persistence strategy of the client, + this will either result in the event being uploaded to Keen + immediately or will result in saving the event to some local cache. + + :param event_collection: the name of the collection to insert the + event to + :param body: dict, the body of the event to insert the event to + :param timestamp: datetime, optional, the timestamp of the event + """ _initialize_client_from_environment() _client.add_event(event_collection, body, timestamp=timestamp) def add_events(events): + """ Adds a batch of events. + + Depending on the persistence strategy of the client, + this will either result in the event being uploaded to Keen + immediately or will result in saving the event to some local cache. + + :param events: dictionary of events + """ _initialize_client_from_environment() - _client.add_events(events) + return _client.add_events(events) def generate_image_beacon(event_collection, body, timestamp=None): + """ Generates an image beacon URL. + + :param event_collection: the name of the collection to insert the + event to + :param body: dict, the body of the event to insert the event to + :param timestamp: datetime, optional, the timestamp of the event + """ _initialize_client_from_environment() return _client.generate_image_beacon(event_collection, body, timestamp=timestamp) -def count(event_collection, timeframe=None, timezone=None, interval=None, filters=None, group_by=None): +def count(event_collection, timeframe=None, timezone=None, interval=None, filters=None, group_by=None, order_by=None, + max_age=None, limit=None): """ Performs a count query Counts the number of events that meet the given criteria. @@ -55,18 +88,26 @@ def count(event_collection, timeframe=None, timezone=None, interval=None, filter :param interval: string, the time interval used for measuring data over time example: "daily" :param filters: array of dict, contains the filters you'd like to apply to the data - example: {["property_name":"device", "operator":"eq", "property_value":"iPhone"}] + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would like to group you results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ _initialize_client_from_environment() return _client.count(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + max_age=max_age, limit=limit) def sum(event_collection, target_property, timeframe=None, timezone=None, interval=None, filters=None, - group_by=None): + group_by=None, order_by=None, max_age=None, limit=None): """ Performs a sum query Adds the values of a target property for events that meet the given criteria. @@ -80,18 +121,26 @@ def sum(event_collection, target_property, timeframe=None, timezone=None, interv :param interval: string, the time interval used for measuring data over time example: "daily" :param filters: array of dict, contains the filters you'd like to apply to the data - example: {["property_name":"device", "operator":"eq", "property_value":"iPhone"}] + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would like to group you results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ _initialize_client_from_environment() return _client.sum(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by, target_property=target_property) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age, limit=limit) def minimum(event_collection, target_property, timeframe=None, timezone=None, interval=None, filters=None, - group_by=None): + group_by=None, order_by=None, max_age=None, limit=None): """ Performs a minimum query Finds the minimum value of a target property for events that meet the given criteria. @@ -105,18 +154,26 @@ def minimum(event_collection, target_property, timeframe=None, timezone=None, in :param interval: string, the time interval used for measuring data over time example: "daily" :param filters: array of dict, contains the filters you'd like to apply to the data - example: {["property_name":"device", "operator":"eq", "property_value":"iPhone"}] + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would like to group you results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ _initialize_client_from_environment() return _client.minimum(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by, target_property=target_property) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age, limit=limit) def maximum(event_collection, target_property, timeframe=None, timezone=None, interval=None, filters=None, - group_by=None): + group_by=None, order_by=None, max_age=None, limit=None): """ Performs a maximum query Finds the maximum value of a target property for events that meet the given criteria. @@ -130,18 +187,26 @@ def maximum(event_collection, target_property, timeframe=None, timezone=None, in :param interval: string, the time interval used for measuring data over time example: "daily" :param filters: array of dict, contains the filters you'd like to apply to the data - example: {["property_name":"device", "operator":"eq", "property_value":"iPhone"}] + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would like to group you results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ _initialize_client_from_environment() return _client.maximum(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by, target_property=target_property) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age, limit=limit) def average(event_collection, target_property, timeframe=None, timezone=None, interval=None, filters=None, - group_by=None): + group_by=None, order_by=None, max_age=None, limit=None): """ Performs a average query Finds the average of a target property for events that meet the given criteria. @@ -155,18 +220,59 @@ def average(event_collection, target_property, timeframe=None, timezone=None, in :param interval: string, the time interval used for measuring data over time example: "daily" :param filters: array of dict, contains the filters you'd like to apply to the data - example: {["property_name":"device", "operator":"eq", "property_value":"iPhone"}] + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would like to group you results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ _initialize_client_from_environment() return _client.average(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by, target_property=target_property) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age, limit=limit) + + +def median(event_collection, target_property, timeframe=None, timezone=None, interval=None, filters=None, + group_by=None, order_by=None, max_age=None, limit=None): + """ Performs a median query + + Finds the median of a target property for events that meet the given criteria. + + :param event_collection: string, the name of the collection to query + :param target_property: string, the name of the event property you would like use + :param timeframe: string or dict, the timeframe in which the events + happened example: "previous_7_days" + :param timezone: int, the timezone you'd like to use for the timeframe + and interval in seconds + :param interval: string, the time interval used for measuring data over + time example: "daily" + :param filters: array of dict, contains the filters you'd like to apply to the data + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] + :param group_by: string or array of strings, the name(s) of the properties you would + like to group you results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds + + """ + _initialize_client_from_environment() + return _client.median(event_collection=event_collection, timeframe=timeframe, timezone=timezone, + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age) -def percentile(event_collection, target_property, percentile, timeframe=None, timezone=None, interval=None, filters=None, - group_by=None): +def percentile(event_collection, target_property, percentile, timeframe=None, timezone=None, interval=None, + filters=None, group_by=None, order_by=None, max_age=None, limit=None): """ Performs a percentile query Finds the percentile of a target property for events that meet the given criteria. @@ -182,9 +288,16 @@ def percentile(event_collection, target_property, percentile, timeframe=None, ti :param interval: string, the time interval used for measuring data over time example: "daily" :param filters: array of dict, contains the filters you'd like to apply to the data - example: {["property_name":"device", "operator":"eq", "property_value":"iPhone"}] + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would like to group you results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ _initialize_client_from_environment() @@ -196,12 +309,15 @@ def percentile(event_collection, target_property, percentile, timeframe=None, ti interval=interval, filters=filters, group_by=group_by, - target_property=target_property + order_by=order_by, + target_property=target_property, + max_age=max_age, + limit=limit ) def count_unique(event_collection, target_property, timeframe=None, timezone=None, interval=None, - filters=None, group_by=None): + filters=None, group_by=None, order_by=None, max_age=None, limit=None): """ Performs a count unique query Counts the unique values of a target property for events that meet the given criteria. @@ -215,18 +331,26 @@ def count_unique(event_collection, target_property, timeframe=None, timezone=Non :param interval: string, the time interval used for measuring data over time example: "daily" :param filters: array of dict, contains the filters you'd like to apply to the data - example: {["property_name":"device", "operator":"eq", "property_value":"iPhone"}] + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would like to group you results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ _initialize_client_from_environment() return _client.count_unique(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by, target_property=target_property) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age, limit=limit) def select_unique(event_collection, target_property, timeframe=None, timezone=None, interval=None, - filters=None, group_by=None): + filters=None, group_by=None, order_by=None, max_age=None, limit=None): """ Performs a select unique query Returns an array of the unique values of a target property for events that meet the given criteria. @@ -240,14 +364,22 @@ def select_unique(event_collection, target_property, timeframe=None, timezone=No :param interval: string, the time interval used for measuring data over time example: "daily" :param filters: array of dict, contains the filters you'd like to apply to the data - example: {["property_name":"device", "operator":"eq", "property_value":"iPhone"}] + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would like to group you results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ _initialize_client_from_environment() return _client.select_unique(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by, target_property=target_property) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age, limit=limit) def extraction(event_collection, timeframe=None, timezone=None, filters=None, latest=None, email=None, @@ -263,7 +395,7 @@ def extraction(event_collection, timeframe=None, timezone=None, filters=None, la :param timezone: int, the timezone you'd like to use for the timeframe and interval in seconds :param filters: array of dict, contains the filters you'd like to apply to the data - example: {["property_name":"device", "operator":"eq", "property_value":"iPhone"}] + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param latest: int, the number of most recent records you'd like to return :param email: string, optional string containing an email address to email results to :param property_names: string or list of strings, used to limit the properties returned @@ -274,7 +406,7 @@ def extraction(event_collection, timeframe=None, timezone=None, filters=None, la filters=filters, latest=latest, email=email, property_names=property_names) -def funnel(steps, timeframe=None, timezone=None): +def funnel(*args, **kwargs): """ Performs a Funnel query Returns an object containing the results for each step of the funnel. @@ -286,14 +418,16 @@ def funnel(steps, timeframe=None, timezone=None): happened example: "previous_7_days" :param timezone: int, the timezone you'd like to use for the timeframe and interval in seconds + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ _initialize_client_from_environment() - return _client.funnel(steps=steps, timeframe=timeframe, timezone=timezone) + return _client.funnel(*args, **kwargs) -def multi_analysis(event_collection, analyses, timeframe=None, interval=None, - timezone=None, filters=None, group_by=None): +def multi_analysis(event_collection, analyses, timeframe=None, interval=None, timezone=None, + filters=None, group_by=None, order_by=None, max_age=None, limit=None): """ Performs a multi-analysis query Returns a dictionary of analysis results. @@ -309,12 +443,188 @@ def multi_analysis(event_collection, analyses, timeframe=None, interval=None, :param timezone: int, the timezone you'd like to use for the timeframe and interval in seconds :param filters: array of dict, contains the filters you'd like to apply to the data - example: {["property_name":"device", "operator":"eq", "property_value":"iPhone"}] + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would like to group you results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ _initialize_client_from_environment() return _client.multi_analysis(event_collection=event_collection, timeframe=timeframe, interval=interval, timezone=timezone, filters=filters, - group_by=group_by, analyses=analyses) + group_by=group_by, order_by=order_by, analyses=analyses, + max_age=max_age, limit=limit) + + +def delete_events(*args, **kwargs): + """ Performs a delete for events. + + Returns true upon success. + + :param event_collection: string, the event collection from which event are being deleted + :param timeframe: string or dict, the timeframe in which the events + happened example: "previous_7_days" + :param timezone: int, the timezone you'd like to use for the timeframe + and interval in seconds + :param filters: array of dict, contains the filters you'd like to apply to the data + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] + + """ + _initialize_client_from_environment() + return _client.delete_events(*args, **kwargs) + + +def get_collection(*args, **kwargs): + """ Returns event collection schema + + :param event_collection: string, the event collection from which schema is to be returned, + if left blank will return schema for all collections + """ + _initialize_client_from_environment() + return _client.get_collection(*args, **kwargs) + + +def get_all_collections(): + """ Returns event collection schema for all events + + """ + _initialize_client_from_environment() + return _client.get_all_collections() + +def create_access_key(name, is_active=True, permitted=[], options={}): + """ Creates a new access key. A master key must be set first. + + :param name: the name of the access key to create + :param is_active: Boolean value dictating whether this key is currently active (default True) + :param permitted: list of strings describing which operation types this key will permit + Legal values include "writes", "queries", "saved_queries", "cached_queries", + "datasets", and "schema". + :param options: dictionary containing more details about the key's permitted and restricted + functionality + """ + _initialize_client_from_environment() + return _client.create_access_key(name=name, is_active=is_active, + permitted=permitted, options=options) + +def list_access_keys(): + """ + Returns a list of all access keys in this project. A master key must be set first. + """ + _initialize_client_from_environment() + return _client.list_access_keys() + +def get_access_key(key): + """ + Returns details on a particular access key. A master key must be set first. + + :param key: the 'key' value of the access key to retreive data from + """ + _initialize_client_from_environment() + return _client.get_access_key(key) + +def update_access_key_name(key, name): + """ + Updates only the name portion of an access key. + + :param key: the 'key' value of the access key to change the name of + :param name: the new name to give this access key + """ + _initialize_client_from_environment() + return _client.update_access_key_name(key, name) + +def add_access_key_permissions(key, permissions): + """ + Adds to the existing list of permissions on this key with the contents of this list. + Will not remove any existing permissions or modify the remainder of the key. + + :param key: the 'key' value of the access key to add permissions to + :param permissions: the new permissions to add to the existing list of permissions + """ + _initialize_client_from_environment() + return _client.add_access_key_permissions(key, permissions) + +def remove_access_key_permissions(key, permissions): + """ + Removes a list of permissions from the existing list of permissions. + Will not remove all existing permissions unless all such permissions are included + in this list. Not to be confused with key revocation. + + See also: revoke_access_key() + + :param key: the 'key' value of the access key to remove some permissions from + :param permissions: the permissions you wish to remove from this access key + """ + _initialize_client_from_environment() + return _client.remove_access_key_permissions(key, permissions) + +def update_access_key_permissions(key, permissions): + """ + Replaces all of the permissions on the access key but does not change + non-permission properties such as the key's name. + + See also: add_access_key_permissions() and remove_access_key_permissions(). + + :param key: the 'key' value of the access key to change the permissions of + :param permissions: the new list of permissions for this key + """ + _initialize_client_from_environment() + return _client.update_access_key_permissions(key, permissions) + +def update_access_key_options(key, options): + """ + Replaces all of the options on the access key but does not change + non-option properties such as permissions or the key's name. + + :param key: the 'key' value of the access key to change the options of + :param options: the new dictionary of options for this key + """ + _initialize_client_from_environment() + return _client.update_access_key_options(key, options) + +def update_access_key_full(key, name, is_active, permitted, options): + """ + Replaces the 'name', 'is_active', 'permitted', and 'options' values of a given key. + A master key must be set first. + + :param key: the 'key' value of the access key for which the values will be replaced + :param name: the new name desired for this access key + :param is_active: whether the key should become enabled (True) or revoked (False) + :param permitted: the new list of permissions desired for this access key + :param options: the new dictionary of options for this access key + """ + _initialize_client_from_environment() + return _client.update_access_key_full(key, name, is_active, permitted, options) + +def revoke_access_key(key): + """ + Revokes an access key. "Bad dog! No biscuit!" + + :param key: the 'key' value of the access key to revoke + """ + _initialize_client_from_environment() + return _client.revoke_access_key(key) + +def unrevoke_access_key(key): + """ + Re-enables an access key. + + :param key: the 'key' value of the access key to re-enable (unrevoke) + """ + _initialize_client_from_environment() + return _client.unrevoke_access_key(key) + + +def delete_access_key(key): + """ + Deletes an access key. + + :param key: the 'key' value of the access key to delete + """ + _initialize_client_from_environment() + return _client.delete_access_key(key) diff --git a/keen/api.py b/keen/api.py index 762f883..0d2a0e9 100644 --- a/keen/api.py +++ b/keen/api.py @@ -7,8 +7,12 @@ from requests.adapters import HTTPAdapter from requests.packages.urllib3.poolmanager import PoolManager -# keen exceptions -from keen import exceptions +# keen +from keen import direction, exceptions, utilities +from keen.utilities import KeenKeys, requires_key + +# json +from requests.compat import json __author__ = 'dkador' @@ -20,6 +24,8 @@ class HTTPMethods(object): GET = 'get' POST = 'post' + DELETE = 'delete' + PUT = 'put' class KeenAdapter(HTTPAdapter): @@ -28,12 +34,11 @@ class KeenAdapter(HTTPAdapter): def init_poolmanager(self, connections, maxsize, block=False): - """ Initialize pool manager with forced TLSv1 support. """ + """ Initialize pool manager """ self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, - block=block, - ssl_version=ssl.PROTOCOL_TLSv1) + block=block) class KeenApi(object): @@ -44,13 +49,15 @@ class KeenApi(object): # the default base URL of the Keen API base_url = "https://api.keen.io" + # the default version of the Keen API api_version = "3.0" # self says it belongs to KeenApi/andOr is the object passed into KeenApi # __init__ create keenapi object whenever KeenApi class is invoked def __init__(self, project_id, write_key=None, read_key=None, - base_url=None, api_version=None, get_timeout=None, post_timeout=None): + base_url=None, api_version=None, get_timeout=None, post_timeout=None, + master_key=None): """ Initializes a KeenApi object @@ -61,12 +68,16 @@ def __init__(self, project_id, write_key=None, read_key=None, are sent :param api_version: string, optional, set this to override what API version is used + :param get_timeout: optional, the timeout on GET requests + :param post_timeout: optional, the timeout on POST requests + :param master_key: a Keen IO Master API Key, needed for deletes """ # super? recreates the object with values passed into KeenApi super(KeenApi, self).__init__() self.project_id = project_id self.write_key = write_key self.read_key = read_key + self.master_key = master_key if base_url: self.base_url = base_url if api_version: @@ -75,43 +86,29 @@ def __init__(self, project_id, write_key=None, read_key=None, self.post_timeout = post_timeout self.session = self._create_session() - def _create_session(self): - - """ Build a session that uses KeenAdapter for SSL """ - - s = requests.Session() - s.mount('https://', KeenAdapter()) - return s - def fulfill(self, method, *args, **kwargs): """ Fulfill an HTTP request to Keen's API. """ return getattr(self.session, method)(*args, **kwargs) + @requires_key(KeenKeys.WRITE) def post_event(self, event): """ Posts a single event to the Keen IO API. The write key must be set first. :param event: an Event to upload """ - if not self.write_key: - raise exceptions.InvalidEnvironmentError( - "The Keen IO API requires a write key to send events. " - "Please set a 'write_key' when initializing the " - "KeenApi object." - ) url = "{0}/{1}/projects/{2}/events/{3}".format(self.base_url, self.api_version, self.project_id, event.event_collection) - headers = {"Content-Type": "application/json", "Authorization": self.write_key} + headers = utilities.headers(self.write_key) payload = event.to_json() response = self.fulfill(HTTPMethods.POST, url, data=payload, headers=headers, timeout=self.post_timeout) - if response.status_code != 201: - error = response.json() - raise exceptions.KeenApiError(error) + self._error_handling(response) + @requires_key(KeenKeys.WRITE) def post_events(self, events): """ @@ -119,42 +116,426 @@ def post_events(self, events): :param events: an Event to upload """ - if not self.write_key: - raise exceptions.InvalidEnvironmentError( - "The Keen IO API requires a write key to send events. " - "Please set a 'write_key' when initializing the " - "KeenApi object." - ) url = "{0}/{1}/projects/{2}/events".format(self.base_url, self.api_version, self.project_id) - headers = {"Content-Type": "application/json", "Authorization": self.write_key} + headers = utilities.headers(self.write_key) payload = json.dumps(events) response = self.fulfill(HTTPMethods.POST, url, data=payload, headers=headers, timeout=self.post_timeout) - if response.status_code != 200: - error = response.json() - raise exceptions.KeenApiError(error) + self._error_handling(response) + return self._get_response_json(response) + - def query(self, analysis_type, params): + def _order_by_is_valid_or_none(self, params): + """ + Validates that a given order_by has proper syntax. + + :param params: Query params. + :return: Returns True if either no order_by is present, or if the order_by is well-formed. + """ + if not "order_by" in params or not params["order_by"]: + return True + + def _order_by_dict_is_not_well_formed(d): + if not isinstance(d, dict): + # Bad type. + return True + if "property_name" in d and d["property_name"]: + if "direction" in d and not direction.is_valid_direction(d["direction"]): + # Bad direction provided. + return True + for k in d: + if k != "property_name" and k != "direction": + # Unexpected key. + return True + # Everything looks good! + return False + # Missing required key. + return True + + # order_by is converted to a list before this point if it wasn't one before. + order_by_list = json.loads(params["order_by"]) + + for order_by in order_by_list: + if _order_by_dict_is_not_well_formed(order_by): + return False + if not "group_by" in params or not params["group_by"]: + # We must have group_by to have order_by make sense. + return False + return True + + def _limit_is_valid_or_none(self, params): + """ + Validates that a given limit is not present or is well-formed. + + :param params: Query params. + :return: Returns True if a limit is present or is well-formed. + """ + if not "limit" in params or not params["limit"]: + return True + if not isinstance(params["limit"], int) or params["limit"] < 1: + return False + if not "order_by" in params: + return False + return True + + @requires_key(KeenKeys.READ) + def query(self, analysis_type, params, all_keys=False): """ Performs a query using the Keen IO analysis API. A read key must be set first. """ - if not self.read_key: - raise exceptions.InvalidEnvironmentError( - "The Keen IO API requires a read key to perform queries. " - "Please set a 'read_key' when initializing the " - "KeenApi object." - ) + if not self._order_by_is_valid_or_none(params): + raise ValueError("order_by given is invalid or is missing required group_by.") + if not self._limit_is_valid_or_none(params): + raise ValueError("limit given is invalid or is missing required order_by.") url = "{0}/{1}/projects/{2}/queries/{3}".format(self.base_url, self.api_version, self.project_id, analysis_type) - headers = {"Authorization": self.read_key} + headers = utilities.headers(self.read_key) payload = params response = self.fulfill(HTTPMethods.GET, url, params=payload, headers=headers, timeout=self.get_timeout) - if response.status_code != 200: - error = response.json() + self._error_handling(response) + + response = response.json() + + if not all_keys: + response = response["result"] + + return response + + @requires_key(KeenKeys.MASTER) + def delete_events(self, event_collection, params): + """ + Deletes events via the Keen IO API. A master key must be set first. + + :param event_collection: string, the event collection from which event are being deleted + + """ + + url = "{0}/{1}/projects/{2}/events/{3}".format(self.base_url, + self.api_version, + self.project_id, + event_collection) + headers = utilities.headers(self.master_key) + response = self.fulfill(HTTPMethods.DELETE, url, params=params, headers=headers, timeout=self.post_timeout) + + self._error_handling(response) + return True + + @requires_key(KeenKeys.READ) + def get_collection(self, event_collection): + """ + Extracts info about a collection using the Keen IO API. A master key must be set first. + + :param event_collection: the name of the collection to retrieve info for + """ + + url = "{0}/{1}/projects/{2}/events/{3}".format(self.base_url, self.api_version, + self.project_id, event_collection) + headers = utilities.headers(self.read_key) + response = self.fulfill(HTTPMethods.GET, url, headers=headers, timeout=self.get_timeout) + self._error_handling(response) + + return response.json() + + @requires_key(KeenKeys.READ) + def get_all_collections(self): + """ + Extracts schema for all collections using the Keen IO API. A read key must be set first. + + """ + + url = "{0}/{1}/projects/{2}/events".format(self.base_url, self.api_version, self.project_id) + headers = utilities.headers(self.read_key) + response = self.fulfill(HTTPMethods.GET, url, headers=headers, timeout=self.get_timeout) + self._error_handling(response) + + return response.json() + + @requires_key(KeenKeys.MASTER) + def create_access_key(self, name, is_active=True, permitted=[], options={}): + """ + Creates a new access key. A master key must be set first. + + :param name: the name of the access key to create + :param is_active: Boolean value dictating whether this key is currently active (default True) + :param permitted: list of strings describing which operation types this key will permit + Legal values include "writes", "queries", "saved_queries", "cached_queries", + "datasets", and "schema". + :param options: dictionary containing more details about the key's permitted and restricted + functionality + """ + + url = "{0}/{1}/projects/{2}/keys".format(self.base_url, self.api_version, self.project_id) + headers = utilities.headers(self.master_key) + + payload_dict = { + "name": name, + "is_active": is_active, + "permitted": permitted, + "options": options + } + payload = json.dumps(payload_dict) + + response = self.fulfill(HTTPMethods.POST, url, data=payload, headers=headers, timeout=self.get_timeout) + self._error_handling(response) + return response.json() + + @requires_key(KeenKeys.MASTER) + def list_access_keys(self): + """ + Returns a list of all access keys in this project. A master key must be set first. + """ + url = "{0}/{1}/projects/{2}/keys".format(self.base_url, self.api_version, self.project_id) + headers = utilities.headers(self.master_key) + response = self.fulfill(HTTPMethods.GET, url, headers=headers, timeout=self.get_timeout) + self._error_handling(response) + + return response.json() + + @requires_key(KeenKeys.MASTER) + def get_access_key(self, key): + """ + Returns details on a particular access key. A master key must be set first. + + :param key: the 'key' value of the access key to retreive data from + """ + url = "{0}/{1}/projects/{2}/keys/{3}".format(self.base_url, self.api_version, self.project_id, + key) + headers = utilities.headers(self.master_key) + response = self.fulfill(HTTPMethods.GET, url, headers=headers, timeout=self.get_timeout) + self._error_handling(response) + + return response.json() + + @staticmethod + def _build_access_key_dict(access_key): + """ + Populates a dictionary payload usable in a POST request from a full access key object. + + :param access_key: the access_key to copy data from + """ + return { + "name": access_key["name"], + "is_active": access_key["is_active"], + "permitted": access_key["permitted"], + "options": access_key["options"] + } + + def _update_access_key_pair(self, key, field, val): + """ + Helper for updating access keys in a DRY fashion. + """ + # Get current state via HTTPS. + current_access_key = self.get_access_key(key) + + # Copy and only change the single parameter. + payload_dict = KeenApi._build_access_key_dict(current_access_key) + payload_dict[field] = val + + # Now just treat it like a full update. + return self.update_access_key_full(key, **payload_dict) + + @requires_key(KeenKeys.MASTER) + def update_access_key_name(self, key, name): + """ + Updates only the name portion of an access key. + + :param key: the 'key' value of the access key to change the name of + :param name: the new name to give this access key + """ + return self._update_access_key_pair(key, "name", name) + + @requires_key(KeenKeys.MASTER) + def add_access_key_permissions(self, key, permissions): + """ + Adds to the existing list of permissions on this key with the contents of this list. + Will not remove any existing permissions or modify the remainder of the key. + + :param key: the 'key' value of the access key to add permissions to + :param permissions: the new permissions to add to the existing list of permissions + """ + # Get current state via HTTPS. + current_access_key = self.get_access_key(key) + + # Copy and only change the single parameter. + payload_dict = KeenApi._build_access_key_dict(current_access_key) + + # Turn into sets to avoid duplicates. + old_permissions = set(payload_dict["permitted"]) + new_permissions = set(permissions) + combined_permissions = old_permissions.union(new_permissions) + payload_dict["permitted"] = list(combined_permissions) + + # Now just treat it like a full update. + return self.update_access_key_full(key, **payload_dict) + + @requires_key(KeenKeys.MASTER) + def remove_access_key_permissions(self, key, permissions): + """ + Removes a list of permissions from the existing list of permissions. + Will not remove all existing permissions unless all such permissions are included + in this list. Not to be confused with key revocation. + + See also: revoke_access_key() + + :param key: the 'key' value of the access key to remove some permissions from + :param permissions: the permissions you wish to remove from this access key + """ + # Get current state via HTTPS. + current_access_key = self.get_access_key(key) + + # Copy and only change the single parameter. + payload_dict = KeenApi._build_access_key_dict(current_access_key) + + # Turn into sets to avoid duplicates. + old_permissions = set(payload_dict["permitted"]) + removal_permissions = set(permissions) + reduced_permissions = old_permissions.difference(removal_permissions) + payload_dict["permitted"] = list(reduced_permissions) + + # Now just treat it like a full update. + return self.update_access_key_full(key, **payload_dict) + + @requires_key(KeenKeys.MASTER) + def update_access_key_permissions(self, key, permissions): + """ + Replaces all of the permissions on the access key but does not change + non-permission properties such as the key's name. + + See also: add_access_key_permissions() and remove_access_key_permissions(). + + :param key: the 'key' value of the access key to change the permissions of + :param permissions: the new list of permissions for this key + """ + return self._update_access_key_pair(key, "permitted", permissions) + + @requires_key(KeenKeys.MASTER) + def update_access_key_options(self, key, options): + """ + Replaces all of the options on the access key but does not change + non-option properties such as permissions or the key's name. + + :param key: the 'key' value of the access key to change the options of + :param options: the new dictionary of options for this key + """ + return self._update_access_key_pair(key, "options", options) + + + @requires_key(KeenKeys.MASTER) + def update_access_key_full(self, key, name, is_active, permitted, options): + """ + Replaces the 'name', 'is_active', 'permitted', and 'options' values of a given key. + A master key must be set first. + + :param key: the 'key' value of the access key for which the values will be replaced + :param name: the new name desired for this access key + :param is_active: whether the key should become enabled (True) or revoked (False) + :param permitted: the new list of permissions desired for this access key + :param options: the new dictionary of options for this access key + """ + url = "{0}/{1}/projects/{2}/keys/{3}".format(self.base_url, self.api_version, + self.project_id, key) + headers = utilities.headers(self.master_key) + payload_dict = { + "name": name, + "is_active": is_active, + "permitted": permitted, + "options": options + } + payload = json.dumps(payload_dict) + response = self.fulfill(HTTPMethods.POST, url, data=payload, headers=headers, timeout=self.get_timeout) + self._error_handling(response) + return response.json() + + @requires_key(KeenKeys.MASTER) + def revoke_access_key(self, key): + """ + Revokes an access key. "Bad dog! No biscuit!" + + :param key: the 'key' value of the access key to revoke + """ + url = "{0}/{1}/projects/{2}/keys/{3}/revoke".format(self.base_url, self.api_version, + self.project_id, key) + headers = utilities.headers(self.master_key) + response = self.fulfill(HTTPMethods.POST, url, headers=headers, timeout=self.get_timeout) + + self._error_handling(response) + return response.json() + + @requires_key(KeenKeys.MASTER) + def unrevoke_access_key(self, key): + """ + Re-enables an access key. + + :param key: the 'key' value of the access key to re-enable (unrevoke) + """ + url = "{0}/{1}/projects/{2}/keys/{3}/unrevoke".format(self.base_url, self.api_version, + self.project_id, key) + headers = utilities.headers(self.master_key) + response = self.fulfill(HTTPMethods.POST, url, headers=headers, timeout=self.get_timeout) + + self._error_handling(response) + return response.json() + + @requires_key(KeenKeys.MASTER) + def delete_access_key(self, key): + """ + Deletes an access key. + + :param key: the 'key' value of the access key to delete + """ + url = "{0}/{1}/projects/{2}/keys/{3}".format(self.base_url, self.api_version, self.project_id, key) + headers = utilities.headers(self.master_key) + response = self.fulfill(HTTPMethods.DELETE, url, headers=headers, timeout=self.get_timeout) + + self._error_handling(response) + return True + + def _error_handling(self, res): + """ + Helper function to do the error handling + + :params res: the response from a request + """ + + # making the error handling generic so if an status_code starting with 2 doesn't exist, we raise the error + if res.status_code // 100 != 2: + error = self._get_response_json(res) raise exceptions.KeenApiError(error) - return response.json()["result"] + def _get_response_json(self, res): + """ + Helper function to extract the JSON body out of a response OR throw an exception. + + :param res: the response from a request + :return: the JSON body OR throws an exception + """ + + try: + error = res.json() + except ValueError: + error = { + "message": "The API did not respond with JSON, but: {0}".format(res.text[:1000]), + "error_code": "{0}".format(res.status_code) + } + return error + + def _create_session(self): + + """ Build a session that uses KeenAdapter for SSL """ + + s = requests.Session() + s.mount('https://', KeenAdapter()) + return s + + def _get_read_key(self): + return self.read_key + + def _get_write_key(self): + return self.write_key + + def _get_master_key(self): + return self.master_key \ No newline at end of file diff --git a/keen/cached_datasets.py b/keen/cached_datasets.py new file mode 100644 index 0000000..a765884 --- /dev/null +++ b/keen/cached_datasets.py @@ -0,0 +1,91 @@ +import json + +from keen.api import HTTPMethods +from keen.utilities import KeenKeys, headers, requires_key + + +class CachedDatasetsInterface: + + def __init__(self, api): + self.api = api + self._cached_datasets_url = "{0}/{1}/projects/{2}/datasets".format( + self.api.base_url, self.api.api_version, self.api.project_id + ) + + @requires_key(KeenKeys.READ) + def all(self): + """ Fetch all Cached Datasets for a Project. Read key must be set. + """ + return self._get_json(HTTPMethods.GET, + self._cached_datasets_url, + self._get_master_key()) + + @requires_key(KeenKeys.READ) + def get(self, dataset_name): + """ Fetch a single Cached Dataset for a Project. Read key must be set. + + :param dataset_name: Name of Cached Dataset (not `display_name`) + """ + url = "{0}/{1}".format(self._cached_datasets_url, dataset_name) + return self._get_json(HTTPMethods.GET, url, self._get_read_key()) + + @requires_key(KeenKeys.MASTER) + def create(self, dataset_name, query, index_by, display_name): + """ Create a Cached Dataset for a Project. Master key must be set. + """ + url = "{0}/{1}".format(self._cached_datasets_url, dataset_name) + payload = { + "query": query, + "index_by": index_by, + "display_name": display_name + } + return self._get_json(HTTPMethods.PUT, url, self._get_master_key(), json=payload) + + @requires_key(KeenKeys.READ) + def results(self, dataset_name, index_by, timeframe): + """ Retrieve results from a Cached Dataset. Read key must be set. + """ + url = "{0}/{1}/results".format(self._cached_datasets_url, dataset_name) + + index_by = index_by if isinstance(index_by, str) else json.dumps(index_by) + timeframe = timeframe if isinstance(timeframe, str) else json.dumps(timeframe) + + query_params = { + "index_by": index_by, + "timeframe": timeframe + } + + return self._get_json( + HTTPMethods.GET, url, self._get_read_key(), params=query_params + ) + + @requires_key(KeenKeys.MASTER) + def delete(self, dataset_name): + """ Delete a Cached Dataset. Master Key must be set. + """ + url = "{0}/{1}".format(self._cached_datasets_url, dataset_name) + self._get_json(HTTPMethods.DELETE, url, self._get_master_key()) + return True + + def _get_json(self, http_method, url, key, *args, **kwargs): + response = self.api.fulfill( + http_method, + url, + headers=headers(key), + *args, + **kwargs) + + self.api._error_handling(response) + + try: + response = response.json() + except ValueError: + response = "No JSON available." + + return response + + def _get_read_key(self): + return self.api._get_read_key() + + def _get_master_key(self): + return self.api._get_master_key() diff --git a/keen/client.py b/keen/client.py index b37a406..2ce6eff 100644 --- a/keen/client.py +++ b/keen/client.py @@ -2,7 +2,7 @@ import copy import json import sys -from keen import persistence_strategies, exceptions +from keen import persistence_strategies, exceptions, saved_queries, cached_datasets from keen.api import KeenApi from keen.persistence_strategies import BasePersistenceStrategy @@ -37,7 +37,10 @@ def to_json(self): """ event_as_dict = copy.deepcopy(self.event_body) if self.timestamp: - event_as_dict["keen"] = {"timestamp": self.timestamp.isoformat()} + if "keen" in event_as_dict: + event_as_dict["keen"]["timestamp"] = self.timestamp.isoformat() + else: + event_as_dict["keen"] = {"timestamp": self.timestamp.isoformat()} return json.dumps(event_as_dict) @@ -57,7 +60,8 @@ class KeenClient(object): """ def __init__(self, project_id, write_key=None, read_key=None, - persistence_strategy=None, api_class=KeenApi, get_timeout=305, post_timeout=305): + persistence_strategy=None, api_class=KeenApi, get_timeout=305, post_timeout=305, + master_key=None, base_url=None): """ Initializes a KeenClient object. :param project_id: the Keen IO project ID @@ -67,6 +71,7 @@ def __init__(self, project_id, write_key=None, read_key=None, the event :param get_timeout: optional, the timeout on GET requests :param post_timeout: optional, the timeout on POST requests + :param master_key: a Keen IO Master API Key """ super(KeenClient, self).__init__() @@ -76,7 +81,8 @@ def __init__(self, project_id, write_key=None, read_key=None, # Set up an api client to be used for querying and optionally passed # into a default persistence strategy. self.api = api_class(project_id, write_key=write_key, read_key=read_key, - get_timeout=get_timeout, post_timeout=post_timeout) + get_timeout=get_timeout, post_timeout=post_timeout, + master_key=master_key, base_url=base_url) if persistence_strategy: # validate the given persistence strategy @@ -91,6 +97,8 @@ def __init__(self, project_id, write_key=None, read_key=None, self.persistence_strategy = persistence_strategy self.get_timeout = get_timeout self.post_timeout = post_timeout + self.saved_queries = saved_queries.SavedQueriesInterface(self.api) + self.cached_datasets = cached_datasets.CachedDatasetsInterface(self.api) if sys.version_info[0] < 3: @staticmethod @@ -109,7 +117,6 @@ def check_project_id(project_id): if not project_id or not isinstance(project_id, str): raise exceptions.InvalidProjectIdError(project_id) - def add_event(self, event_collection, event_body, timestamp=None): """ Adds an event. @@ -135,7 +142,7 @@ def add_events(self, events): :param events: dictionary of events """ - self.persistence_strategy.batch_persist(events) + return self.persistence_strategy.batch_persist(events) def generate_image_beacon(self, event_collection, event_body, timestamp=None): """ Generates an image beacon URL. @@ -153,6 +160,158 @@ def generate_image_beacon(self, event_collection, event_body, timestamp=None): self.api.write_key.decode(sys.getdefaultencoding()), self._base64_encode(event_json) ) + def delete_events(self, event_collection, timeframe=None, timezone=None, filters=None): + """ Deletes events. + + :param event_collection: string, the event collection from which event are being deleted + :param timeframe: string or dict, the timeframe in which the events happened + example: "previous_7_days" + :param timezone: int, the timezone you'd like to use for the timeframe + and interval in seconds + :param filters: array of dict, contains the filters you'd like to apply to the data + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] + + """ + params = self.get_params(timeframe=timeframe, timezone=timezone, filters=filters) + return self.api.delete_events(event_collection, params) + + def get_collection(self, event_collection): + """ Returns event collection schema + + :param event_collection: string, the event collection from which schema is to be returned, + if left blank will return schema for all collections + """ + + return self.api.get_collection(event_collection) + + def get_all_collections(self): + """ Returns event collection schema for all events + + """ + + return self.api.get_all_collections() + + def create_access_key(self, name, is_active=True, permitted=[], options={}): + """ + Creates a new access key. A master key must be set first. + + :param name: the name of the access key to create + :param is_active: Boolean value dictating whether this key is currently active (default True) + :param permitted: list of strings describing which operation types this key will permit + Legal values include "writes", "queries", "saved_queries", "cached_queries", + "datasets", and "schema". + :param options: dictionary containing more details about the key's permitted and restricted + functionality + """ + + return self.api.create_access_key(name=name, is_active=is_active, + permitted=permitted, options=options) + + def list_access_keys(self): + """ + Returns a list of all access keys in this project. A master key must be set first. + """ + return self.api.list_access_keys() + + def get_access_key(self, key): + """ + Returns details on a particular access key. A master key must be set first. + + :param key: the 'key' value of the access key to retreive data from + """ + return self.api.get_access_key(key) + + def update_access_key_name(self, key, name): + """ + Updates only the name portion of an access key. + + :param key: the 'key' value of the access key to change the name of + :param name: the new name to give this access key + """ + return self.api.update_access_key_name(key, name) + + def add_access_key_permissions(self, key, permissions): + """ + Adds to the existing list of permissions on this key with the contents of this list. + Will not remove any existing permissions or modify the remainder of the key. + + :param key: the 'key' value of the access key to add permissions to + :param permissions: the new permissions to add to the existing list of permissions + """ + return self.api.add_access_key_permissions(key, permissions) + + def remove_access_key_permissions(self, key, permissions): + """ + Removes a list of permissions from the existing list of permissions. + Will not remove all existing permissions unless all such permissions are included + in this list. Not to be confused with key revocation. + + See also: revoke_access_key() + + :param key: the 'key' value of the access key to remove some permissions from + :param permissions: the permissions you wish to remove from this access key + """ + return self.api.remove_access_key_permissions(key, permissions) + + def update_access_key_permissions(self, key, permissions): + """ + Replaces all of the permissions on the access key but does not change + non-permission properties such as the key's name. + + See also: add_access_key_permissions() and remove_access_key_permissions(). + + :param key: the 'key' value of the access key to change the permissions of + :param permissions: the new list of permissions for this key + """ + return self.api.update_access_key_permissions(key, permissions) + + def update_access_key_options(self, key, options): + """ + Replaces all of the options on the access key but does not change + non-option properties such as permissions or the key's name. + + :param key: the 'key' value of the access key to change the options of + :param options: the new dictionary of options for this key + """ + return self.api.update_access_key_options(key, options) + + def update_access_key_full(self, key, name, is_active, permitted, options): + """ + Replaces the 'name', 'is_active', 'permitted', and 'options' values of a given key. + A master key must be set first. + + :param key: the 'key' value of the access key for which the values will be replaced + :param name: the new name desired for this access key + :param is_active: whether the key should become enabled (True) or revoked (False) + :param permitted: the new list of permissions desired for this access key + :param options: the new dictionary of options for this access key + """ + return self.api.update_access_key_full(key, name, is_active, permitted, options) + + def revoke_access_key(self, key): + """ + Revokes an access key. "Bad dog! No biscuit!" + + :param key: the 'key' value of the access key to revoke + """ + return self.api.revoke_access_key(key) + + def unrevoke_access_key(self, key): + """ + Re-enables an access key. + + :param key: the 'key' value of the access key to re-enable (unrevoke) + """ + return self.api.unrevoke_access_key(key) + + def delete_access_key(self, key): + """ + Deletes an access key. + + :param key: the 'key' value of the access key to delete + """ + return self.api.delete_access_key(key) + def _base64_encode(self, string_to_encode): """ Base64 encodes a string, with either Python 2 or 3. @@ -175,7 +334,8 @@ def _url_escape(self, url): import urllib.parse return urllib.parse.quote(url) - def count(self, event_collection, timeframe=None, timezone=None, interval=None, filters=None, group_by=None): + def count(self, event_collection, timeframe=None, timezone=None, interval=None, + filters=None, group_by=None, order_by=None, max_age=None, limit=None): """ Performs a count query Counts the number of events that meet the given criteria. @@ -190,15 +350,23 @@ def count(self, event_collection, timeframe=None, timezone=None, interval=None, :param filters: array of dict, contains the filters you'd like to apply to the data example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would - like to group you results by. example: "customer.id" or ["browser","operating_system"] + like to group your results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ params = self.get_params(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + max_age=max_age, limit=limit) return self.api.query("count", params) def sum(self, event_collection, target_property, timeframe=None, timezone=None, interval=None, filters=None, - group_by=None): + group_by=None, order_by=None, max_age=None, limit=None): """ Performs a sum query Adds the values of a target property for events that meet the given criteria. @@ -214,15 +382,23 @@ def sum(self, event_collection, target_property, timeframe=None, timezone=None, :param filters: array of dict, contains the filters you'd like to apply to the data example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would - like to group you results by. example: "customer.id" or ["browser","operating_system"] + like to group your results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ params = self.get_params(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by, target_property=target_property) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age, limit=limit) return self.api.query("sum", params) - def minimum(self, event_collection, target_property, timeframe=None, timezone=None, interval=None, filters=None, - group_by=None): + def minimum(self, event_collection, target_property, timeframe=None, timezone=None, interval=None, + filters=None, group_by=None, order_by=None, max_age=None, limit=None): """ Performs a minimum query Finds the minimum value of a target property for events that meet the given criteria. @@ -238,15 +414,23 @@ def minimum(self, event_collection, target_property, timeframe=None, timezone=No :param filters: array of dict, contains the filters you'd like to apply to the data example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would - like to group you results by. example: "customer.id" or ["browser","operating_system"] + like to group your results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ params = self.get_params(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by, target_property=target_property) + interval=interval, filters=filters, group_by=group_by, + target_property=target_property, max_age=max_age, limit=limit) return self.api.query("minimum", params) - def maximum(self, event_collection, target_property, timeframe=None, timezone=None, interval=None, filters=None, - group_by=None): + def maximum(self, event_collection, target_property, timeframe=None, timezone=None, interval=None, + filters=None, group_by=None, order_by=None, max_age=None, limit=None): """ Performs a maximum query Finds the maximum value of a target property for events that meet the given criteria. @@ -262,15 +446,23 @@ def maximum(self, event_collection, target_property, timeframe=None, timezone=No :param filters: array of dict, contains the filters you'd like to apply to the data example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would - like to group you results by. example: "customer.id" or ["browser","operating_system"] + like to group your results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ params = self.get_params(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by, target_property=target_property) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age, limit=limit) return self.api.query("maximum", params) - def average(self, event_collection, target_property, timeframe=None, timezone=None, interval=None, filters=None, - group_by=None): + def average(self, event_collection, target_property, timeframe=None, timezone=None, interval=None, + filters=None, group_by=None, order_by=None, max_age=None, limit=None): """ Performs a average query Finds the average of a target property for events that meet the given criteria. @@ -286,15 +478,55 @@ def average(self, event_collection, target_property, timeframe=None, timezone=No :param filters: array of dict, contains the filters you'd like to apply to the data example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would - like to group you results by. example: "customer.id" or ["browser","operating_system"] + like to group your results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ params = self.get_params(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by, target_property=target_property) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age, limit=limit) return self.api.query("average", params) - def percentile(self, event_collection, target_property, percentile, timeframe=None, timezone=None, interval=None, filters=None, - group_by=None): + def median(self, event_collection, target_property, timeframe=None, timezone=None, interval=None, + filters=None, group_by=None, order_by=None, max_age=None, limit=None): + """ Performs a median query + + Finds the median of a target property for events that meet the given criteria. + + :param event_collection: string, the name of the collection to query + :param target_property: string, the name of the event property you would like use + :param timeframe: string or dict, the timeframe in which the events + happened example: "previous_7_days" + :param timezone: int, the timezone you'd like to use for the timeframe + and interval in seconds + :param interval: string, the time interval used for measuring data over + time example: "daily" + :param filters: array of dict, contains the filters you'd like to apply to the data + example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] + :param group_by: string or array of strings, the name(s) of the properties you would + like to group your results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds + + """ + params = self.get_params(event_collection=event_collection, timeframe=timeframe, timezone=timezone, + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age, limit=limit) + return self.api.query("median", params) + + def percentile(self, event_collection, target_property, percentile, timeframe=None, timezone=None, + interval=None, filters=None, group_by=None, order_by=None, max_age=None, limit=None): """ Performs a percentile query Finds the percentile of a target property for events that meet the given criteria. @@ -312,7 +544,14 @@ def percentile(self, event_collection, target_property, percentile, timeframe=No :param filters: array of dict, contains the filters you'd like to apply to the data example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would - like to group you results by. example: "customer.id" or ["browser","operating_system"] + like to group your results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ params = self.get_params( @@ -323,12 +562,15 @@ def percentile(self, event_collection, target_property, percentile, timeframe=No interval=interval, filters=filters, group_by=group_by, - target_property=target_property + order_by=order_by, + target_property=target_property, + max_age=max_age, + limit=limit ) return self.api.query("percentile", params) def count_unique(self, event_collection, target_property, timeframe=None, timezone=None, interval=None, - filters=None, group_by=None): + filters=None, group_by=None, order_by=None, max_age=None, limit=None): """ Performs a count unique query Counts the unique values of a target property for events that meet the given criteria. @@ -344,15 +586,23 @@ def count_unique(self, event_collection, target_property, timeframe=None, timezo :param filters: array of dict, contains the filters you'd like to apply to the data example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would - like to group you results by. example: "customer.id" or ["browser","operating_system"] + like to group your results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ params = self.get_params(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by, target_property=target_property) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age, limit=limit) return self.api.query("count_unique", params) def select_unique(self, event_collection, target_property, timeframe=None, timezone=None, interval=None, - filters=None, group_by=None): + filters=None, group_by=None, order_by=None, max_age=None, limit=None): """ Performs a select unique query Returns an array of the unique values of a target property for events that meet the given criteria. @@ -368,15 +618,23 @@ def select_unique(self, event_collection, target_property, timeframe=None, timez :param filters: array of dict, contains the filters you'd like to apply to the data example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would - like to group you results by. example: "customer.id" or ["browser","operating_system"] + like to group your results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ params = self.get_params(event_collection=event_collection, timeframe=timeframe, timezone=timezone, - interval=interval, filters=filters, group_by=group_by, target_property=target_property) + interval=interval, filters=filters, group_by=group_by, order_by=order_by, + target_property=target_property, max_age=max_age, limit=limit) return self.api.query("select_unique", params) - def extraction(self, event_collection, timeframe=None, timezone=None, filters=None, latest=None, email=None, - property_names=None): + def extraction(self, event_collection, timeframe=None, timezone=None, filters=None, latest=None, + email=None, property_names=None): """ Performs a data extraction Returns either a JSON object of events or a response @@ -392,12 +650,13 @@ def extraction(self, event_collection, timeframe=None, timezone=None, filters=No :param latest: int, the number of most recent records you'd like to return :param email: string, optional string containing an email address to email results to :param property_names: string or list of strings, used to limit the properties returned + """ params = self.get_params(event_collection=event_collection, timeframe=timeframe, timezone=timezone, filters=filters, latest=latest, email=email, property_names=property_names) return self.api.query("extraction", params) - def funnel(self, steps, timeframe=None, timezone=None): + def funnel(self, steps, timeframe=None, timezone=None, max_age=None, all_keys=False): """ Performs a Funnel query Returns an object containing the results for each step of the funnel. @@ -409,13 +668,22 @@ def funnel(self, steps, timeframe=None, timezone=None): happened example: "previous_7_days" :param timezone: int, the timezone you'd like to use for the timeframe and interval in seconds + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds + :all_keys: set to true to return all keys on response (i.e. "result", "actors", "steps") """ - params = self.get_params(steps=steps, timeframe=timeframe, timezone=timezone) - return self.api.query("funnel", params) + params = self.get_params( + steps=steps, + timeframe=timeframe, + timezone=timezone, + max_age=max_age, + ) + + return self.api.query("funnel", params, all_keys=all_keys) def multi_analysis(self, event_collection, analyses, timeframe=None, interval=None, timezone=None, filters=None, - group_by=None): + group_by=None, order_by=None, max_age=None, limit=None): """ Performs a multi-analysis query Returns a dictionary of analysis results. @@ -433,7 +701,14 @@ def multi_analysis(self, event_collection, analyses, timeframe=None, interval=No :param filters: array of dict, contains the filters you'd like to apply to the data example: [{"property_name":"device", "operator":"eq", "property_value":"iPhone"}] :param group_by: string or array of strings, the name(s) of the properties you would - like to group you results by. example: "customer.id" or ["browser","operating_system"] + like to group your results by. example: "customer.id" or ["browser","operating_system"] + :param order_by: dictionary or list of dictionary objects containing the property_name(s) + to order by and the desired direction(s) of sorting. + Example: {"property_name":"result", "direction":keen.direction.DESCENDING} + May not be used without a group_by specified. + :param limit: positive integer limiting the displayed results of a query using order_by + :param max_age: an integer, greater than 30 seconds, the maximum 'staleness' you're + willing to trade for increased query performance, in seconds """ params = self.get_params( @@ -443,13 +718,17 @@ def multi_analysis(self, event_collection, analyses, timeframe=None, interval=No timezone=timezone, filters=filters, group_by=group_by, - analyses=analyses) + order_by=order_by, + analyses=analyses, + max_age=max_age, + limit=limit + ) return self.api.query("multi_analysis", params) def get_params(self, event_collection=None, timeframe=None, timezone=None, interval=None, filters=None, - group_by=None, target_property=None, latest=None, email=None, analyses=None, steps=None, - property_names=None, percentile=None): + group_by=None, order_by=None, target_property=None, latest=None, email=None, analyses=None, + steps=None, property_names=None, percentile=None, max_age=None, limit=None): params = {} if event_collection: params["event_collection"] = event_collection @@ -469,6 +748,13 @@ def get_params(self, event_collection=None, timeframe=None, timezone=None, inter params["group_by"] = json.dumps(group_by) else: params["group_by"] = group_by + if order_by: + if isinstance(order_by, list): + params["order_by"] = json.dumps(order_by) + else: + params["order_by"] = json.dumps([order_by]) + if limit: + params["limit"] = limit if target_property: params["target_property"] = target_property if latest: @@ -483,5 +769,7 @@ def get_params(self, event_collection=None, timeframe=None, timezone=None, inter params["property_names"] = json.dumps(property_names) if percentile: params["percentile"] = percentile + if max_age: + params["max_age"] = max_age return params diff --git a/keen/direction.py b/keen/direction.py new file mode 100644 index 0000000..fbbd36a --- /dev/null +++ b/keen/direction.py @@ -0,0 +1,14 @@ +# Helper file for avoiding string literals prone to typos with order_by. + +ASCENDING = "ASC" +DESCENDING = "DESC" + +def is_valid_direction(direction_string): + """ + Determines whether the given string is a valid direction string + for order_by. + + :param direction_string: A string representing an order_by direction. + :return: True if the string is a valid direction string, False otherwise. + """ + return direction_string in [ASCENDING, DESCENDING] diff --git a/keen/persistence_strategies.py b/keen/persistence_strategies.py index d46cbbc..48f2aca 100644 --- a/keen/persistence_strategies.py +++ b/keen/persistence_strategies.py @@ -41,7 +41,7 @@ def batch_persist(self, events): :param events: a batch of events to persist """ - self.api.post_events(events) + return self.api.post_events(events) class RedisPersistenceStrategy(BasePersistenceStrategy): diff --git a/keen/saved_queries.py b/keen/saved_queries.py new file mode 100644 index 0000000..0260a0b --- /dev/null +++ b/keen/saved_queries.py @@ -0,0 +1,190 @@ + +import json +import six + +try: + from collections.abc import Mapping # Python >=3.3 +except ImportError: + from collections import Mapping + +from keen.api import HTTPMethods +from keen import utilities +from keen.utilities import KeenKeys, requires_key + + +class SavedQueriesInterface: + + def __init__(self, api): + self.api = api + self.saved_query_url = "{0}/{1}/projects/{2}/queries/saved".format( + self.api.base_url, self.api.api_version, self.api.project_id + ) + + @requires_key(KeenKeys.MASTER) + def all(self): + """ + Gets all saved queries for a project from the Keen IO API. + Master key must be set. + """ + + response = self._get_json(HTTPMethods.GET, self.saved_query_url, self._get_master_key()) + + return response + + @requires_key(KeenKeys.MASTER) + def get(self, query_name): + """ + Gets a single saved query for a project from the Keen IO API given a + query name. + Master key must be set. + """ + + url = "{0}/{1}".format(self.saved_query_url, query_name) + response = self._get_json(HTTPMethods.GET, url, self._get_master_key()) + + return response + + @requires_key(KeenKeys.READ) + def results(self, query_name): + """ + Gets a single saved query with a 'result' object for a project from the + Keen IO API given a query name. + Read or Master key must be set. + """ + + url = "{0}/{1}/result".format(self.saved_query_url, query_name) + response = self._get_json(HTTPMethods.GET, url, self._get_read_key()) + + return response + + @requires_key(KeenKeys.MASTER) + def create(self, query_name, saved_query): + """ + Creates the saved query via a PUT request to Keen IO Saved Query endpoint. + Master key must be set. + """ + url = "{0}/{1}".format(self.saved_query_url, query_name) + payload = saved_query + + # To support clients that may have already called dumps() to work around how this used to + # work, make sure it's not a str. Hopefully it's some sort of mapping. When we actually + # try to send the request, client code will get an InvalidJSONError if payload isn't + # a json-formatted string. + if not isinstance(payload, str): + payload = json.dumps(saved_query) + + response = self._get_json(HTTPMethods.PUT, url, self._get_master_key(), data=payload) + + return response + + @requires_key(KeenKeys.MASTER) + def update_full(self, query_name, saved_query_full_definition): + """ + Updates the saved query via a PUT request to Keen IO Saved Query + endpoint. The entire query definition must be provided--anything + excluded will be considered an explicit removal of that property. + + Master key must be set. + """ + + return self.create(query_name, saved_query_full_definition) + + @requires_key(KeenKeys.MASTER) + def update(self, query_name, saved_query_attributes): + """ + Given a dict of attributes to be updated, update only those attributes + in the Saved Query at the resource given by 'query_name'. This will + perform two HTTP requests--one to fetch the query definition, and one + to set the new attributes. This method will intend to preserve any + other properties on the query. + + Master key must be set. + """ + + query_name_attr_name = "query_name" + refresh_rate_attr_name = "refresh_rate" + query_attr_name = "query" + metadata_attr_name = "metadata" + + old_saved_query = self.get(query_name) + + # Create a new query def to send back. We cannot send values for attributes like 'urls', + # 'last_modified_date', 'run_information', etc. + new_saved_query = { + query_name_attr_name: old_saved_query[query_name_attr_name], # expected + refresh_rate_attr_name: old_saved_query[refresh_rate_attr_name], # expected + query_attr_name: {} + } + + # If metadata was set, preserve it. The Explorer UI currently stores information here. + old_metadata = (old_saved_query[metadata_attr_name] + if metadata_attr_name in old_saved_query + else None) + + if old_metadata: + new_saved_query[metadata_attr_name] = old_metadata + + # Preserve any non-empty properties of the existing query. We get back values like None + # for 'group_by', 'interval' or 'timezone', but those aren't accepted values when updating. + old_query = old_saved_query[query_attr_name] # expected + + # Shallow copy since we want the entire object heirarchy to start with. + for (key, value) in six.iteritems(old_query): + if value: + new_saved_query[query_attr_name][key] = value + + # Now, recursively overwrite any attributes passed in. + SavedQueriesInterface._deep_update(new_saved_query, saved_query_attributes) + + return self.create(query_name, new_saved_query) + + @requires_key(KeenKeys.MASTER) + def delete(self, query_name): + """ + Deletes a saved query from a project with a query name. + Master key must be set. + """ + + url = "{0}/{1}".format(self.saved_query_url, query_name) + self._get_json(HTTPMethods.DELETE, url, self._get_master_key()) + + return True + + @staticmethod + def _deep_update(mapping, updates): + for (key, value) in six.iteritems(updates): + if isinstance(mapping, Mapping): + if isinstance(value, Mapping): + next_level_value = SavedQueriesInterface._deep_update(mapping.get(key, {}), + value) + mapping[key] = next_level_value + else: + mapping[key] = value + else: + # Turn the original key into a mapping if it wasn't already + mapping = { key: value } + + return mapping + + def _get_json(self, http_method, url, key, *args, **kwargs): + response = self.api.fulfill( + http_method, + url, + headers=utilities.headers(key), + *args, + **kwargs) + + self.api._error_handling(response) + + try: + response = response.json() + except ValueError: + response = "No JSON available." + + return response + + def _get_read_key(self): + return self.api.read_key + + def _get_master_key(self): + return self.api.master_key diff --git a/keen/scoped_keys.py b/keen/scoped_keys.py index 4697c3b..30794ec 100644 --- a/keen/scoped_keys.py +++ b/keen/scoped_keys.py @@ -1,6 +1,7 @@ import binascii import json import os +import six from Crypto.Cipher import AES from keen import Padding @@ -8,22 +9,58 @@ __author__ = 'dkador' # the block size for the cipher object; must be 16, 24, or 32 for AES -BLOCK_SIZE = 32 +OLD_BLOCK_SIZE = 32 +DEFAULT_ENCODING = 'UTF-8' -def _pad(s): + +def ensure_bytes(s, encoding=None): + if isinstance(s, six.text_type): + return s.encode(encoding or DEFAULT_ENCODING) + return s + + +def pad_aes256(s): + """ + Pads an input string to a given block size. + :param s: string + :returns: The padded string. + """ + if len(s) % AES.block_size == 0: + return s + + return Padding.appendPadding(s, blocksize=AES.block_size) + + +def unpad_aes256(s): + """ + Removes padding from an input string based on a given block size. + :param s: string + :returns: The unpadded string. + """ + if not s: + return s + + try: + return Padding.removePadding(s, blocksize=AES.block_size) + except AssertionError: + # if there's an error while removing padding, just return s. + return s + + +def old_pad(s): """ Pads an input string to a given block size. :param s: string :returns: The padded string. """ - if len(s) % BLOCK_SIZE == 0: + if len(s) % OLD_BLOCK_SIZE == 0: return s - return Padding.appendPadding(s, blocksize=BLOCK_SIZE) + return Padding.appendPadding(s, blocksize=OLD_BLOCK_SIZE) -def _unpad(s): +def old_unpad(s): """ Removes padding from an input string based on a given block size. :param s: string @@ -33,14 +70,68 @@ def _unpad(s): return s try: - return Padding.removePadding(s, blocksize=BLOCK_SIZE) + return Padding.removePadding(s, blocksize=OLD_BLOCK_SIZE) except AssertionError: # if there's an error while removing padding, just return s. return s -# encrypt with AES, encode with hex -def _encode_aes(key, plaintext): +# encrypt with AES-256-CBC, encode with hex +def encode_aes256(key, plaintext): + """ + Utility method to encode some given plaintext with the given key. Important thing to note: + + This is not a general purpose encryption method - it has specific semantics (see below for + details). + + Takes the given hex string key and converts it to a 256 bit binary blob. Then pads the given + plaintext to AES block size which is always 16 bytes, regardless of AES key size. Then + encrypts using AES-256-CBC using a random IV. Then converts both the IV and the ciphertext + to hex. Finally returns the IV appended by the ciphertext. + + :param key: string, 64 hex chars long + :param plaintext: string, any amount of data + """ + if len(key) != 64: + raise TypeError("encode_aes256() expects a 256 bit key encoded as a 64 hex character string") + + # generate AES.block_size cryptographically secure random bytes for our IV (initial value) + iv = os.urandom(AES.block_size) + # set up an AES cipher object + cipher = AES.new(binascii.unhexlify(key.encode('ascii')), mode=AES.MODE_CBC, IV=iv) + # encrypt the plaintext after padding it + ciphertext = cipher.encrypt(ensure_bytes(pad_aes256(plaintext))) + # append the hexed IV and the hexed ciphertext + iv_plus_encrypted = binascii.hexlify(iv) + binascii.hexlify(ciphertext) + # return that + return iv_plus_encrypted + + +def decode_aes256(key, iv_plus_encrypted): + """ + Utility method to decode a payload consisting of the hexed IV + the hexed ciphertext using + the given key. See above for more details. + + :param key: string, 64 hex characters long + :param iv_plus_encrypted: string, a hexed IV + hexed ciphertext + """ + # grab first AES.block_size bytes (aka 2 * AES.block_size characters of hex) - that's the IV + iv_size = 2 * AES.block_size + hexed_iv = iv_plus_encrypted[:iv_size] + # grab everything else - that's the ciphertext (aka encrypted message) + hexed_ciphertext = iv_plus_encrypted[iv_size:] + # unhex the iv and ciphertext + iv = binascii.unhexlify(hexed_iv) + ciphertext = binascii.unhexlify(hexed_ciphertext) + # set up the correct AES cipher object + cipher = AES.new(binascii.unhexlify(key.encode('ascii')), mode=AES.MODE_CBC, IV=iv) + # decrypt! + plaintext = cipher.decrypt(ciphertext) + # return the unpadded version of this + return unpad_aes256(plaintext) + + +def old_encode_aes(key, plaintext): """ Utility method to encode some given plaintext with the given key. Important thing to note: @@ -57,16 +148,16 @@ def _encode_aes(key, plaintext): # generate 16 cryptographically secure random bytes for our IV (initial value) iv = os.urandom(16) # set up an AES cipher object - cipher = AES.new(_pad(key), mode=AES.MODE_CBC, IV=iv) + cipher = AES.new(ensure_bytes(old_pad(key)), mode=AES.MODE_CBC, IV=iv) # encrypte the plaintext after padding it - ciphertext = cipher.encrypt(_pad(plaintext)) + ciphertext = cipher.encrypt(ensure_bytes(old_pad(plaintext))) # append the hexed IV and the hexed ciphertext iv_plus_encrypted = binascii.hexlify(iv) + binascii.hexlify(ciphertext) # return that return iv_plus_encrypted -def _decode_aes(key, iv_plus_encrypted): +def old_decode_aes(key, iv_plus_encrypted): """ Utility method to decode a payload consisting of the hexed IV + the hexed ciphertext using the given key. See above for more details. @@ -82,18 +173,27 @@ def _decode_aes(key, iv_plus_encrypted): iv = binascii.unhexlify(hexed_iv) ciphertext = binascii.unhexlify(hexed_ciphertext) # set up the correct AES cipher object - cipher = AES.new(_pad(key), mode=AES.MODE_CBC, IV=iv) + cipher = AES.new(ensure_bytes(old_pad(key)), mode=AES.MODE_CBC, IV=iv) # decrypt! plaintext = cipher.decrypt(ciphertext) # return the unpadded version of this - return _unpad(plaintext) + return old_unpad(plaintext) def encrypt(api_key, options): options_string = json.dumps(options) - return _encode_aes(api_key, options_string) + if len(api_key) == 64: + return encode_aes256(api_key, options_string) + else: + return old_encode_aes(api_key, options_string) def decrypt(api_key, scoped_key): - json_string = _decode_aes(api_key, scoped_key) - return json.loads(json_string) \ No newline at end of file + if len(api_key) == 64: + json_string = decode_aes256(api_key, scoped_key) + else: + json_string = old_decode_aes(api_key, scoped_key) + try: + return json.loads(json_string) + except TypeError: + return json.loads(json_string.decode()) diff --git a/keen/tests/access_key_tests.py b/keen/tests/access_key_tests.py new file mode 100644 index 0000000..027f95c --- /dev/null +++ b/keen/tests/access_key_tests.py @@ -0,0 +1,133 @@ + +from keen.tests.base_test_case import BaseTestCase +from keen.tests.client_tests import MockedResponse +from mock import patch + +import keen + +__author__ = 'BlackVegetable' + +class AccessKeyTests(BaseTestCase): + + ACCESS_KEY_NAME = "Bob_Key" + ACCESS_KEY_ID = "320104AEFFC569EEE60BCAC9BB064DFF9897E391AB8C59608AC0869AFD291B4E" + ACCESS_KEY_RESPONSE = MockedResponse( + status_code=201, + json_response={'name': "Bob_Key", + 'is_active': True, + 'permitted': [], + 'key': '320104AEFFC569EEE60BCAC9BB064DFF9897E391AB8C59608AC0869AFD291B4E', + 'project_id': '55777979e085574e8ad3523c', + 'options': {'saved_queries': None, + 'writes': None, + 'datasets': None, + 'cached_queries': None, + 'queries': None}}) + + UPDATED_ACCESS_KEY_RESPONSE = MockedResponse( + status_code=201, + json_response={'name': "Jim_Key", + 'is_active': False, + 'permitted': ["queries"], + 'key': '320104AEFFC569EEE60BCAC9BB064DFF9897E391AB8C59608AC0869AFD291B4E', + 'project_id': '55777979e085574e8ad3523c', + 'options': {'saved_queries': None, + 'writes': None, + 'datasets': None, + 'cached_queries': None, + 'queries': { + "filters": [{ + "property_name": "customer.id", + "operator": "eq", + "property_value": "asdf12345z" + }]}}}) + + NO_CONTENT_RESPONSE = MockedResponse(status_code=204, json_response="") + + def setUp(self): + super(AccessKeyTests, self).setUp() + keen.project_id = "55777979e085574e8ad3523c" + keen.write_key = "DEADBEEF" + keen.read_key = "BADFEED" + keen.master_key = "BADHORSE" + self.keys_uri_prefix = "https://api.keen.io/3.0/projects/{0}/keys".format(keen.project_id) + + def _assert_proper_permissions(self, method, permission): + self.assertTrue(permission in method.call_args[1]["headers"]["Authorization"]) + + @patch("requests.Session.post") + def test_create_access_key(self, post): + post.return_value = self.ACCESS_KEY_RESPONSE + resp = keen.create_access_key(self.ACCESS_KEY_NAME) + self.assertTrue(self.ACCESS_KEY_NAME in post.call_args[1]["data"]) + self._assert_proper_permissions(post, keen.master_key) + self.assertEqual(resp, self.ACCESS_KEY_RESPONSE.json()) + + @patch("requests.Session.get") + def test_list_access_keys(self, get): + get.return_value = self.ACCESS_KEY_RESPONSE + resp = keen.list_access_keys() + self.assertEqual(self.keys_uri_prefix, get.call_args[0][0]) + self._assert_proper_permissions(get, keen.master_key) + self.assertEqual(resp, self.ACCESS_KEY_RESPONSE.json()) + + @patch("requests.Session.get") + def test_get_access_key(self, get): + get.return_value = self.ACCESS_KEY_RESPONSE + resp = keen.get_access_key(self.ACCESS_KEY_ID) + self.assertEqual("{0}/{1}".format(self.keys_uri_prefix, self.ACCESS_KEY_ID), get.call_args[0][0]) + self._assert_proper_permissions(get, keen.master_key) + self.assertEqual(resp, self.ACCESS_KEY_RESPONSE.json()) + + @patch("requests.Session.post") + def test_revoke_access_key(self, post): + post.return_value = self.NO_CONTENT_RESPONSE + resp = keen.revoke_access_key(self.ACCESS_KEY_ID) + self.assertEqual("{0}/{1}/revoke".format(self.keys_uri_prefix, self.ACCESS_KEY_ID), post.call_args[0][0]) + self._assert_proper_permissions(post, keen.master_key) + self.assertEqual(resp, self.NO_CONTENT_RESPONSE.json()) + + @patch("requests.Session.post") + def test_unrevoke_access_key(self, post): + post.return_value = self.NO_CONTENT_RESPONSE + resp = keen.unrevoke_access_key(self.ACCESS_KEY_ID) + self.assertEqual("{0}/{1}/unrevoke".format(self.keys_uri_prefix, self.ACCESS_KEY_ID), post.call_args[0][0]) + self._assert_proper_permissions(post, keen.master_key) + self.assertEqual(resp, self.NO_CONTENT_RESPONSE.json()) + + @patch("requests.Session.delete") + def test_delete_access_key(self, delete): + delete.return_value = self.NO_CONTENT_RESPONSE + resp = keen.delete_access_key(self.ACCESS_KEY_ID) + self.assertEqual("{0}/{1}".format(self.keys_uri_prefix, self.ACCESS_KEY_ID), delete.call_args[0][0]) + self._assert_proper_permissions(delete, keen.master_key) + self.assertEqual(resp, True) + + @patch("requests.Session.post") + def test_update_access_key_full(self, post): + # The update tests have a significant amount of logic that will not be tested via blackbox testing without + # un-mocking Keen's API. So this is the only test that will really cover any of them, and not even very + # well. + post.return_value = self.UPDATED_ACCESS_KEY_RESPONSE + options_dict = {"queries": self.UPDATED_ACCESS_KEY_RESPONSE.json_response["options"]["queries"]} + resp = keen.update_access_key_full(self.ACCESS_KEY_ID, + name=self.UPDATED_ACCESS_KEY_RESPONSE.json_response["name"], + is_active=self.UPDATED_ACCESS_KEY_RESPONSE.json_response["is_active"], + permitted=self.UPDATED_ACCESS_KEY_RESPONSE.json_response["permitted"], + options=options_dict) + self.assertEqual("{0}/{1}".format(self.keys_uri_prefix, self.ACCESS_KEY_ID), post.call_args[0][0]) + self._assert_proper_permissions(post, keen.master_key) + self.assertEqual(resp, self.UPDATED_ACCESS_KEY_RESPONSE.json()) + + @patch("requests.Session.get") + @patch("requests.Session.post") + def test_sanity_of_update_functions(self, post, get): + post.return_value = self.UPDATED_ACCESS_KEY_RESPONSE + get.return_value = self.UPDATED_ACCESS_KEY_RESPONSE + # Ensure at the very least that the other access key functions don't crash when run. + keen.update_access_key_name(self.ACCESS_KEY_ID, name="Marzipan") + keen.add_access_key_permissions(self.ACCESS_KEY_ID, ["writes"]) + keen.remove_access_key_permissions(self.ACCESS_KEY_ID, ["writes"]) + keen.update_access_key_permissions(self.ACCESS_KEY_ID, ["writes", "cached_queries"]) + keen.update_access_key_options(self.ACCESS_KEY_ID, options={}) + self.assertTrue(True) # Best test assertion ever. diff --git a/keen/tests/base_test_case.py b/keen/tests/base_test_case.py index 0f861ed..c40f99b 100644 --- a/keen/tests/base_test_case.py +++ b/keen/tests/base_test_case.py @@ -1,5 +1,5 @@ import string -import unittest +import unittest2 as unittest __author__ = 'dkador' diff --git a/keen/tests/cached_datasets_tests.py b/keen/tests/cached_datasets_tests.py new file mode 100644 index 0000000..1e49bf0 --- /dev/null +++ b/keen/tests/cached_datasets_tests.py @@ -0,0 +1,413 @@ +import json + +import responses + +from keen import exceptions +from keen.client import KeenClient +from keen.tests.base_test_case import BaseTestCase + + +class CachedDatasetsTestCase(BaseTestCase): + + def setUp(self): + super(CachedDatasetsTestCase, self).setUp() + self.organization_id = "1234xxxx5678" + self.project_id = "xxxx1234" + self.read_key = "abcd5678read" + self.master_key = "abcd5678master" + self.client = KeenClient( + project_id=self.project_id, + read_key=self.read_key, + master_key=self.master_key + ) + + self.datasets = [ + { + "project_id": self.project_id, + "organization_id": self.organization_id, + "dataset_name": "DATASET_NAME_1", + "display_name": "a first dataset wee", + "query": { + "project_id": self.project_id, + "analysis_type": "count", + "event_collection": "best collection", + "filters": [ + { + "property_name": "request.foo", + "operator": "lt", + "property_value": 300, + } + ], + "timeframe": "this_500_hours", + "timezone": "US/Pacific", + "interval": "hourly", + "group_by": ["exception.name"], + }, + "index_by": ["project.id"], + "last_scheduled_date": "2016-11-04T18:03:38.430Z", + "latest_subtimeframe_available": "2016-11-04T19:00:00.000Z", + "milliseconds_behind": 3600000, + }, + { + "project_id": self.project_id, + "organization_id": self.organization_id, + "dataset_name": "DATASET_NAME_10", + "display_name": "tenth dataset wee", + "query": { + "project_id": self.project_id, + "analysis_type": "count", + "event_collection": "tenth best collection", + "filters": [], + "timeframe": "this_500_days", + "timezone": "UTC", + "interval": "daily", + "group_by": ["analysis_type"], + }, + "index_by": ["project.organization.id"], + "last_scheduled_date": "2016-11-04T19:28:36.639Z", + "latest_subtimeframe_available": "2016-11-05T00:00:00.000Z", + "milliseconds_behind": 3600000, + }, + ] + + def test_get_all_raises_with_no_keys(self): + client = KeenClient(project_id=self.project_id) + + with self.assertRaises(exceptions.InvalidEnvironmentError): + client.cached_datasets.all() + + @responses.activate + def test_get_all(self): + keen_response = { + "datasets": self.datasets, + "next_page_url": ( + "https://api.keen.io/3.0/projects/{0}/datasets?" + "limit=LIMIT&after_name={1}" + ).format(self.project_id, self.datasets[-1]['dataset_name']) + } + + url = "{0}/{1}/projects/{2}/datasets".format( + self.client.api.base_url, + self.client.api.api_version, + self.project_id + ) + + responses.add( + responses.GET, url, status=200, json=keen_response + ) + + all_cached_datasets = self.client.cached_datasets.all() + + self.assertEquals(all_cached_datasets, keen_response) + + def test_get_one_raises_with_no_keys(self): + client = KeenClient(project_id=self.project_id) + + with self.assertRaises(exceptions.InvalidEnvironmentError): + client.cached_datasets.get() + + @responses.activate + def test_get_one(self): + keen_response = self.datasets[0] + + url = "{0}/{1}/projects/{2}/datasets/{3}".format( + self.client.api.base_url, + self.client.api.api_version, + self.project_id, + self.datasets[0]['dataset_name'] + ) + + def test_create_raises_with_no_keys(self): + client = KeenClient(project_id=self.project_id) + + with self.assertRaises(exceptions.InvalidEnvironmentError): + client.cached_datasets.create( + "NEW_DATASET", {}, "product.id", "My new dataset" + ) + + def test_create_raises_with_read_key(self): + client = KeenClient(project_id=self.project_id, read_key=self.read_key) + + with self.assertRaises(exceptions.InvalidEnvironmentError): + client.cached_datasets.create( + "NEW_DATASET", {}, "product.id", "My new dataset" + ) + + @responses.activate + def test_create(self): + dataset_name = "NEW_DATASET" + display_name = "My new dataset" + query = { + "project_id": "PROJECT ID", + "analysis_type": "count", + "event_collection": "purchases", + "filters": [ + { + "property_name": "price", + "operator": "gte", + "property_value": 100 + } + ], + "timeframe": "this_500_days", + "timezone": None, + "interval": "daily", + "group_by": ["ip_geo_info.country"] + } + index_by = "product.id" + + keen_response = { + "project_id": self.project_id, + "organization_id": self.organization_id, + "dataset_name": dataset_name, + "display_name": display_name, + "query": query, + "index_by": index_by, + "last_scheduled_date": "1970-01-01T00:00:00.000Z", + "latest_subtimeframe_available": "1970-01-01T00:00:00.000Z", + "milliseconds_behind": 3600000 + } + + url = "{0}/{1}/projects/{2}/datasets/{3}".format( + self.client.api.base_url, + self.client.api.api_version, + self.project_id, + dataset_name + ) + + responses.add(responses.PUT, url, status=201, json=keen_response) + + dataset = self.client.cached_datasets.create( + dataset_name, query, index_by, display_name + ) + + self.assertEqual(dataset, keen_response) + + def test_results_raises_with_no_keys(self): + client = KeenClient(project_id=self.project_id) + + with self.assertRaises(exceptions.InvalidEnvironmentError): + client.cached_datasets.results( + "DATASET_ONE", "product.id", "this_100_days" + ) + + @responses.activate + def test_results(self): + keen_response = { + "result": [ + { + "timeframe": { + "start": "2016-11-02T00:00:00.000Z", + "end": "2016-11-02T00:01:00.000Z" + }, + "value": [ + { + "exception.name": "ValueError", + "result": 20 + }, + { + "exception.name": "KeyError", + "result": 18 + } + ] + }, + { + "timeframe": { + "start": "2016-11-02T01:00:00.000Z", + "end": "2016-11-02T02:00:00.000Z" + }, + "value": [ + { + "exception.name": "ValueError", + "result": 1 + }, + { + "exception.name": "KeyError", + "result": 13 + } + ] + } + ] + } + + dataset_name = self.datasets[0]["dataset_name"] + index_by = self.project_id + timeframe = "this_two_hours" + + url = "{0}/{1}/projects/{2}/datasets/{3}/results?index_by={4}&timeframe={5}".format( + self.client.api.base_url, + self.client.api.api_version, + self.project_id, + dataset_name, + index_by, + timeframe + ) + + responses.add( + responses.GET, + url, + status=200, + json=keen_response, + match_querystring=True + ) + + results = self.client.cached_datasets.results( + dataset_name, index_by, timeframe + ) + + self.assertEqual(results, keen_response) + + @responses.activate + def test_results_absolute_timeframe(self): + keen_response = { + "result": [ + { + "timeframe": { + "start": "2016-11-02T00:00:00.000Z", + "end": "2016-11-02T00:01:00.000Z" + }, + "value": [ + { + "exception.name": "ValueError", + "result": 20 + }, + { + "exception.name": "KeyError", + "result": 18 + } + ] + }, + { + "timeframe": { + "start": "2016-11-02T01:00:00.000Z", + "end": "2016-11-02T02:00:00.000Z" + }, + "value": [ + { + "exception.name": "ValueError", + "result": 1 + }, + { + "exception.name": "KeyError", + "result": 13 + } + ] + } + ] + } + + dataset_name = self.datasets[0]["dataset_name"] + index_by = self.project_id + timeframe = { + "start": "2016-11-02T00:00:00.000Z", + "end": "2016-11-02T02:00:00.000Z" + } + + url = "{0}/{1}/projects/{2}/datasets/{3}/results?index_by={4}&timeframe={5}".format( + self.client.api.base_url, + self.client.api.api_version, + self.project_id, + dataset_name, + index_by, + json.dumps(timeframe) + ) + + responses.add( + responses.GET, + url, + status=200, + json=keen_response, + match_querystring=True + ) + + results = self.client.cached_datasets.results( + dataset_name, index_by, timeframe + ) + + self.assertEqual(results, keen_response) + + @responses.activate + def test_results_multiple_index_by(self): + keen_response = { + "result": [ + { + "timeframe": { + "start": "2016-11-02T00:00:00.000Z", + "end": "2016-11-02T00:01:00.000Z" + }, + "value": [ + { + "exception.name": "ValueError", + "result": 20 + }, + { + "exception.name": "KeyError", + "result": 18 + } + ] + }, + { + "timeframe": { + "start": "2016-11-02T01:00:00.000Z", + "end": "2016-11-02T02:00:00.000Z" + }, + "value": [ + { + "exception.name": "ValueError", + "result": 1 + }, + { + "exception.name": "KeyError", + "result": 13 + } + ] + } + ] + } + + dataset_name = self.datasets[0]["dataset_name"] + index_by = [self.project_id, 'another_id'] + timeframe = "this_two_hours" + + url = "{0}/{1}/projects/{2}/datasets/{3}/results?index_by={4}&timeframe={5}".format( + self.client.api.base_url, + self.client.api.api_version, + self.project_id, + dataset_name, + json.dumps(index_by), + timeframe + ) + + responses.add( + responses.GET, + url, + status=200, + json=keen_response, + match_querystring=True + ) + + results = self.client.cached_datasets.results( + dataset_name, index_by, timeframe + ) + + self.assertEqual(results, keen_response) + + def test_delete_raises_with_no_keys(self): + client = KeenClient(project_id=self.project_id) + with self.assertRaises(exceptions.InvalidEnvironmentError): + client.cached_datasets.delete("MY_DATASET_NAME") + + def test_create_raises_with_read_key(self): + client = KeenClient(project_id=self.project_id, read_key=self.read_key) + with self.assertRaises(exceptions.InvalidEnvironmentError): + client.cached_datasets.delete("MY_DATASET_NAME") + + @responses.activate + def test_delete(self): + dataset_name = "MY_DATASET_NAME" + url = "{0}/{1}/projects/{2}/datasets/{3}".format( + self.client.api.base_url, self.client.api.api_version, self.project_id, dataset_name) + responses.add(responses.DELETE, url, status=204) + + response = self.client.cached_datasets.delete(dataset_name) + + self.assertTrue(response) diff --git a/keen/tests/client_tests.py b/keen/tests/client_tests.py index 20a5c4f..a78939c 100644 --- a/keen/tests/client_tests.py +++ b/keen/tests/client_tests.py @@ -6,22 +6,53 @@ import requests from keen import exceptions, persistence_strategies, scoped_keys import keen -from keen.client import KeenClient +from keen.client import KeenClient, Event from keen.tests.base_test_case import BaseTestCase +from mock import patch, MagicMock import sys __author__ = 'dkador' +class MockedResponse(object): + def __init__(self, status_code, json_response, text=None): + self.status_code = status_code + self.json_response = json_response + self.text = text + + def json(self): + return self.json_response + + +class MockedFailedResponse(MockedResponse): + def json(self): + return self.json_response + + +class MockedMalformedJsonResponse(MockedResponse): + def json(self): + raise ValueError + + +@patch("requests.Session.post") class ClientTests(BaseTestCase): + + SINGLE_ADD_RESPONSE = MockedResponse(status_code=201, json_response={"result": {"hello": "goodbye"}}) + + MULTI_ADD_RESPONSE = MockedResponse(status_code=200, json_response={"collection_a": [{"success": True}]}) + MULTI_ADD_RESPONSE_FAILURE = MockedResponse(status_code=200, json_response={"collection_a": [{"success": False, + "error": "message"}]}) + def setUp(self): super(ClientTests, self).setUp() + api_key = "2e79c6ec1d0145be8891bf668599c79a" keen._client = None - keen.project_id = None - keen.write_key = None - keen.read_key = None + keen.project_id = "5004ded1163d66114f000000" + keen.write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}) + keen.read_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["read"]}) + keen.master_key = None - def test_init(self): + def test_init(self, post): def positive_helper(project_id, **kwargs): client = KeenClient(project_id, **kwargs) self.assert_not_equal(client, None) @@ -64,15 +95,13 @@ def negative_helper(expected_exception, project_id, "project_id", persistence_strategy=persistence_strategies.DirectPersistenceStrategy) - def test_direct_persistence_strategy(self): - project_id = "5004ded1163d66114f000000" - api_key = "2e79c6ec1d0145be8891bf668599c79a" - write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}) - read_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["read"]}) - client = KeenClient(project_id, write_key=write_key, read_key=read_key) - client.add_event("python_test", {"hello": "goodbye"}) - client.add_event("python_test", {"hello": "goodbye"}) - client.add_events( + def test_direct_persistence_strategy(self, post): + post.return_value = self.SINGLE_ADD_RESPONSE + keen.add_event("python_test", {"hello": "goodbye"}) + keen.add_event("python_test", {"hello": "goodbye"}) + + post.return_value = self.MULTI_ADD_RESPONSE + response = keen.add_events( { "sign_ups": [{ "username": "timmy", @@ -85,45 +114,71 @@ def test_direct_persistence_strategy(self): {"price": 7} ]} ) + self.assertEqual(self.MULTI_ADD_RESPONSE.json_response, response) - def test_module_level_add_event(self): - keen.project_id = "5004ded1163d66114f000000" - api_key = "2e79c6ec1d0145be8891bf668599c79a" - keen.write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}) - # client = KeenClient(project_id, write_key=write_key, read_key=read_key) + post.return_value = self.MULTI_ADD_RESPONSE_FAILURE + response = keen.add_events( + { + "sign_ups": [{ + "username": "timmy", + "referred_by": "steve", + "son_of": "my_mom" + }], + "purchases": [ + {"price": 5}, + {"price": 6}, + {"price": 7} + ]} + ) + self.assertEqual(self.MULTI_ADD_RESPONSE_FAILURE.json_response, response) + + def test_module_level_add_event(self, post): + post.return_value = self.SINGLE_ADD_RESPONSE keen.add_event("python_test", {"hello": "goodbye"}) - def test_module_level_add_events(self): - keen.project_id = "5004ded1163d66114f000000" - api_key = "2e79c6ec1d0145be8891bf668599c79a" - keen.write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}) - # client = KeenClient(project_id, write_key=write_key, read_key=read_key) + def test_module_level_add_events(self, post): + post.return_value = self.MULTI_ADD_RESPONSE keen.add_events({"python_test": [{"hello": "goodbye"}]}) - @raises(requests.Timeout) - def test_post_timeout_single(self): - keen.project_id = "5004ded1163d66114f000000" - api_key = "2e79c6ec1d0145be8891bf668599c79a" - keen.write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}) - client = KeenClient(keen.project_id, write_key=keen.write_key, read_key=None, - post_timeout=0.0001) - client.add_event("python_test", {"hello": "goodbye"}) + def test_post_timeout_single(self, post): + post.side_effect = requests.Timeout + self.assert_raises(requests.Timeout, keen.add_event, "python_test", {"hello": "goodbye"}) - @raises(requests.Timeout) - def test_post_timeout_batch(self): - keen.project_id = "5004ded1163d66114f000000" - api_key = "2e79c6ec1d0145be8891bf668599c79a" - keen.write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}) - client = KeenClient(keen.project_id, write_key=keen.write_key, read_key=None, - post_timeout=0.0001) - client.add_events({"python_test": [{"hello": "goodbye"}]}) + def test_post_timeout_batch(self, post): + post.side_effect = requests.Timeout + self.assert_raises(requests.Timeout, keen.add_events, {"python_test": [{"hello": "goodbye"}]}) + + def test_malformed_json_response(self, post): + post.return_value = MockedMalformedJsonResponse( + status_code=401, + json_response=" ", + text="test error text" + ) + + exception = None + + try: + keen.add_event("python_test", {"hello": "goodbye"}) + except exceptions.KeenApiError as e: + + exception = e - def test_environment_variables(self): + self.assertTrue(post.return_value.text in str(exception)) + self.assertTrue(str(post.return_value.status_code) in str(exception)) + + def test_environment_variables(self, post): + post.return_value = MockedFailedResponse( + status_code=401, + # "message" is the description, "error_code" is the name of the class. + json_response={"message": "authorization error", "error_code": "AdminOnlyEndpointError"}, + ) # try addEvent w/out having environment variables keen._client = None keen.project_id = None keen.write_key = None keen.read_key = None + keen.master_key = None + keen.base_url = None self.assert_raises(exceptions.InvalidEnvironmentError, keen.add_event, "python_test", {"hello": "goodbye"}) @@ -134,26 +189,108 @@ def test_environment_variables(self): # force client to reinitialize keen._client = None + os.environ["KEEN_PROJECT_ID"] = "12345" os.environ["KEEN_WRITE_KEY"] = "abcde" + self.assert_raises(exceptions.KeenApiError, keen.add_event, "python_test", {"hello": "goodbye"}) - def test_configure_through_code(self): - keen.project_id = "123456" - self.assert_raises(exceptions.InvalidEnvironmentError, - keen.add_event, "python_test", {"hello": "goodbye"}) + def test_new_client_instance(self, post): + exp_project_id = "xxxx1234" + exp_write_key = "yyyy4567" + exp_read_key = "zzzz8912" + exp_master_key = "abcd3456" + exp_base_url = "keen.base.url" + + # create Client instance + client = KeenClient( + project_id=exp_project_id, + write_key=exp_write_key, + read_key=exp_read_key, + master_key=exp_master_key, + base_url=exp_base_url + ) - # force client to reinitialize - keen._client = None - keen.write_key = "abcdef" - self.assert_raises(exceptions.KeenApiError, - keen.add_event, "python_test", {"hello": "goodbye"}) + # assert values + self.assertEquals(exp_project_id, client.api.project_id) + self.assertEquals(exp_write_key, client.api.write_key) + self.assertEquals(exp_read_key, client.api.read_key) + self.assertEquals(exp_master_key, client.api.master_key) + self.assertEquals(exp_base_url, client.api.base_url) - def test_generate_image_beacon(self): - keen.project_id = "5004ded1163d66114f000000" - api_key = "2e79c6ec1d0145be8891bf668599c79a" - keen.write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}) + def test_set_keys_using_env_var(self, post): + # reset Client settings + keen._client = None + keen.project_id = None + keen.write_key = None + keen.read_key = None + keen.master_key = None + keen.base_url = None + + # set env vars + exp_project_id = os.environ["KEEN_PROJECT_ID"] = "xxxx5678" + exp_write_key = os.environ["KEEN_WRITE_KEY"] = "yyyy8901" + exp_read_key = os.environ["KEEN_READ_KEY"] = "zzzz2345" + exp_master_key = os.environ["KEEN_MASTER_KEY"] = "abcd1234" + exp_base_url = os.environ["KEEN_BASE_URL"] = "keen.base.url" + + keen._initialize_client_from_environment() + + # test values + self.assertEquals(exp_project_id, keen.project_id) + self.assertEquals(exp_write_key, keen.write_key) + self.assertEquals(exp_read_key, keen.read_key) + self.assertEquals(exp_master_key, keen.master_key) + self.assertEquals(exp_base_url, keen.base_url) + self.assertEquals(exp_project_id, keen._client.api.project_id) + self.assertEquals(exp_write_key, keen._client.api.write_key) + self.assertEquals(exp_read_key, keen._client.api.read_key) + self.assertEquals(exp_master_key, keen._client.api.master_key) + self.assertEquals(exp_base_url, keen._client.api.base_url) + + # remove env vars + del os.environ["KEEN_PROJECT_ID"] + del os.environ["KEEN_WRITE_KEY"] + del os.environ["KEEN_READ_KEY"] + del os.environ["KEEN_MASTER_KEY"] + del os.environ["KEEN_BASE_URL"] + + def test_set_keys_using_package_var(self, post): + exp_project_id = keen.project_id = "uuuu5678" + exp_write_key = keen.write_key = "vvvv8901" + exp_read_key = keen.read_key = "wwwww2345" + exp_master_key = keen.master_key = "abcd4567" + exp_base_url = keen.base_url = "keen.base.url" + + keen._initialize_client_from_environment() + + # test values + self.assertEquals(exp_project_id, keen.project_id) + self.assertEquals(exp_write_key, keen.write_key) + self.assertEquals(exp_read_key, keen.read_key) + self.assertEquals(exp_master_key, keen.master_key) + self.assertEquals(exp_project_id, keen._client.api.project_id) + self.assertEquals(exp_write_key, keen._client.api.write_key) + self.assertEquals(exp_read_key, keen._client.api.read_key) + self.assertEquals(exp_master_key, keen._client.api.master_key) + self.assertEquals(exp_base_url, keen._client.api.base_url) + + def test_configure_through_code(self, post): + client = KeenClient(project_id="123456", read_key=None, write_key=None) + self.assert_raises(exceptions.InvalidEnvironmentError, + client.add_event, "python_test", {"hello": "goodbye"}) + # force client to reinitialize + client = KeenClient(project_id="123456", read_key=None, write_key="abcdef") + with patch("requests.Session.post") as post: + post.return_value = MockedFailedResponse( + status_code=401, + json_response={"message": "authorization error", "error_code": "AdminOnlyEndpointError"}, + ) + self.assert_raises(exceptions.KeenApiError, + client.add_event, "python_test", {"hello": "goodbye"}) + + def test_generate_image_beacon(self, post): event_collection = "python_test hello!?" event_data = {"a": "b"} data = self.base64_encode(json.dumps(event_data)) @@ -170,19 +307,9 @@ def test_generate_image_beacon(self): url = client.generate_image_beacon(event_collection, event_data) self.assert_equal(expected, url) - # make sure URL works - response = requests.get(url) - self.assert_equal(200, response.status_code) - self.assert_equal(b"GIF89a\x01\x00\x01\x00\x80\x01\x00\xff\xff\xff\x00\x00\x00!\xf9\x04\x01\n\x00\x01\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02L\x01\x00;", - response.content) - - def test_generate_image_beacon_timestamp(self): + def test_generate_image_beacon_timestamp(self, post): # make sure using a timestamp works - keen.project_id = "5004ded1163d66114f000000" - api_key = "2e79c6ec1d0145be8891bf668599c79a" - keen.write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}) - event_collection = "python_test" event_data = {"a": "b"} timestamp = datetime.datetime.utcnow() @@ -214,8 +341,32 @@ def url_escape(self, url): return urllib.parse.quote(url) +class EventTests(BaseTestCase): + + def test_custom_addon(self): + event = Event( + '', + '', + {'keen': {'addons': {'asdf': 1}}}, + timestamp=datetime.datetime.now(), + ) + as_json = json.loads(event.to_json()) + + self.assertEqual(as_json['keen']['addons']['asdf'], 1) + self.assertTrue('timestamp' in as_json['keen']) + +@patch("requests.Session.get") class QueryTests(BaseTestCase): + + INT_RESPONSE = MockedResponse(status_code=200, json_response={"result": 2}) + + LIST_RESPONSE = MockedResponse( + status_code=200, json_response={"result": [{"value": {"total": 1}}, {"value": {"total": 2}}]}) + + LIST_RESPONSE_DESCENDING = MockedResponse( + status_code=200, json_response={"result": [{"value": {"total": 2}}, {"value": {"total": 1}}]}) + def setUp(self): super(QueryTests, self).setUp() keen._client = None @@ -223,58 +374,75 @@ def setUp(self): api_key = "2e79c6ec1d0145be8891bf668599c79a" keen.write_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}) keen.read_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["read"]}) - keen.add_event("query test", {"number": 5, "string": "foo"}) - keen.add_event("step2", {"number": 5, "string": "foo"}) + # keen.add_event("query test", {"number": 5, "string": "foo"}) + # keen.add_event("step2", {"number": 5, "string": "foo"}) def tearDown(self): keen.project_id = None keen.write_key = None keen.read_key = None + keen.master_key = None + keen.base_url = None keen._client = None super(QueryTests, self).tearDown() def get_filter(self): return [{"property_name": "number", "operator": "eq", "property_value": 5}] - def test_count(self): + def test_count(self, get): + get.return_value = self.INT_RESPONSE resp = keen.count("query test", timeframe="today", filters=self.get_filter()) self.assertEqual(type(resp), int) - def test_sum(self): + def test_sum(self, get): + get.return_value = self.INT_RESPONSE resp = keen.sum("query test", target_property="number", timeframe="today") self.assertEqual(type(resp), int) - def test_minimum(self): + def test_minimum(self, get): + get.return_value = self.INT_RESPONSE resp = keen.minimum("query test", target_property="number", timeframe="today") self.assertEqual(type(resp), int) - def test_maximum(self): + def test_maximum(self, get): + get.return_value = self.INT_RESPONSE resp = keen.maximum("query test", target_property="number", timeframe="today") self.assertEqual(type(resp), int) - def test_average(self): + def test_average(self, get): + get.return_value = self.INT_RESPONSE resp = keen.average("query test", target_property="number", timeframe="today") self.assertTrue(type(resp) in (int, float), type(resp)) - def test_percentile(self): + def test_median(self, get): + get.return_value = self.INT_RESPONSE + resp = keen.median("query test", target_property="number", timeframe="today") + self.assertTrue(type(resp) in (int, float), type(resp)) + + def test_percentile(self, get): + get.return_value = self.INT_RESPONSE resp = keen.percentile("query test", target_property="number", percentile=80, timeframe="today") self.assertTrue(type(resp) in (int, float), type(resp)) - def test_count_unique(self): + def test_count_unique(self, get): + get.return_value = self.INT_RESPONSE resp = keen.count_unique("query test", target_property="number", timeframe="today") self.assertEqual(type(resp), int) - def test_select_unique(self): + def test_select_unique(self, get): + get.return_value = self.LIST_RESPONSE resp = keen.select_unique("query test", target_property="number", timeframe="today") self.assertEqual(type(resp), list) - def test_extraction(self): - resp = keen.extraction("query test", timeframe="today", property_names=["number"]) + def test_extraction(self, get): + get.return_value = self.LIST_RESPONSE + resp = keen.extraction("query test", timeframe="today", property_names=["number"]) self.assertEqual(type(resp), list) for event in resp: self.assertTrue("string" not in event) - def test_multi_analysis(self): + def test_multi_analysis(self, get): + get.return_value = self.LIST_RESPONSE resp = keen.multi_analysis("query test", analyses={"total": {"analysis_type": "sum", "target_property": "number"}}, timeframe="today", interval="hourly") @@ -282,37 +450,101 @@ def test_multi_analysis(self): for result in resp: self.assertEqual(type(result["value"]["total"]), int) - def test_funnel(self): + def test_funnel(self, get): + get.return_value = self.LIST_RESPONSE + step1 = { - "event_collection": "query test", - "actor_property": "number", + "event_collection": "signed up", + "actor_property": "visitor.guid", "timeframe": "today" } step2 = { - "event_collection": "step2", - "actor_property": "number", + "event_collection": "completed profile", + "actor_property": "user.guid", "timeframe": "today" } + resp = keen.funnel([step1, step2]) self.assertEqual(type(resp), list) - def test_group_by(self): + def test_funnel_return_all_keys(self, get): + get.return_value = MockedResponse(status_code=200, json_response={ + "result": [], + "actors": [], + "random_key": [], + "steps": [] + }) + + resp = keen.funnel([1, 2], all_keys=True) + self.assertEquals(type(resp), dict) + self.assertTrue("actors" in resp) + self.assertTrue("random_key" in resp) + + def test_group_by(self, get): + get.return_value = self.LIST_RESPONSE resp = keen.count("query test", timeframe="today", group_by="number") self.assertEqual(type(resp), list) - def test_multi_group_by(self): + def test_multi_group_by(self, get): + get.return_value = self.LIST_RESPONSE resp = keen.count("query test", timeframe="today", group_by=["number", "string"]) self.assertEqual(type(resp), list) - def test_interval(self): + def test_interval(self, get): + get.return_value = self.LIST_RESPONSE resp = keen.count("query test", timeframe="this_2_days", interval="daily") self.assertEqual(type(resp), list) - def test_passing_custom_api_client(self): + def test_order_by(self, get): + get.return_value = self.LIST_RESPONSE_DESCENDING + collection = "query_test" + limit = 2 + order_by = {"property_name": "result", "direction": keen.direction.DESCENDING} + resp = keen.count(collection, timeframe="today", group_by="number", order_by=order_by, limit=limit) + self.assertTrue("https://api.keen.io/3.0/projects/{0}/queries/count".format(keen.project_id) in + get.call_args[0][0]) + self.assertEqual(2, get.call_args[1]["params"]["limit"]) + self.assertEqual(collection, get.call_args[1]["params"]["event_collection"]) + # Order by should always be wrapped in a list, and stringified. + # This test is broken because JSON. It thinks double quoted strings != single quoted strings. + # ... annoying, to say the least. + # self.assertTrue(str([order_by]), get.call_args[1]["params"]["order_by"]) + # So instead let's use this only slightly useful test: + self.assertTrue("order_by" in get.call_args[1]["params"]) + self.assertTrue(keen.read_key in get.call_args[1]["headers"]["Authorization"]) + self.assertEqual(resp, self.LIST_RESPONSE_DESCENDING.json()["result"]) + + def test_order_by_invalid_limit(self, get): + collection = "query_test" + limit = -1 # Limit should be positive + order_by = {"property_name": "result", "direction": keen.direction.DESCENDING} + self.assertRaises(ValueError, keen.count, collection, timeframe="today", group_by="number", order_by=order_by, + limit=limit) + + def test_order_by_invalid_direction(self, get): + collection = "query_test" + limit = 2 + order_by = {"property_name": "result", "direction": "INVALID"} + self.assertRaises(ValueError, keen.count, collection, timeframe="today", group_by="number", order_by=order_by, + limit=limit) + + def test_order_by_no_group_by(self, get): + collection = "query_test" + limit = 2 + order_by = {"property_name": "result", "direction": keen.direction.DESCENDING} + self.assertRaises(ValueError, keen.count, collection, timeframe="today", order_by=order_by, limit=limit) + + def test_limit_no_order_by(self, get): + collection = "query_test" + limit = 2 + self.assertRaises(ValueError, keen.count, collection, timeframe="today", group_by="number", limit=limit) + + def test_passing_invalid_custom_api_client(self, get): class CustomApiClient(object): - def __init__(self, project_id, - write_key=None, read_key=None, - base_url=None, api_version=None, **kwargs): + api_version = "3.0" + + def __init__(self, project_id, write_key=None, read_key=None, + base_url=None, api_version=None, **kwargs): super(CustomApiClient, self).__init__() self.project_id = project_id self.write_key = write_key @@ -323,22 +555,90 @@ def __init__(self, project_id, self.api_version = api_version api_key = "2e79c6ec1d0145be8891bf668599c79a" - client = KeenClient("5004ded1163d66114f000000", write_key=scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}), read_key=scoped_keys.encrypt(api_key, {"allowed_operations": ["read"]}), api_class=CustomApiClient) + client = KeenClient("5004ded1163d66114f000000", + write_key=scoped_keys.encrypt(api_key, {"allowed_operations": ["write"]}), + read_key=scoped_keys.encrypt(api_key, {"allowed_operations": ["read"]}), + api_class=CustomApiClient, + base_url="keen.base.url") # Should raise an error, we never added this method on our class # But it shows it is actually using our class self.assertRaises(TypeError, client.add_event) - @raises(requests.Timeout) - def test_timeout_count(self): - keen.project_id = "5004ded1163d66114f000000" - api_key = "2e79c6ec1d0145be8891bf668599c79a" - keen.read_key = scoped_keys.encrypt(api_key, {"allowed_operations": ["read"]}) + def test_timeout_count(self, get): + get.side_effect = requests.Timeout client = KeenClient(keen.project_id, write_key=None, read_key=keen.read_key, get_timeout=0.0001) - resp = client.count("query test", timeframe="today", filters=self.get_filter()) + self.assert_raises(requests.Timeout, client.count, "query test", timeframe="today", filters=self.get_filter()) + # Make sure the requests library was called with `timeout`. + self.assert_equals(get.call_args[1]["timeout"], 0.0001) + + +@patch("requests.Session.delete") +class DeleteTests(BaseTestCase): + + def setUp(self): + super(DeleteTests, self).setUp() + keen._client = None + keen.project_id = "1k4jb23kjbkjkjsd" + keen.master_key = "sdofnasofagaergub" + keen.base_url = None + + def tearDown(self): + keen._client = None + keen.project_id = None + keen.master_key = None + super(DeleteTests, self).tearDown() + + def test_delete_events(self, delete): + delete.return_value = MockedResponse(status_code=204, json_response=[]) + # Assert that the mocked delete function is called the way we expect. + keen.delete_events("foo", filters=[{"property_name": 'username', "operator": 'eq', "property_value": 'Bob'}]) + # Check that the URL is generated correctly. + self.assertEqual("https://api.keen.io/3.0/projects/1k4jb23kjbkjkjsd/events/foo", delete.call_args[0][0]) + # Check that the master_key is in the Authorization header. + self.assertTrue(keen.master_key in delete.call_args[1]["headers"]["Authorization"]) + +@patch("requests.Session.get") +class GetTests(BaseTestCase): + + SINGLE_ADD_RESPONSE = MockedResponse(status_code=201, json_response={"result": {"hello": "goodbye"}}) + + LIST_RESPONSE = MockedResponse( + status_code=200, json_response={"result": [{"value": {"total": 1}}, {"value": {"total": 2}}]}) + + def setUp(self): + super(GetTests, self).setUp() + keen._client = None + keen.project_id = "1k4jb23kjbkjkjsd" + keen.read_key = "sdofnasofagaergub" + keen.base_url = None + + def tearDown(self): + keen._client = None + keen.project_id = None + keen.read_key = None + super(GetTests, self).tearDown() + + def test_get_collection(self, get): + get.return_value = self.SINGLE_ADD_RESPONSE + # Assert that the mocked get function is called the way we expect. + keen.get_collection("foo") + # Check that the URL is generated correctly + self.assertEqual("https://api.keen.io/3.0/projects/1k4jb23kjbkjkjsd/events/foo", get.call_args[0][0]) + # Check that the master_key is in the Authorization header + self.assertTrue(keen.read_key in get.call_args[1]["headers"]["Authorization"]) + + def test_get_all_collections(self, get): + get.return_value = self.LIST_RESPONSE + # Assert that the mocked get function is called the way we expect + keen.get_all_collections() + # Check that the URL is generated correctly + self.assertEqual("https://api.keen.io/3.0/projects/1k4jb23kjbkjkjsd/events", get.call_args[0][0]) + # Check that the master_key in the Authorization header + self.assertTrue(keen.read_key in get.call_args[1]["headers"]["Authorization"]) # only need to test unicode separately in python2 -if sys.version_info[0] > 3: +if sys.version_info[0] < 3: class UnicodeTests(BaseTestCase): def setUp(self): @@ -348,12 +648,14 @@ def setUp(self): api_key = unicode("2e79c6ec1d0145be8891bf668599c79a") keen.write_key = unicode(api_key) - def test_unicode(self): + @patch("requests.Session.post", MagicMock(return_value=MockedResponse(status_code=201, json_response=[0, 1, 2]))) + def test_add_event_with_unicode(self): keen.add_event(unicode("unicode test"), {unicode("number"): 5, "string": unicode("foo")}) def tearDown(self): keen.project_id = None keen.write_key = None keen.read_key = None + keen.master_key = None keen._client = None super(UnicodeTests, self).tearDown() diff --git a/keen/tests/saved_query_tests.py b/keen/tests/saved_query_tests.py new file mode 100644 index 0000000..b90c9b3 --- /dev/null +++ b/keen/tests/saved_query_tests.py @@ -0,0 +1,240 @@ +import json +import re +from requests.exceptions import HTTPError +import responses + +from keen.tests.base_test_case import BaseTestCase +from keen.client import KeenClient +from keen import exceptions + + +class SavedQueryTests(BaseTestCase): + + def setUp(self): + super(SavedQueryTests, self).setUp() + self.exp_project_id = "xxxx1234" + exp_master_key = "abcd3456" + self.client = KeenClient( + project_id=self.exp_project_id, + read_key="efgh7890", + master_key=exp_master_key + ) + + def test_get_all_saved_queries_keys(self): + client = KeenClient(project_id="123123") + self.assertRaises( + exceptions.InvalidEnvironmentError, client.saved_queries.all + ) + + @responses.activate + def test_get_all_saved_queries(self): + saved_queries_response = [ + { "query_name": "first-saved-query", "query": {} }, + { "query_name": "second-saved-query", "query": {} } + ] + url = "{0}/{1}/projects/{2}/queries/saved".format( + self.client.api.base_url, + self.client.api.api_version, + self.exp_project_id + ) + responses.add( + responses.GET, url, status=200, json=saved_queries_response + ) + + all_saved_queries = self.client.saved_queries.all() + + self.assertEquals(all_saved_queries, saved_queries_response) + + def test_get_one_saved_query_keys(self): + client = KeenClient(project_id="123123") + self.assertRaises( + exceptions.InvalidEnvironmentError, + lambda: client.saved_queries.get("saved-query-name") + ) + + @responses.activate + def test_get_one_saved_query(self): + saved_queries_response = { + "query_name": "saved-query-name" + } + url = "{0}/{1}/projects/{2}/queries/saved/saved-query-name".format( + self.client.api.base_url, + self.client.api.api_version, + self.exp_project_id + ) + responses.add( + responses.GET, url, status=200, json=saved_queries_response + ) + + saved_query = self.client.saved_queries.get("saved-query-name") + + self.assertEquals(saved_query, saved_queries_response) + + def test_create_saved_query_master_key(self): + client = KeenClient(project_id="123123") + self.assertRaises( + exceptions.InvalidEnvironmentError, + lambda: client.saved_queries.create("saved-query-name", {}) + ) + + @responses.activate + def test_create_saved_query(self): + saved_queries_response = { + "query_name": "saved-query-name" + } + url = "{0}/{1}/projects/{2}/queries/saved/saved-query-name".format( + self.client.api.base_url, + self.client.api.api_version, + self.exp_project_id + ) + responses.add( + responses.PUT, url, status=201, json=saved_queries_response + ) + + saved_query = self.client.saved_queries.create("saved-query-name", saved_queries_response) + + self.assertEqual(saved_query, saved_queries_response) + + @responses.activate + def test_update_saved_query(self): + unacceptable_attr = "run_information" + metadata_attr_name = "metadata" + + original_query = { + "query_name": "saved-query-name", + "refresh_rate": 14400, + "query": { + "analysis_type": "average", + "event_collection": "TheCollection", + "target_property": "TheProperty", + "timeframe": "this_2_weeks" + }, + metadata_attr_name: { "foo": "bar" }, + unacceptable_attr: { "foo": "bar" } + } + + url = "{0}/{1}/projects/{2}/queries/saved/saved-query-name".format( + self.client.api.base_url, + self.client.api.api_version, + self.exp_project_id + ) + + responses.add( + responses.GET, url, status=200, json=original_query + ) + + updated_query = { "query": {} } + new_analysis_type = "sum" + updated_query["query"]["analysis_type"] = new_analysis_type + + def request_callback(request): + payload = json.loads(request.body) + + # Ensure update() round-trips some necessary things like "metadata" + self.assertEqual(payload[metadata_attr_name], original_query[metadata_attr_name]) + + # Ensure update() doesn't pass unacceptable attributes + self.assertNotIn(unacceptable_attr, payload) + + # Ensure update() merges deep updates + self.assertEqual(payload["query"]["analysis_type"], new_analysis_type) + payload["query"]["analysis_type"] = "average" + payload[unacceptable_attr] = original_query[unacceptable_attr] + self.assertEqual(payload, original_query) + + headers = {} + return (200, headers, json.dumps(updated_query)) + + responses.add_callback( + responses.PUT, + url, + callback=request_callback, + content_type='application/json', + ) + + saved_query = self.client.saved_queries.update("saved-query-name", updated_query) + + self.assertEqual(saved_query, updated_query) + + @responses.activate + def test_update_full_saved_query(self): + saved_queries_response = { + "query_name": "saved-query-name", + "refresh_rate": 14400, + "query": { + "analysis_type": "average", + "event_collection": "TheCollection", + "target_property": "TheProperty", + "timeframe": "this_2_weeks" + } + } + + url = "{0}/{1}/projects/{2}/queries/saved/saved-query-name".format( + self.client.api.base_url, + self.client.api.api_version, + self.exp_project_id + ) + + # Unlike update(), update_full() should not be fetching the existing definition. + exception = HTTPError("No GET expected when performing a full update.") + responses.add(responses.GET, re.compile(".*"), body=exception) + + def request_callback(request): + payload = json.loads(request.body) + + # Ensure update_full() passes along the unaltered complete Saved/Cached Query def. + self.assertEqual(payload, saved_queries_response) + headers = {} + return (200, headers, json.dumps(saved_queries_response)) + + responses.add_callback( + responses.PUT, + url, + callback=request_callback, + content_type='application/json', + ) + + saved_query = self.client.saved_queries.update_full("saved-query-name", saved_queries_response) + + self.assertEqual(saved_query, saved_queries_response) + + def test_delete_saved_query_master_key(self): + client = KeenClient(project_id="123123", read_key="123123") + self.assertRaises( + exceptions.InvalidEnvironmentError, + lambda: client.saved_queries.delete("saved-query-name") + ) + + @responses.activate + def test_delete_saved_query(self): + url = "{0}/{1}/projects/{2}/queries/saved/saved-query-name".format( + self.client.api.base_url, + self.client.api.api_version, + self.exp_project_id + ) + responses.add( + responses.DELETE, url, status=204, json="" + ) + + response = self.client.saved_queries.delete("saved-query-name") + + self.assertEquals(response, True) + + @responses.activate + def test_saved_query_results(self): + saved_queries_response = { + "query_name": "saved-query-name", + "results": {} + } + url = "{0}/{1}/projects/{2}/queries/saved/saved-query-name/result".format( + self.client.api.base_url, + self.client.api.api_version, + self.exp_project_id + ) + responses.add( + responses.GET, url, status=209, json=saved_queries_response + ) + + response = self.client.saved_queries.results("saved-query-name") + + self.assertEquals(response, saved_queries_response) diff --git a/keen/tests/scoped_key_tests.py b/keen/tests/scoped_key_tests.py new file mode 100644 index 0000000..ff65c31 --- /dev/null +++ b/keen/tests/scoped_key_tests.py @@ -0,0 +1,42 @@ +from keen import scoped_keys +from keen.tests.base_test_case import BaseTestCase + + +class ScopedKeyTests(BaseTestCase): + api_key = "24077ACBCB198BAAA2110EDDB673282F8E34909FD823A15C55A6253A664BE368" + bad_api_key = "24077ACBCB198BAAA2110EDDB673282F8E34909FD823A15C55A6253A664BE369" + old_api_key = "ab428324dbdbcfe744" + old_bad_api_key = "badbadbadbad" + options = { + "filters": [{ + "property_name": "accountId", + "operator": "eq", + "property_value": "123456" + }] + } + + def test_scoped_key_encrypts_and_decrypts(self): + encrypted = scoped_keys.encrypt(self.api_key, self.options) + decrypted = scoped_keys.decrypt(self.api_key, encrypted) + self.assert_equal(decrypted, self.options) + + def test_scoped_key_fails_decryption_bad_key(self): + encrypted = scoped_keys.encrypt(self.api_key, self.options) + try: + scoped_keys.decrypt(self.bad_api_key, encrypted) + self.fail("shouldn't get here") + except ValueError as e: + self.assert_not_equal(e, None) + + def test_old_scoped_key_encrypts_and_decrypts(self): + encrypted = scoped_keys.encrypt(self.old_api_key, self.options) + decrypted = scoped_keys.decrypt(self.old_api_key, encrypted) + self.assert_equal(decrypted, self.options) + + def test_old_scoped_key_fails_decryption_on_bad_key(self): + encrypted = scoped_keys.encrypt(self.old_api_key, self.options) + try: + scoped_keys.decrypt(self.old_bad_api_key, encrypted) + self.fail("shouldn't get here") + except ValueError as e: + self.assert_not_equal(e, None) diff --git a/keen/utilities.py b/keen/utilities.py new file mode 100644 index 0000000..6d8ab14 --- /dev/null +++ b/keen/utilities.py @@ -0,0 +1,94 @@ + +from functools import wraps + +# keen +from keen import exceptions + + +VERSION = "0.7.0" + +def version(): + """ + Retrieves the current version of the SDK + """ + return VERSION + + +def headers(api_key): + """ + Helper function to easily get the correct headers for an endpoint + + :params api_key: The appropriate API key for the request being made + """ + + return { + "Content-Type": "application/json", + "Authorization": api_key, + "Keen-Sdk": "python-{0}".format(version()) + } + + +class KeenKeys(object): + """ Keen API key types. """ + + READ = 'read' + WRITE = 'write' + MASTER = 'master' + + +class switch(object): + """ Python switch recipe. """ + + def __init__(self, value): + self.value = value + self.fall = False + + def __iter__(self): + yield self.match + raise StopIteration + + def match(self, *args): + """Whether or not to enter a given case statement""" + + self.fall = self.fall or not args + self.fall = self.fall or (self.value in args) + + return self.fall + + +def _throw_key_missing(key, relying_on_master): + message = ("The Keen IO API requires a {0} key to perform queries. " + "Please set a '{0}_key' when initializing the " + "KeenApi object.") + + if relying_on_master: + message += ' The "master_key" is set, but one should prefer the key with least privilege.' + + raise exceptions.InvalidEnvironmentError(message.format(key)) + + +def requires_key(key_type): + def requires_key_decorator(func): + + @wraps(func) + def method_wrapper(self, *args, **kwargs): + for case in switch(key_type): + if case(KeenKeys.READ): + if not self._get_read_key(): + _throw_key_missing(KeenKeys.READ, bool(self._get_master_key())) + break + + if case(KeenKeys.WRITE): + if not self._get_write_key(): + _throw_key_missing(KeenKeys.WRITE, bool(self._get_master_key())) + break + + if case(KeenKeys.MASTER): + if not self._get_master_key(): + _throw_key_missing(KeenKeys.MASTER, False) + break + + return func(self, *args, **kwargs) + + return method_wrapper + return requires_key_decorator diff --git a/requirements.txt b/requirements.txt index c786cbc..e087a6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -Padding==0.4 -nose==1.1.2 -pycrypto==2.6 -requests==2.2.1 +pycryptodome>=3.4 +requests>=2.5,<3.0 +six~=1.10 diff --git a/setup.py b/setup.py index 299a905..41a3be0 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,53 @@ #!/usr/bin/env python -from distutils.core import setup - -setup(name="keen", - version="0.3.5", - description="Python Client for Keen IO", - author="Keen IO", - author_email="team@keen.io", - url="https://github.com/keenlabs/KeenClient-Python", - packages=["keen"], - install_requires=["requests", "pycrypto", "Padding"], - tests_require=["nose"] +from setuptools import setup +import os, sys, codecs + +try: + # nose uses multiprocessing if available. + # but setup.py atexit fails if it's loaded too late. + # Traceback (most recent call last): + # File "...python2.6/atexit.py", line 24, in _run_exitfuncs + # func(*targs, **kargs) + # File "...python2.6/multiprocessing/util.py", line 258, in _exit_function + # info('process shutting down') + # TypeError: 'NoneType' object is not callable + import multiprocessing # NOQA +except ImportError: + pass + +setup_path = os.path.dirname(__file__) +reqs_file = open(os.path.join(setup_path, 'requirements.txt'), 'r') +reqs = reqs_file.readlines() +reqs_file.close() + +tests_require = ['nose', 'mock', 'responses==0.5.1', 'unittest2'] + +setup( + name="keen", + version="0.7.0", + description="Python Client for Keen IO", + long_description=codecs.open(os.path.join('README.rst'), 'r', encoding='UTF-8').read(), + author="Keen IO", + author_email="team@keen.io", + url="https://github.com/keenlabs/KeenClient-Python", + packages=["keen"], + install_requires=reqs, + tests_require=tests_require, + test_suite='nose.collector', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Topic :: Software Development :: Libraries :: Python Modules', + ] ) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..a96aa14 --- /dev/null +++ b/tox.ini @@ -0,0 +1,5 @@ +[tox] +envlist = py26,py27,py32,py33,pypy + +[testenv] +commands=python setup.py test