diff --git a/.gitignore b/.gitignore index 334f7b11..a050624f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,9 +11,15 @@ Snowboy.pm *.swp *.swo +/examples/C/pa_stable_v19_20140130.tgz +/examples/C/pa_stable_v190600_20161030.tgz +/examples/C/portaudio +/examples/C/demo /examples/C++/pa_stable_v19_20140130.tgz +/examples/C++/pa_stable_v190600_20161030.tgz /examples/C++/portaudio /examples/C++/demo +/examples/C++/demo2 /examples/Java/Demo.class /examples/Perl/data/ /examples/iOS/Obj-C/Pods/Pods.xcodeproj/xcuserdata/ @@ -25,10 +31,14 @@ Snowboy.pm /swig/Android/OpenBLAS-0.2.18.tar.gz /swig/Android/android-ndk-r11c-darwin-x86_64.zip +/swig/Android/android-ndk-r14b-darwin-x86_64.zip /swig/Android/android-ndk-r11c-linux-x86_64.zip /swig/Android/OpenBLAS-Android/ +/swig/Android/OpenBLAS-Android-ARM32/ /swig/Android/android-ndk-r11c/ +/swig/Android/android-ndk-r14b/ /swig/Android/ndk_install/ +/swig/Android/ndk_install_32bit/ /swig/Android/java/ /swig/Android/jniLibs/ /swig/Java/java/ @@ -38,3 +48,7 @@ Snowboy.pm /node_modules /lib/node/binding /lib/node/index.js + +/dist +**/snowboy.egg-info +/.idea diff --git a/.npmignore b/.npmignore index 58c9c95d..6f1b3476 100644 --- a/.npmignore +++ b/.npmignore @@ -17,7 +17,6 @@ snowboydetect.py /node_modules /lib/node/*.ts -/lib/node/*.d.ts .npmignore .travis.yml diff --git a/.travis.yml b/.travis.yml index 56780c5c..306bea20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,8 @@ env: - NODE_VERSION="5.0.0" - NODE_VERSION="6.0.0" - NODE_VERSION="7.0.0" + - NODE_VERSION="8.0.0" + - NODE_VERSION="9.0.0" before_install: # use the correct version of node @@ -47,10 +49,10 @@ before_install: - COMMIT_MESSAGE=$(git show -s --format=%B $TRAVIS_COMMIT | tr -d '\n') # put local node-pre-gyp on PATH - export PATH=./node_modules/.bin/:$PATH -# put global node-gyp on PATH +# put global node-gyp and nan on PATH - npm install node-gyp -g # install aws-sdk so it is available for publishing -- npm install aws-sdk +- npm install aws-sdk nan typescript @types/node # figure out if we should publish or republish - PUBLISH_BINARY=false - REPUBLISH_BINARY=false diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..10f35791 --- /dev/null +++ b/LICENSE @@ -0,0 +1,206 @@ +THIS LICENSE GOVERNS THE SOURCE CODE, THE LIBRARIES, THE RESOURCE FILES, AS WELL +AS THE HOTWORD MODEL snowboy/resources/snowboy.umdl PROVIDED IN THIS REPOSITORY. +ALL OTHER HOTWORD MODELS ARE GOVERNED BY THEIR OWN LICENSES. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..eed4385f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,12 @@ +recursive-include include * +recursive-include lib * +recursive-include swig/Python * +recursive-include resources * +include README.md + +exclude *.txt +exclude *.pyc +global-exclude .DS_Store _snowboydetect.so +prune resources/alexa +prune lib/ios +prune lib/android \ No newline at end of file diff --git a/README.md b/README.md index 4eabfd8e..c4063d45 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,16 @@ +*Dear KITT.AI users,* + +*We are writing this update to let you know that we plan to shut down all KITT.AI products (Snowboy, NLU and Chatflow) by Dec. 31st, 2020.* + +*we launched our first product Snowboy in 2016, and then NLU and Chatflow later that year. Since then, we have served more than 85,000 developers, worldwide, accross all our products. It has been 4 extraordinary years in our life, and we appreciate the opportunity to be able to serve the community.* + +*The field of artificial intelligence is moving rapidly. As much as we like our products, we still see that they are getting outdated and are becoming difficult to maintain. All official websites/APIs for our products will be taken down by Dec. 31st, 2020. Our github repositories will remain open, but only community support will be available from this point beyond.* + +*Thank you all, and goodbye!* + +*The KITT.AI Team +Mar. 18th, 2020* + # Snowboy Hotword Detection by [KITT.AI](http://kitt.ai). @@ -8,65 +21,89 @@ by [KITT.AI](http://kitt.ai). [Discussion Group](https://groups.google.com/a/kitt.ai/forum/#!forum/snowboy-discussion) (or send email to snowboy-discussion@kitt.ai) -(The discussion group is new since September 2016 as we are getting many messages every day. Please send general questions there. For bugs, use Github issues.) +[Commercial application FAQ](README_commercial.md) -Version: 1.2.0 (3/25/2017) +Version: 1.3.0 (2/19/2018) ## Alexa support -Snowboy now brings hands-free experience to the [Alexa AVS sample app](https://github.com/alexa/alexa-avs-sample-app) on Raspberry Pi! See more info below regarding the performance and how you can use other hotword models. +Snowboy now brings hands-free experience to the [Alexa AVS sample app](https://github.com/alexa/avs-device-sdk/wiki/Raspberry-Pi-Quick-Start-Guide-with-Script) on Raspberry Pi! See more info below regarding the performance and how you can use other hotword models. The following instructions currently support AVS sdk Version 1.12.1. **Performance** -The performance of hotword detection usually depends on the actually environment, e.g., is it used with a quality microphone, is it used on the street, in a kitchen, or is there any background noise, etc. So we feel it is best for the users to evaluate it in their real environment. For the evaluation purpose, we have prepared an Android app which can be installed and run out of box. The app is here: +The performance of hotword detection usually depends on the actual environment, e.g., is it used with a quality microphone, is it used on the street, in a kitchen, or is there any background noise, etc. So we feel it is best for the users to evaluate it in their real environment. For the evaluation purpose, we have prepared an Android app which can be installed and run out of box: [SnowboyAlexaDemo.apk](https://github.com/Kitt-AI/snowboy/raw/master/resources/alexa/SnowboyAlexaDemo.apk) (please uninstall any previous versions first if you have installed this app before). + +**Kittai KWD Engine** + +* Set up [Alexa AVS sample app](https://github.com/alexa/avs-device-sdk/wiki/Raspberry-Pi-Quick-Start-Guide-with-Script) following the official AVS instructions -* **resources/alexa/SnowboyAlexaDemo.apk** +* Apply patch to replace the Sensory KWD engine with Kittai engine +``` +# Copy the patch file to the root directory of Alexa AVS sample app. Please replace $ALEXA_AVS_SAMPLE_APP_PATH with the actual path where you +# cloned the Alexa AVS sample app repository, and replace $SNOWBOY_ROOT_PATH with the actual path where you clone the Snowboy repository +cd $ALEXA_AVS_SAMPLE_APP_PATH +cp $SNOWBOY_PATH/resource/alexa/alexa-avs-sample-app/avs-kittai.patch ./ + +# Apply the patch, this will modify the scripts setup.sh and pi.sh +patch < avs-kittai.patch +``` + +* Re-compile the avs-device-sdk and sample app +``` +sudo bash setup.sh config.json +``` + +* Run the sample app +``` +sudo bash startsample.sh +``` + +Here is a [demo video](https://www.youtube.com/watch?v=wiLEr6TeE58) for how to use Snowboy hotword engine in Alexa Voice Service. **Personal model** + * Create your personal hotword model through our [website](https://snowboy.kitt.ai) or [hotword API](https://snowboy.kitt.ai/api/v1/train/) -* Replace the hotword model in [Alexa AVS sample app](https://github.com/alexa/alexa-avs-sample-app) (after installation) with your personal model + +* Put your personal model in [snowboy/resources](https://github.com/Kitt-AI/snowboy/tree/master/resources) ``` -# Please replace YOUR_PERSONAL_MODEL.pmdl with the personal model you just -# created, and $ALEXA_AVS_SAMPLE_APP_PATH with the actual path where you -# cloned the Alexa AVS sample app repository. -cp YOUR_PERSONAL_MODEL.pmdl $ALEXA_AVS_SAMPLE_APP_PATH/samples/wakeWordAgent/ext/resources/alexa.umdl -``` +# Please put YOUR_PERSONAL_MODEL.pmdl in $ALEXA_AVS_SAMPLE_APP_PATH/third-party/snowboy/resources, +# and $ALEXA_AVS_SAMPLE_APP_PATH with the actual path where you put the Alexa AVS sample app repository. + +cp YOUR_PERSONAL_MODEL.pmdl $ALEXA_AVS_SAMPLE_APP_PATH/third-party/snowboy/resources/ -* Set `APPLY_FRONTEND` to `false` and update `SENSITIVITY` in the [Alexa AVS sample app code](https://github.com/alexa/alexa-avs-sample-app/blob/master/samples/wakeWordAgent/src/KittAiSnowboyWakeWordEngine.cpp) and re-compile +``` +* Replace the model name 'alexa.umdl' with your personal model name, update `KITT_AI_SENSITIVITY`, set `KITT_AI_APPLY_FRONT_END_PROCESSING` to `false` in the [Alexa AVS sample app code](https://github.com/alexa/avs-device-sdk/blob/master/KWD/KWDProvider/src/KeywordDetectorProvider.cpp) and re-compile ``` -# Please replace $ALEXA_AVS_SAMPLE_APP_PATH with the actual path where you -# cloned the Alexa AVS sample app repository. -cd $ALEXA_AVS_SAMPLE_APP_PATH/samples/wakeWordAgent/src/ - -# Modify KittAiSnowboyWakeWordEngine.cpp and update SENSITIVITY at line 28. -# Modify KittAiSnowboyWakeWordEngine.cpp and set APPLY_FRONTEND to false at -# line 30. -make +# Modify $ALEXA_AVS_SAMPLE_APP_PATH/avs-device-sdk/blob/master/KWD/KWDProvider/src/KeywordDetectorProvider.cpp: +# Replace the model name 'alexa.umdl' with your personal model name 'YOUR_PERSONAL_MODEL.pmdl' at line 52 +# Update `KITT_AI_SENSITIVITY` at line 26 +# Set `KITT_AI_APPLY_FRONT_END_PROCESSING` to `false` at line 32 +sudo bash setup.sh config.json ``` * Run the wake word agent with engine set to `kitt_ai`! +Here is a [demo video](https://www.youtube.com/watch?v=9Bj8kdfwG7I) for how to use a personal model in Alexa Voice Service. + **Universal model** -* Replace the hotword model in [Alexa AVS sample app](https://github.com/alexa/alexa-avs-sample-app) (after installation) with your universal model +* Put your personal model in [snowboy/resources](https://github.com/Kitt-AI/snowboy/tree/master/resources) ``` -# Please replace YOUR_UNIVERSAL_MODEL.umdl with the personal model you just -# created, and $ALEXA_AVS_SAMPLE_APP_PATH with the actual path where you -# cloned the Alexa AVS sample app repository. -cp YOUR_UNIVERSAL_MODEL.umdl $ALEXA_AVS_SAMPLE_APP_PATH/samples/wakeWordAgent/ext/resources/alexa.umdl -``` +# Please put YOUR_UNIVERSAL_MODEL.umdl in $ALEXA_AVS_SAMPLE_APP_PATH/third-party/snowboy/resources, +# and $ALEXA_AVS_SAMPLE_APP_PATH with the actual path where you put the Alexa AVS sample app repository. -* Update `SENSITIVITY` in the [Alexa AVS sample app code](https://github.com/alexa/alexa-avs-sample-app/blob/master/samples/wakeWordAgent/src/KittAiSnowboyWakeWordEngine.cpp) and re-compile +cp YOUR_UNIVERSAL_MODEL.umdl $ALEXA_AVS_SAMPLE_APP_PATH/third-party/snowboy/resources/ ``` -# Please replace $ALEXA_AVS_SAMPLE_APP_PATH with the actual path where you -# cloned the Alexa AVS sample app repository. -cd $ALEXA_AVS_SAMPLE_APP_PATH/samples/wakeWordAgent/src/ -# Modify KittAiSnowboyWakeWordEngine.cpp and update SENSITIVITY at line 28. -make +* Replace the model name 'alexa.umdl' with your universal model name, update `KITT_AI_SENSITIVITY` in the [Alexa AVS sample app code](https://github.com/alexa/avs-device-sdk/blob/master/KWD/KWDProvider/src/KeywordDetectorProvider.cpp) and re-compile +``` +# Modify $ALEXA_AVS_SAMPLE_APP_PATH/avs-device-sdk/blob/master/KWD/KWDProvider/src/KeywordDetectorProvider.cpp: +# Replace the model name 'alexa.umdl' with your universal model name 'YOUR_UNIVERSAL_MODEL.umdl' at line 52 +# Update `KITT_AI_SENSITIVITY` at line 26 +sudo bash setup.sh config.json ``` * Run the wake word agent with engine set to `kitt_ai`! @@ -112,16 +149,14 @@ less than 10% CPU on the weakest Pi (single-core 700MHz ARMv6). * Apache licensed! -Currently Snowboy supports: +Currently Snowboy supports (look into the [lib](lib) folder): * all versions of Raspberry Pi (with Raspbian based on Debian Jessie 8.0) * 64bit Mac OS X -* 64bit Ubuntu (12.04 and 14.04) +* 64bit Ubuntu 14.04 * iOS * Android -* Pine64 (Debian Jessie 8.5, 3.10.102 BSP2) -* Intel Edison (Ubilinux based on Debian Wheezy 7.8) -* Samsung Artik (built with Fedora 25 for ARMv7) +* ARM64 (aarch64, Ubuntu 16.04) It ships in the form of a **C++ library** with language-dependent wrappers generated by SWIG. We welcome wrappers for new languages -- feel free to send a @@ -129,11 +164,12 @@ pull request! Currently we have built wrappers for: +* C/C++ * Java/Android -* Go (thanks to @brentnd) +* Go (thanks to @brentnd and @deadprogram) * Node (thanks to @evancohen and @nekuz0r) * Perl (thanks to @iboguslavsky) -* Python +* Python2/Python3 * iOS/Swift3 (thanks to @grimlockrocks) * iOS/Object-C (thanks to @patrickjquinn) @@ -163,18 +199,15 @@ environment. Here is the list of the models, and the parameters that you have to use for them: -* **resources/snowboy.umdl**: Universal model for the hotword "Snowboy". Set -SetSensitivity to 0.5 for better performance. -* **resources/alexa.umdl**: Universal model for the hotword "Alexa". Set -SetSensitivity to 0.5, and preferably set ApplyFrontend (only works on Raspberry -Pi) to true. This model is depressed. -* **resources/alexa/alexa_02092017.umdl**: Universal model for the hotword -"Alexa". This is still work in progress. Set SetSensitivity to 0.15. -* **resources/alexa/alexa-avs-sample-app/alexa.umdl**: Universal model for the -hotword "Alexa" optimized for [Alexa AVS sample app](https://github.com/alexa/alexa-avs-sample-app). -Set SetSensitivity to 0.6, and set ApplyFrontend (only works on Raspberry Pi) -to true. This is so far the best "Alexa" model we released publicly, when -ApplyFrontend is set to true. +* **resources/alexa/alexa-avs-sample-app/alexa.umdl**: Universal model for the hotword "Alexa" optimized for [Alexa AVS sample app](https://github.com/alexa/alexa-avs-sample-app). Set SetSensitivity to 0.6, and set ApplyFrontend to true. This is so far the best "Alexa" model we released publicly, when ApplyFrontend is set to true. +* **resources/models/snowboy.umdl**: Universal model for the hotword "Snowboy". Set SetSensitivity to 0.5 and ApplyFrontend to false. +* **resources/models/jarvis.umdl**: Universal model for the hotword "Jarvis" (https://snowboy.kitt.ai/hotword/29). It has two different models for the hotword Jarvis, so you have to use two sensitivites. Set sensitivities to "0.8,0.80" and ApplyFrontend to true. +* **resources/models/smart_mirror.umdl**: Universal model for the hotword "Smart Mirror" (https://snowboy.kitt.ai/hotword/47). Set sensitivity to Sensitivity to 0.5, and ApplyFrontend to false. +* **resources/models/subex.umdl**: Universal model for the hotword "Subex" (https://snowboy.kitt.ai/hotword/22014). Set sensitivity to Sensitivity to 0.5, and ApplyFrontend to true. +* **resources/models/neoya.umdl**: Universal model for the hotword "Neo ya" (https://snowboy.kitt.ai/hotword/22171). It has two different models for the hotword "Neo ya", so you have to use two sensitivites. Set sensitivities to "0.7,0.7", and ApplyFrontend to true. +* **resources/models/hey_extreme.umdl**: Universal model for the hotword "Hey Extreme" (https://snowboy.kitt.ai/hotword/15428). Set sensitivity to Sensitivity to 0.6, and ApplyFrontend to true. +* **resources/models/computer.umdl**: Universal model for the hotword "Computer" (https://snowboy.kitt.ai/hotword/46). Set sensitivity to Sensitivity to 0.6, and ApplyFrontend to true. +* **resources/models/view_glass.umdl**: Universal model for the hotword "View Glass" (https://snowboy.kitt.ai/hotword/7868). Set Sensitivity to 0.7, and ApplyFrontend to true. ## Precompiled node module @@ -189,13 +222,10 @@ dependencies like `fs`, `wav` or `node-record-lpcm16` depending on which script you use. ## Precompiled Binaries with Python Demo -* 64 bit Ubuntu [12.04](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/ubuntu1204-x86_64-1.2.0.tar.bz2) - / [14.04](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/ubuntu1404-x86_64-1.2.0.tar.bz2) -* [MacOS X](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/osx-x86_64-1.2.0.tar.bz2) +* 64 bit Ubuntu [14.04](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/ubuntu1404-x86_64-1.3.0.tar.bz2) +* [MacOS X](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/osx-x86_64-1.3.0.tar.bz2) * Raspberry Pi with Raspbian 8.0, all versions - ([1/2/3/Zero](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/rpi-arm-raspbian-8.0-1.2.0.tar.bz2)) -* Pine64 (Debian Jessie 8.5 (3.10.102)) ([download](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/pine64-debian-jessie-1.2.0.tar.bz2)) -* Intel Edison (Ubilinux based on Debian Wheezy 7.8) ([download](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/edison-ubilinux-1.2.0.tar.bz2)) + ([1/2/3/Zero](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/rpi-arm-raspbian-8.0-1.3.0.tar.bz2)) If you want to compile a version against your own environment/language, read on. @@ -225,13 +255,25 @@ Make sure that you can record audio with your microphone: rec t.wav -### Ubuntu/Raspberry Pi/Pine64 +### Ubuntu/Raspberry Pi/Pine64/Nvidia Jetson TX1/Nvidia Jetson TX2 -First `apt-get` install `swig`, `sox`, `portaudio` and its Python binding `pyaudio`: +First `apt-get` install `sox`, `portaudio` and its Python binding `pyaudio`: - sudo apt-get install swig3.0 python-pyaudio python3-pyaudio sox + sudo apt-get install python-pyaudio python3-pyaudio sox pip install pyaudio +Compile a supported swig version (3.0.10 or above) + + wget http://downloads.sourceforge.net/swig/swig-3.0.10.tar.gz + sudo apt-get install libpcre3 libpcre3-dev + ./configure --prefix=/usr \ + --without-clisp \ + --without-maximum-compile-warnings && + make + make install && + install -v -m755 -d /usr/share/doc/swig-3.0.10 && + cp -v -R Doc/* /usr/share/doc/swig-3.0.10 + Then install the `atlas` matrix computing library: sudo apt-get install libatlas-base-dev @@ -249,7 +291,8 @@ Compiling a node addon for Linux and the Raspberry Pi requires the installation Then to compile the addon run the following from the root of the snowboy repository: - node-pre-gyp clean configure build + npm install + ./node_modules/node-pre-gyp/bin/node-pre-gyp clean configure build ## Compile a Java Wrapper @@ -346,6 +389,7 @@ Full README and tutorial is in [Android README](examples/Android/README.md) and Android Alexa Demo +We have prepared an Android app which can be installed and run out of box: [SnowboyAlexaDemo.apk](https://github.com/Kitt-AI/snowboy/raw/master/resources/alexa/SnowboyAlexaDemo.apk) (please uninstall any previous one first if you installed this app before). ## Quick Start for Python Demo @@ -383,6 +427,15 @@ See [Full Documentation](http://docs.kitt.ai/snowboy). ## Change Log +**v1.3.0, 2/19/2018** + +* Added Frontend processing for all platforms +* Added `resources/models/smart_mirror.umdl` for https://snowboy.kitt.ai/hotword/47 +* Added `resources/models/jarvis.umdl` for https://snowboy.kitt.ai/hotword/29 +* Added README for Chinese +* Cleaned up the supported platforms +* Re-structured the model path + **v1.2.0, 3/25/2017** * Added better Alexa model for [Alexa AVS sample app](https://github.com/alexa/alexa-avs-sample-app) diff --git a/README_ZH_CN.md b/README_ZH_CN.md new file mode 100644 index 00000000..8b94f525 --- /dev/null +++ b/README_ZH_CN.md @@ -0,0 +1,427 @@ +# Snowboy 唤醒词检测 + +[KITT.AI](http://kitt.ai)出品。 + +[Home Page](https://snowboy.kitt.ai) + +[Full Documentation](http://docs.kitt.ai/snowboy) 和 [FAQ](http://docs.kitt.ai/snowboy#faq) + +[Discussion Group](https://groups.google.com/a/kitt.ai/forum/#!forum/snowboy-discussion) (或者发送邮件给 snowboy-discussion@kitt.ai) + +（因为我们每天都会收到很多消息，从2016年9月开始建立了讨论组。请在这里发送一般性的讨论。关于错误，请使用Github问题标签。） + +版本：1.3.0（2/19/2018） + +## Alexa支持 + +Snowboy现在为运行在Raspberry Pi上的[Alexa AVS sample app](https://github.com/alexa/alexa-avs-sample-app)提供了hands-free的体验！有关性能以及如何使用其他唤醒词模型，请参阅下面的信息。 + +**性能** + +唤醒检测的性能通常依赖于实际的环境，例如，它是否与高质量麦克风一起使用，是否在街道上，在厨房中，是否有背景噪音等等. 所以对于性能，我们觉得最好是在使用者真实的环境中进行评估。为了方便评估，我们准备了一个可以直接安装训醒的Android应用程序：[SnowboyAlexaDemo.apk](https://github.com/Kitt-AI/snowboy/raw/master/resources/alexa/SnowboyAlexaDemo.apk) (如果您之前安装了此应用程序，请先卸载它) 。 + +**个人模型** + +* 用以下方式创建您的个人模型：[website](https://snowboy.kitt.ai) 或者 [hotword API](https://snowboy.kitt.ai/api/v1/train/) +* 将[Alexa AVS sample app](https://github.com/alexa/alexa-avs-sample-app)（安装后）的唤醒词模型替换为您的个人模型 + +``` +# Please replace YOUR_PERSONAL_MODEL.pmdl with the personal model you just +# created, and $ALEXA_AVS_SAMPLE_APP_PATH with the actual path where you +# cloned the Alexa AVS sample app repository. +cp YOUR_PERSONAL_MODEL.pmdl $ALEXA_AVS_SAMPLE_APP_PATH/samples/wakeWordAgent/ext/resources/alexa.umdl +``` + +* 在[Alexa AVS sample app code](https://github.com/alexa/alexa-avs-sample-app/blob/master/samples/wakeWordAgent/src/KittAiSnowboyWakeWordEngine.cpp)中设置 `APPLY_FRONTEND` 为 `false`，更新 `SENSITIVITY`，并重新编译 + +``` +# Please replace $ALEXA_AVS_SAMPLE_APP_PATH with the actual path where you +# cloned the Alexa AVS sample app repository. +cd $ALEXA_AVS_SAMPLE_APP_PATH/samples/wakeWordAgent/src/ + +# Modify KittAiSnowboyWakeWordEngine.cpp and update SENSITIVITY at line 28. +# Modify KittAiSnowboyWakeWordEngine.cpp and set APPLY_FRONTEND to false at +# line 30. +make +``` + +* 运行程序，并且把唤醒词引擎设置为`kitt_ai` + + +**通用模型** + +* 将[Alexa AVS sample app](https://github.com/alexa/alexa-avs-sample-app)（安装后）的唤醒词模型替换为您的通用模型 + +``` +# Please replace YOUR_UNIVERSAL_MODEL.umdl with the personal model you just +# created, and $ALEXA_AVS_SAMPLE_APP_PATH with the actual path where you +# cloned the Alexa AVS sample app repository. +cp YOUR_UNIVERSAL_MODEL.umdl $ALEXA_AVS_SAMPLE_APP_PATH/samples/wakeWordAgent/ext/resources/alexa.umdl +``` + +* 在[Alexa AVS sample app code](https://github.com/alexa/alexa-avs-sample-app/blob/master/samples/wakeWordAgent/src/KittAiSnowboyWakeWordEngine.cpp) 中更新 `SENSITIVITY`, 并重新编译 + +``` +# Please replace $ALEXA_AVS_SAMPLE_APP_PATH with the actual path where you +# cloned the Alexa AVS sample app repository. +cd $ALEXA_AVS_SAMPLE_APP_PATH/samples/wakeWordAgent/src/ + +# Modify KittAiSnowboyWakeWordEngine.cpp and update SENSITIVITY at line 28. +make +``` + +* 运行程序，并且把唤醒词引擎设置为`kitt_ai` + + +## 个人唤醒词训练服务 + +Snowboy现在通过 `https://snowboy.kitt.ai/api/v1/train/` 端口提供 **个人唤醒词训练服务**, 请查看[Full Documentation](http://docs.kitt.ai/snowboy)和示例[Python/Bash script](examples/REST_API)（非常欢迎贡献其他的语言)。 + +简单来说，`POST` 下面代码到https://snowboy.kitt.ai/api/v1/train： + + { + "name": "a word", + "language": "en", + "age_group": "10_19", + "gender": "F", + "microphone": "mic type", + "token": "", + "voice_samples": [ + {wave: ""}, + {wave: ""}, + {wave: ""} + ] + } + +然后您会获得一个训练好的个人模型！ + + +## 介绍 + +Snowboy是一款可定制的唤醒词检测引擎，可为您创建像 "OK Google" 或 "Alexa" 这样的唤醒词。Snowboy基于神经网络，具有以下特性： + +* **高度可定制**：您可以自由定义自己的唤醒词 - +比如说“open sesame”，“garage door open”或 “hello dreamhouse”等等。 + +* **总是在监听** 但保护您的个人隐私：Snowboy不使用互联网，不会将您的声音传输到云端。 + +* **轻量级和嵌入式的**：它可以轻松在Raspberry Pi上运行，甚至在最弱的Pi（单核700MHz ARMv6）上，Snowboy占用的CPU也少于10%。 + +* Apache授权! + +目前Snowboy支持（查看lib文件夹）： + +* 所有版本的Raspberry Pi（Raspbian基于Debian Jessie 8.0） +* 64位Mac OS X +* 64位Ubuntu 14.04 +* iOS +* Android +* ARM64（aarch64，Ubuntu 16.04) + +Snowboy底层库由C++写成，通过swig被封装成能在多种操作系统和语言上使用的软件库。我们欢迎新语言的封装，请随时发送你们的Pull Request！ + +目前我们已经现实封装的有: + +* C/C++ +* Java / Android +* Go（thanks to @brentnd and @deadprogram） +* Node（thanks to @evancohen和@ nekuz0r） +* Perl（thanks to @iboguslavsky） +* Python2/Python3 +* iOS / Swift3（thanks to @grimlockrocks） +* iOS / Object-C（thanks to @patrickjquinn） + +如果您想要支持其他硬件或操作系统，请将您的请求发送至[snowboy@kitt.ai](mailto:snowboy.kitt.ai) + +注意：**Snowboy还不支持Windows** 。请在 *nix平台上编译Snowboy。 + +## Snowboy模型的定价 + +黑客：免费 + +* 个人使用 +* 社区支持 + +商业：请通过[snowboy@kitt.ai](mailto:snowboy@kitt.ai)与我们联系 + +* 个人使用 +* 商业许可证 +* 技术支持 + +## 预训练的通用模型 + +为了测试方便，我们提供一些事先训练好的通用模型。当您测试那些模型时，请记住他们可能没有为您的特定设备或环境进行过优化。 + +以下是模型列表和您必须使用的参数： + +* **resources/alexa/alexa-avs-sample-app/alexa.umdl**：这个是为[Alexa AVS sample app](https://github.com/alexa/alexa-avs-sample-app)优化过的唤醒词为“Alexa”的通用模型，将`SetSensitivity`设置为`0.6`，并将`ApplyFrontend`设置为true。当`ApplyFrontend`设置为`true`时，这是迄今为止我们公开发布的最好的“Alexa”的模型。 +* **resources/models/snowboy.umdl**：唤醒词为“snowboy”的通用模型。将`SetSensitivity`设置为`0.5`，`ApplyFrontend`设置为`false`。 +* **resources/models/jarvis.umdl**: 唤醒词为“Jarvis” (https://snowboy.kitt.ai/hotword/29) 的通用模型，其中包含了对应于“Jarvis”的两个唤醒词模型，所以需要设置两个`sensitivity`。将`SetSensitivity`设置为`0.8,0.8`，`ApplyFrontend`设置为`true`。 +* **resources/models/smart_mirror.umdl**: 唤醒词为“Smart Mirror” (https://snowboy.kitt.ai/hotword/47) 的通用模型。将`SetSensitivity`设置为`0.5`，`ApplyFrontend`设置为`false`。 +* **resources/models/subex.umdl**: 唤醒词为“Subex”(https://snowboy.kitt.ai/hotword/22014) 的通用模型。将`SetSensitivity`设置为`0.5`，`ApplyFrontend`设置为`true`。 +* **resources/models/neoya.umdl**: 唤醒词为“Neo ya”(https://snowboy.kitt.ai/hotword/22171) 的通用模型。其中包含了对应于“Neo ya”的两个>唤醒词模型，所以需要设置两个`sensitivity`。将`SetSensitivity`设置为`0.7,0.7`，`ApplyFrontend`设置为`true`。 +* **resources/models/hey_extreme.umdl**: 唤醒词为“Hey Extreme” (https://snowboy.kitt.ai/hotword/15428)的通用模型。将`SetSensitivity`设置为`0.6`，`ApplyFrontend`设置为`true`。 +* **resources/models/computer.umdl**: 唤醒词为“Computer” (https://snowboy.kitt.ai/hotword/46) 的通用模型。将`SetSensitivity`设置为`0.6`，`ApplyFrontend`设置为`true`。 +* **resources/models/view_glass.umdl**: 唤醒词为“View Glass” (https://snowboy.kitt.ai/hotword/7868) 的通用模型。将`SetSensitivity`设置为`0.7`，`ApplyFrontend`设置为`true`。 + +## 预编译node模块 + +Snowboy为一下平台编译了node模块：64位Ubuntu，MacOS X和Raspberry Pi（Raspbian 8.0+）。快速安装运行： + + npm install --save snowboy + +有关示例用法，请参阅examples/Node文件夹。根据您使用的脚本，可能需要安装依赖关系库例如fs，wav或node-record-lpcm16。 + +## 预编译Python Demo的二进制文件 +* 64 bit Ubuntu [12.04](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/ubuntu1204-x86_64-1.2.0.tar.bz2) + / [14.04](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/ubuntu1404-x86_64-1.3.0.tar.bz2) +* [MacOS X](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/osx-x86_64-1.3.0.tar.bz2) +* Raspberry Pi with Raspbian 8.0, all versions + ([1/2/3/Zero](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/rpi-arm-raspbian-8.0-1.3.0.tar.bz2)) +* Pine64 (Debian Jessie 8.5 (3.10.102)), Nvidia Jetson TX1 and Nvidia Jetson TX2 ([download](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/pine64-debian-jessie-1.2.0.tar.bz2)) +* Intel Edison (Ubilinux based on Debian Wheezy 7.8) ([download](https://s3-us-west-2.amazonaws.com/snowboy/snowboy-releases/edison-ubilinux-1.2.0.tar.bz2)) + +如果您要根据自己的环境/语言编译版本，请继续阅读。 + +## 依赖 + +要运行demo，您可能需要以下内容，具体取决于您使用的示例和您正在使用的平台： + +* SoX（音频转换） +* PortAudio或PyAudio（音频录音） +* SWIG 3.0.10或以上（针对不同语言/平台编译Snowboy） +* ATLAS或OpenBLAS（矩阵计算） + +在下面您还可以找到在Mac OS X，Ubuntu或Raspberry Pi上安装依赖关系所需的确切命令。 + +### Mac OS X + +`brew` 安装 `swig`，`sox`，`portaudio` 和绑定了 `pyaudio`的Python： + + brew install swig portaudio sox + pip install pyaudio + +如果您没有安装Homebrew，请在这里[here](http://brew.sh/)下载。如果没有pip，可以在这里[here](https://pip.pypa.io/en/stable/installing/)安装。 + +确保您可以用麦克风录制音频： + + rec t.wav + +### Ubuntu / Raspberry Pi / Pine64 / Nvidia Jetson TX1 / Nvidia Jetson TX2 + +首先 `apt-get` 安装 `swig`，`sox`，`portaudio`和绑定了 `pyaudio` 的 Python： + + sudo apt-get install swig3.0 python-pyaudio python3-pyaudio sox + pip install pyaudio + +然后安装 `atlas` 矩阵计算库： + + sudo apt-get install libatlas-base-dev + +确保您可以用麦克风录制音频： + + rec t.wav + +如果您需要额外设置您的音频（特别是Raspberry Pi），请参阅[full documentation](http://docs.kitt.ai/snowboy)。 + +## 编译Node插件 + +为Linux和Raspberry Pi编译node插件需要安装以下依赖项: + + sudo apt-get install libmagic-dev libatlas-base-dev + +然后编译插件，从snowboy代码库的根目录运行以下内容： + + npm install + ./node_modules/node-pre-gyp/bin/node-pre-gyp clean configure build + +## 编译Java Wrapper + + # Make sure you have JDK installed. + cd swig/Java + make + +SWIG将生成一个包含转换成Java封装的`java`目录和一个包含JNI库的`jniLibs`目录。 + +运行Java示例脚本： + + cd examples/Java + make run + +## 编译Python Wrapper + + cd swig/Python + make + +SWIG将生成一个_snowboydetect.so文件和一个简单（但难以阅读）的python 封装snowboydetect.py。我们已经提供了一个更容易读懂的python封装snowboydecoder.py。 + +如果不能make，请适配`swig/Python`中的Makefile到您自己的系统设置。 + +## 编译GO Warpper + + cd examples/Go + go get github.com/Kitt-AI/snowboy/swig/Go + go build -o snowboy main.go + ./snowboy ../../resources/snowboy.umdl ../../resources/snowboy.wav + +期望输出： + + Snowboy detecting keyword in ../../resources/snowboy.wav + Snowboy detected keyword 1 + + +更多细节，请阅读 'examples/Go/readme.md'。 + +## 编译Perl wrapper + + cd swig/Perl + make + +Perl示例包括使用KITT.AI RESTful API训练个人唤醒词，在检测到唤醒之后添加Google Speech API等。要运行示例，请执行以下操作 + + cd examples/Perl + + # Install cpanm, if you don't already have it. + curl -L https://cpanmin.us | perl - --sudo App::cpanminus + + # Install the dependencies. Note, on Linux you will have to install the + # PortAudio package first, using e.g.: + # apt-get install portaudio19-dev + sudo cpanm --installdeps . + + # Run the unit test. + ./snowboy_unit_test.pl + + # Run the personal model training example. + ./snowboy_RESTful_train.pl + + # Run the Snowboy Google Speech API example. By default it uses the Snowboy + # universal hotword. + ./snowboy_googlevoice.pl [Hotword_Model] + +## 编译iOS wrapper + +在Objective-C中使用Snowboy库不需要封装. 它与Objective-C中使用C++库基本相同. 我们为iOS设备编写了一个 "fat" 静态库，请参阅这里的库`lib/ios/libsnowboy-detect.a`。 + +在Objective-C中初始化Snowboy检测器： + + snowboy::SnowboyDetect* snowboyDetector = new snowboy::SnowboyDetect( + std::string([[[NSBundle mainBundle]pathForResource:@"common" ofType:@"res"] UTF8String]), + std::string([[[NSBundle mainBundle]pathForResource:@"snowboy" ofType:@"umdl"] UTF8String])); + snowboyDetector->SetSensitivity("0.45"); // Sensitivity for each hotword + snowboyDetector->SetAudioGain(2.0); // Audio gain for detection + +在Objective-C中运行唤醒词检测： + + int result = snowboyDetector->RunDetection(buffer[0], bufferSize); // buffer[0] is a float array + +您可能需要按照一定的频率调用RunDetection()，从而控制CPU使用率和检测延迟。 + +感谢@patrickjquinn和@grimlockrocks，我们现在有了在Objective-C和Swift3中使用Snowboy的例子。看看下面的例子`examples/iOS/`和下面的截图！ + +Obj-C Example Swift3 Example + +# 编译Android Wrapper + +完整的README和教程在[Android README](examples/Android/README.md)，这里是一个截图： + +Android Alexa Demo + +我们准备了一个可以安装并运行的Android应用程序：[SnowboyAlexaDemo.apk](https://github.com/Kitt-AI/snowboy/raw/master/resources/alexa/SnowboyAlexaDemo.apk)（如果您之前安装了此应用程序，请先卸载它们)。 + +## Python demo快速入门 + +进入 `examples/Python` 文件夹并打开你的python控制台： + + In [1]: import snowboydecoder + + In [2]: def detected_callback(): + ....: print "hotword detected" + ....: + + In [3]: detector = snowboydecoder.HotwordDetector("resources/snowboy.umdl", sensitivity=0.5, audio_gain=1) + + In [4]: detector.start(detected_callback) + +然后对你的麦克风说"snowboy"，看看是否Snowboy检测到你。 + +这个 `snowboy.umdl` 文件是一个 "通用" 模型，可以检测不同的人说 "snowboy" 。 如果你想要其他的唤醒词，请去[snowboy.kitt.ai](https://snowboy.kitt.ai)录音，训练和下载你自己的个人模型(一个.pmdl文件)。 + +当 `sensitiviy` 设置越高，唤醒越容易触发。但是你也可能会收到更多的误唤醒。 + +`audio_gain` 控制是否增加（> 1）或降低（<1）输入音量。 + +我们提供了两个演示文件 `demo.py`, `demo2.py` 以显示更多的用法。 + +注意：如果您看到以下错误： + + TypeError: __init__() got an unexpected keyword argument 'model_str' + +您可能正在使用旧版本的SWIG. 请升级SWIG。我们已经测试过SWIG 3.0.7和3.0.8。 + +## 高级用法与演示 + +请参阅[Full Documentation](http://docs.kitt.ai/snowboy)。 + +## 更改日志 + +**v1.3.0, 2/19/2018** + +* 添加前端处理到所有平台 +* 添加`resources/models/smart_mirror.umdl` 给 https://snowboy.kitt.ai/hotword/47 +* 添加`resources/models/jarvis.umdl` 给 https://snowboy.kitt.ai/hotword/29 +* 添加中文文档 +* 清理支持的平台 +* 重新定义了模型路径 + +**v1.2.0, 3/25/2017** + +* 为[Alexa AVS sample app](https://github.com/alexa/alexa-avs-sample-app)添加更好的Alexa模型 +* 新的解码器，适用于像Alexa这样的简短的词条 + +**v1.1.1, 3/24/2017** + +* 添加Android演示 +* 添加了iOS演示 +* 增加了三星Artik支持 +* 添加Go支持 +* 增加了英特尔爱迪生支持 +* 增加了Pine64的支持 +* 增加了Perl支持 +* 添加了更强大的“Alexa”模型（umdl） +* 通过/api/v1/train终端提供Hotword即服务。 +* 解码器没有改变 + +**v1.1.0, 9/20/2016** + +* 添加了Node的库 +* 增加了对Python3的支持 +* 增加了通用模型 alexa.umdl +* 更新通用模型snowboy.umdl，使其在嘈杂的环境中工作 + +**v1.0.4, 7/13/2016** + +* 更新通用snowboy.umdl模型，使其更加健壮 +* 各种改进加快检测 +* Bug修复 + +**v1.0.3, 6/4/2016** + +* 更新的通用snowboy.umdl模型，使其在非语音环境中更加强大 +* 修正使用float作为输入数据时的错误 +* 为Android ARMV7架构增加了库支持 +* 为iOS添加了库 + +**v1.0.2, 5/24/2016** + +* 更新通用snowboy.umdl模型 +* 添加C ++示例，文档将在下一个版本中 + +**v1.0.1, 5/16/2016** + +* VAD现在返回-2为静音，-1为错误，0为语音，大于0为触发了唤醒 +* 添加了Raspberry Pi的静态库，以防人们想自己编译而不是使用二进制版本 + +**v1.0.0, 5/10/2016** + +* 初始版本 diff --git a/README_commercial.md b/README_commercial.md new file mode 100644 index 00000000..b2daa37d --- /dev/null +++ b/README_commercial.md @@ -0,0 +1,134 @@ +# Common Questions for a Commercial Application + +You are looking for a way to put Snowboy in a commercial application. We have compiled a large collection of common questions from our customers all over the world in various industries. + + +## Universal models (paid) vs. personal models (free) + +Personal models: + +* are the models you downloaded from https://snowboy.kitt.ai or using our `/train` SaaS API. +* are good for quick demos +* are built with only 3 voice samples +* are not noise robust and you'll get a lot of false alarms in real environment +* only work on your own voice or a very similar voice, thus is speaker dependent +* are free + +Universal models: + +* are built using a lot more voice samples (at least thousands) +* take effort to collect those voice samples +* take a lot of GPU time to train +* are more robust against noise +* are mostly speaker independent (with challenges on children's voice and accents) +* cannot be built by yourself using the web interface or the SaaS API +* cost you money + +### FAQ for universal & personal models + +Q: **If I record multiple times on snowboy.kitt.ai, can I improve the personal models?** +A: No. Personal models only take 3 voice samples to build. Each time you record new voices, the previous samples are overwritten and not used in your current model. + + +Q: **How can I get a universal model for free?** +A: The *one and only* way: Ask 500 people to log in to snowboy.kitt.ai, contribute their voice samples to a particular hotword, then ask us to build a universal model for that hotword. + +Q: **Can I use your API to collect voices from 500 people and increment the sample counter from snowboy.kitt.ai?** +A: No. The [SaaS](https://github.com/kitt-ai/snowboy#hotword-as-a-service) API is separate from the website. + +Q: **How long does it take to get a universal model?** +A: Usually a month. + +## Licensing + + +### Explain your license again? + +Everything on Snowboy's GitHub repo is Apache licensed, including various sample applications and wrapper codes, though the Snowboy library is binary code compiled against different platforms. + +With that said, if you built an application from https://github.com/kitt-ai/snowboy or personal models downloaded from https://snowboy.kitt.ai, you don't need to pay a penny. + +If you want to use a universal model with your own customized hotword, you'll need an **evaluation license** and a **commercial license**. + +### Evaluation license + +Each hotword is different. When you train a universal model with your own hotword, nobody can guarantee that it works on your system without any flaws. Thus you'll need to get an evaluation license first to test whether your universal model works for you. + +An evaluation license: + +* gives you a 90 day window to evaluate the universal model we build for you +* costs you money + +**Warning: an evaluation license will expire after 90 days. Make sure you don't use the model with evaluation license in production systems.** Get a commercial license from us for your production system. + +#### Evaluation license FAQ + +Q: **How much does it cost?** +A: A few thousand dollars. + +Q: **Can I get a discount as a {startup, student, NGO}?** +A: No. Our pricing is already at least half of what others charge. + +Q: **How can you make sure your universal model works for me?** +A: We simply can't. However we have a few sample universal models from our GitHub [repo](https://github.com/Kitt-AI/snowboy/tree/master/resources), including "alexa.umdl", "snowboy.umdl", and "smart_mirror.umdl". The "alexa.umdl" model is enhanced with a lot more data and is not a typical case. So pay attention to test "snowboy.umdl" and "smart_mirror.umdl". They offer similar performance to your model. + + +### Commercial license + +After evaluation, if you feel want to go with Snowboy, you'll need a commercial license to deploy it. We usually charge a flat fee per unit of hardware you sell. + +#### Commercial license FAQ + +Q: **Is it a one-time license or subscription-based license?** +A: It's a perpetual license for each device. Since the Snowboy library runs *offline* on your device, you can run it forever without worrying about any broken and dependent web services. + +Q: **What's your pricing structure?** +A: We have tiered pricing depending on your volume. We charge less if you sell more. + +Q: **Can you give me one example?** +A: For instance, if your product is a talking robot with a $300 price tag, and you sell at least 100,000 units per year, we'll probably charge you $1 per unit once you go over 100,000 units. If your product is a smart speaker with a $30 price tag, we won't charge you $1, but you'll have to sell a lot more to make the business sense to us. + +Q: **I plan to sell 1000 units a year, can I license your software for $1 per unit?** +A: No. In that way we only make $1000 a year, which is not worth the amount of time we put on your hotword. + +Q: **I make a cellphone app, not a hardware product, what's the pricing structure?** +A: Depends on how you generate revenue. For instance, if your app is priced at $1.99, we'll collect cents per paid user, assuming you have a large user base. If you only have 2000 paid users, we'll make a revenue of less than a hundred dollars and it won't make sense to us. + + +### What's the process of getting a license? + +1. Make sure Snowboy can run on your system +2. Reach out to us with your hotword name, commercial application, and target market +3. Discuss with us about **commercial license** fee to make sure our pricing fits your budget +4. Sign an evaluation contract, pay 50% of invoice +5. We'll train a universal model for you and give you an **evaluation license** of 90 days +6. Test the model and discuss how we can improve it +7. If you decide to go with it, get a commercial license from us + +## General Questions + +### What language does Snowboy support? + +We support North American English and Chinese the best. We can deal with a bit of Indian accents as well. For other languages, we'll need to first listen to your hotword (please send us a few .wav voice samples) before we can engage. + +### How many voice samples do you need? + +Usually 1500 voice samples from 500 people to get started. The more the better. If your hotword is in English, we can collect the voice samples for you. Otherwise you'll need to collect it yourself and send to us. + +### What's the format on voice samples? + +16000Hz sample rate, 16 bit integer, mono channel, .wav files. + +### Does Snowboy do: AEC, VAD, Noise Suppression, Beam Forming? + +Snowboy has a weak support for VAD and noise suppression, as we found some customers would use Snowboy without a microphone array. Snowboy is not a audio frontend processing toolkit thus does not support AEC and beam forming. + +If your application wants to support far-field speech, i.e., verbal communication at least 3 feet away, you'll need a microphone array to enhance incoming speech and reduce noise. Please do not reply on Snowboy to do everything. + +### Can you compile Snowboy for my platform? + +If your platform is not listed [here](https://github.com/Kitt-AI/snowboy/tree/master/lib), and you want to get a commercial license from us, please contact us with your toolchain, hardware chip, RAM, OS, GCC/G++ version. Depending on the effort, we might charge an NRE fee for cross compiling. + +### Contact + +If this document doesn't cover what's needed, feel free to reach out to us at snowboy@kitt.ai \ No newline at end of file diff --git a/binding.gyp b/binding.gyp index ab833be4..b2917aef 100644 --- a/binding.gyp +++ b/binding.gyp @@ -31,6 +31,16 @@ '<(module_root_dir)/lib/rpi/libsnowboy-detect.a', ] } + }], + ['OS=="linux" and target_arch=="arm64"', { + 'link_settings': { + 'ldflags': [ + '-Wl,--no-as-needed', + ], + 'libraries': [ + '<(module_root_dir)/lib/aarch64-ubuntu1604/libsnowboy-detect.a', + ] + } }] ], 'cflags': [ diff --git a/examples/Android/README.md b/examples/Android/README.md index a4c429e9..e64f1e81 100644 --- a/examples/Android/README.md +++ b/examples/Android/README.md @@ -1,22 +1,31 @@ -# Snowboy Demo on Adroid +# Snowboy Demo on Android Note: 1. supported building platforms are Android Studio running on Mac OS X or Ubuntu. Windows is not supported. -2. supported target CPU is ARMv7 (most Android phones run on ARM CPUs) +2. supported target CPU is ARMv7 (32bit) and ARMv8 (64bit) (most Android phones run on ARM CPUs) +3. we have prepared an Android app which can be installed and run out of box: [SnowboyAlexaDemo.apk](https://github.com/Kitt-AI/snowboy/raw/master/resources/alexa/SnowboyAlexaDemo.apk) (please uninstall any previous one first if you installed this app before). ## General Workflow -1. Install `swig`. For Mac, do `brew install swig`; for Ubuntu, do `sudo apt-get install swig3.0`. Make sure your `swig` version is at least `3.0.10`. +1. Install `swig`. For Mac, do `brew install swig`; for Ubuntu, do `sudo apt-get install swig3.0`. Make sure your `swig` version is at least `3.0.10`. You'll also need `wget` to download files. 2. Go to `swig/Android` and build swig wrappers for Snowboy: cd swig/Android make + + To make for ARMv8 64bit: + + make BIT=64 Ths will generate a cross-compiled library for ARM: - jniLibs/armeabi-v7a/libsnowboy-detect-android.so + jniLibs/ + ├── arm64-v8a + │   └── libsnowboy-detect-android.so + └── armeabi-v7a + └── libsnowboy-detect-android.so and a few Java wrapper files: @@ -63,9 +72,9 @@ To run hotword detection in Java: You may want to play with the frequency of the calls to `RunDetection()`, which controls the CPU usage and the detection latency. -## TODO +## Common Asks -The following TODOs will be fixed by 2017/4 in the upcoming Snowboy v1.2.0 release. +The following issues have been fixed pushed to `master`. - [x] softfloating point support with OpenBlas - [x] upgrade NDK version to newer than r11c diff --git a/examples/Android/SnowboyAlexaDemo/assets/snowboy/alexa.umdl b/examples/Android/SnowboyAlexaDemo/assets/snowboy/alexa.umdl new file mode 120000 index 00000000..03a4f523 --- /dev/null +++ b/examples/Android/SnowboyAlexaDemo/assets/snowboy/alexa.umdl @@ -0,0 +1 @@ +../../../../../resources/alexa/alexa-avs-sample-app/alexa.umdl \ No newline at end of file diff --git a/examples/Android/SnowboyAlexaDemo/assets/snowboy/alexa_02092017.umdl b/examples/Android/SnowboyAlexaDemo/assets/snowboy/alexa_02092017.umdl deleted file mode 120000 index 843305f7..00000000 --- a/examples/Android/SnowboyAlexaDemo/assets/snowboy/alexa_02092017.umdl +++ /dev/null @@ -1 +0,0 @@ -../../../../../resources/alexa/alexa_02092017.umdl \ No newline at end of file diff --git a/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/Constants.java b/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/Constants.java index 7e758044..5ccc72f1 100644 --- a/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/Constants.java +++ b/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/Constants.java @@ -5,7 +5,7 @@ public class Constants { public static final String ASSETS_RES_DIR = "snowboy"; public static final String DEFAULT_WORK_SPACE = Environment.getExternalStorageDirectory().getAbsolutePath() + "/snowboy/"; - public static final String ACTIVE_UMDL = "alexa_02092017.umdl"; + public static final String ACTIVE_UMDL = "alexa.umdl"; public static final String ACTIVE_RES = "common.res"; public static final String SAVE_AUDIO = Constants.DEFAULT_WORK_SPACE + File.separatorChar + "recording.pcm"; public static final int SAMPLE_RATE = 16000; diff --git a/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/SnowboyVad.java b/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/SnowboyVad.java new file mode 120000 index 00000000..12df56be --- /dev/null +++ b/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/SnowboyVad.java @@ -0,0 +1 @@ +../../../../../../../swig/Android/java/ai/kitt/snowboy/SnowboyVad.java \ No newline at end of file diff --git a/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/audio/AudioDataSaver.java b/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/audio/AudioDataSaver.java index e847847f..5f464507 100644 --- a/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/audio/AudioDataSaver.java +++ b/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/audio/AudioDataSaver.java @@ -15,6 +15,19 @@ public class AudioDataSaver implements AudioDataReceivedListener { private static final String TAG = AudioDataSaver.class.getSimpleName(); + // file size of when to delete and create a new recording file + private final float MAX_RECORDING_FILE_SIZE_IN_MB = 50f; + + // initial file size of recording file + private final float INITIAL_FILE_SIZE_IN_MB = 1.3f; + + // converted max file size + private final float MAX_RECORDING_FILE_SIZE_IN_BYTES + = (MAX_RECORDING_FILE_SIZE_IN_MB - INITIAL_FILE_SIZE_IN_MB) * 1024 * 1024; + + // keeps track of recording file size + private int recordingFileSizeCounterInBytes = 0; + private File saveFile = null; private DataOutputStream dataOutputStreamInstance = null; @@ -25,7 +38,7 @@ public AudioDataSaver() { @Override public void start() { - if(null != saveFile) { + if (null != saveFile) { if (saveFile.exists()) { saveFile.delete(); } @@ -36,7 +49,7 @@ public void start() { } try { - BufferedOutputStream bufferedStreamInstance = new BufferedOutputStream( + BufferedOutputStream bufferedStreamInstance = new BufferedOutputStream( new FileOutputStream(this.saveFile)); dataOutputStreamInstance = new DataOutputStream(bufferedStreamInstance); } catch (FileNotFoundException e) { @@ -48,8 +61,14 @@ public void start() { @Override public void onAudioDataReceived(byte[] data, int length) { try { - if(null != dataOutputStreamInstance) { + if (null != dataOutputStreamInstance) { + if (recordingFileSizeCounterInBytes >= MAX_RECORDING_FILE_SIZE_IN_BYTES) { + stop(); + start(); + recordingFileSizeCounterInBytes = 0; + } dataOutputStreamInstance.write(data, 0, length); + recordingFileSizeCounterInBytes += length; } } catch (IOException e) { Log.e(TAG, "IO Exception on saving audio file " + saveFile.toString(), e); @@ -58,7 +77,7 @@ public void onAudioDataReceived(byte[] data, int length) { @Override public void stop() { - if(null != dataOutputStreamInstance) { + if (null != dataOutputStreamInstance) { try { dataOutputStreamInstance.close(); } catch (IOException e) { diff --git a/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/audio/PlaybackThread.java b/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/audio/PlaybackThread.java index 43f033e5..41574bd7 100644 --- a/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/audio/PlaybackThread.java +++ b/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/audio/PlaybackThread.java @@ -25,6 +25,7 @@ public PlaybackThread() { private Thread thread; private boolean shouldContinue; + protected AudioTrack audioTrack; public boolean playing() { return thread != null; @@ -50,9 +51,18 @@ public void stopPlayback() { return; shouldContinue = false; + relaseAudioTrack(); thread = null; } + protected void relaseAudioTrack() { + if (audioTrack != null) { + try { + audioTrack.release(); + } catch (Exception e) {} + } + } + public short[] readPCM() { try { File recordFile = new File(Constants.SAVE_AUDIO); @@ -62,11 +72,6 @@ public short[] readPCM() { byte[] audioData = new byte[(int)recordFile.length()]; - // int i = 0; - // while (dataInputStream.available() > 0) { - // audioData[i] = dataInputStream.readByte(); - // i++; - // } dataInputStream.read(audioData); dataInputStream.close(); Log.v(TAG, "audioData size: " + audioData.length); @@ -89,7 +94,7 @@ private void play() { int bufferSizeInBytes = samples.length * shortSizeInBytes; Log.v(TAG, "shortSizeInBytes: " + shortSizeInBytes + " bufferSizeInBytes: " + bufferSizeInBytes); - AudioTrack audioTrack = new AudioTrack( + audioTrack = new AudioTrack( AudioManager.STREAM_MUSIC, Constants.SAMPLE_RATE, AudioFormat.CHANNEL_OUT_MONO, @@ -97,13 +102,14 @@ private void play() { bufferSizeInBytes, AudioTrack.MODE_STREAM); - audioTrack.play(); - - audioTrack.write(samples, 0, samples.length); - Log.v(TAG, "Audio playback started"); + if (audioTrack.getState() == AudioTrack.STATE_INITIALIZED) { + audioTrack.play(); + audioTrack.write(samples, 0, samples.length); + Log.v(TAG, "Audio playback started"); + } if (!shouldContinue) { - audioTrack.release(); + relaseAudioTrack(); } } } diff --git a/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/audio/RecordingThread.java b/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/audio/RecordingThread.java index fa768c27..850e955a 100644 --- a/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/audio/RecordingThread.java +++ b/examples/Android/SnowboyAlexaDemo/src/ai/kitt/snowboy/audio/RecordingThread.java @@ -39,6 +39,16 @@ public class RecordingThread { public RecordingThread(Handler handler, AudioDataReceivedListener listener) { this.handler = handler; this.listener = listener; + + detector.SetSensitivity("0.6"); + detector.SetAudioGain(1); + detector.ApplyFrontend(true); + try { + player.setDataSource(strEnvWorkSpace+"ding.wav"); + player.prepare(); + } catch (IOException e) { + Log.e(TAG, "Playing ding sound error", e); + } } private void sendMessage(MsgEnum what, Object obj){ @@ -49,15 +59,6 @@ private void sendMessage(MsgEnum what, Object obj){ } public void startRecording() { - detector.SetSensitivity("0.6"); - //-detector.SetAudioGain(1); - detector.ApplyFrontend(true); - try { - player.setDataSource(strEnvWorkSpace+"ding.wav"); - player.prepare(); - } catch (IOException e) { - Log.e(TAG, "Playing ding sound error", e); - } if (thread != null) return; @@ -108,8 +109,9 @@ private void record() { Log.v(TAG, "Start recording"); long shortsRead = 0; + detector.Reset(); while (shouldContinue) { - record.read(audioBuffer, 0, audioBuffer.length, AudioRecord.READ_BLOCKING); + record.read(audioBuffer, 0, audioBuffer.length); if (null != listener) { listener.onAudioDataReceived(audioBuffer, audioBuffer.length); @@ -126,12 +128,12 @@ private void record() { if (result == -2) { // post a higher CPU usage: - sendMessage(MsgEnum.MSG_VAD_NOSPEECH, null); + // sendMessage(MsgEnum.MSG_VAD_NOSPEECH, null); } else if (result == -1) { sendMessage(MsgEnum.MSG_ERROR, "Unknown Detection Error"); } else if (result == 0) { // post a higher CPU usage: - sendMessage(MsgEnum.MSG_VAD_SPEECH, null); + // sendMessage(MsgEnum.MSG_VAD_SPEECH, null); } else if (result > 0) { sendMessage(MsgEnum.MSG_ACTIVE, null); Log.i("Snowboy: ", "Hotword " + Integer.toString(result) + " detected!"); diff --git a/examples/C++/Makefile b/examples/C++/Makefile index 1026a878..c8904032 100644 --- a/examples/C++/Makefile +++ b/examples/C++/Makefile @@ -1,6 +1,6 @@ include demo.mk -BINFILES = demo +BINFILES = demo demo2 all: $(BINFILES) diff --git a/examples/C++/demo.cc b/examples/C++/demo.cc index eaa2d424..330e6ffa 100644 --- a/examples/C++/demo.cc +++ b/examples/C++/demo.cc @@ -200,17 +200,20 @@ int main(int argc, char* argv[]) { // Parameter section. // If you have multiple hotword models (e.g., 2), you should set // and as follows: - // model_filename = "resources/snowboy.umdl,resources/alexa.pmdl"; - // sensitivity_str = "0.4,0.4"; + // model_filename = + // "resources/models/snowboy.umdl,resources/models/smart_mirror.umdl"; + // sensitivity_str = "0.5,0.5"; std::string resource_filename = "resources/common.res"; - std::string model_filename = "resources/snowboy.umdl"; + std::string model_filename = "resources/models/snowboy.umdl"; std::string sensitivity_str = "0.5"; float audio_gain = 1; + bool apply_frontend = false; // Initializes Snowboy detector. snowboy::SnowboyDetect detector(resource_filename, model_filename); detector.SetSensitivity(sensitivity_str); detector.SetAudioGain(audio_gain); + detector.ApplyFrontend(apply_frontend); // Initializes PortAudio. You may use other tools to capture the audio. PortAudioWrapper pa_wrapper(detector.SampleRate(), diff --git a/examples/C++/demo.mk b/examples/C++/demo.mk index 38e8a7a7..db136d11 100644 --- a/examples/C++/demo.mk +++ b/examples/C++/demo.mk @@ -33,13 +33,7 @@ else ifeq ($(shell uname), Linux) -Wno-unused-local-typedefs -Winit-self -rdynamic \ -DHAVE_POSIX_MEMALIGN -I$(PORTAUDIOINC) LDLIBS += -ldl -lm -Wl,-Bstatic -Wl,-Bdynamic -lrt -lpthread $(PORTAUDIOLIBS)\ - -L/usr/lib/atlas-base -lf77blas -lcblas -llapack_atlas -latlas - ifneq ($(wildcard $(PORTAUDIOINC)/pa_linux_alsa.h),) - LDLIBS += -lasound - endif - ifneq ($(wildcard $(PORTAUDIOINC)/pa_jack.h),) - LDLIBS += -ljack - endif + -L/usr/lib/atlas-base -lf77blas -lcblas -llapack_atlas -latlas -lasound SNOWBOYDETECTLIBFILE := $(TOPDIR)/lib/ubuntu64/libsnowboy-detect.a ifneq (,$(findstring arm,$(shell uname -m))) SNOWBOYDETECTLIBFILE := $(TOPDIR)/lib/rpi/libsnowboy-detect.a diff --git a/examples/C++/demo2.cc b/examples/C++/demo2.cc new file mode 100644 index 00000000..64a17e6e --- /dev/null +++ b/examples/C++/demo2.cc @@ -0,0 +1,150 @@ +#include +#include "include/snowboy-detect.h" +#include "portaudio.h" + +#define resource_filename "resources/common.res" +#define model_filename "resources/models/snowboy.umdl" +#define sensitivity_str "0.5" +#define audio_gain 1.0 +#define apply_frontend false + +struct wavHeader { //44 byte HEADER only + char RIFF[4]; + int RIFFsize; + char fmt[8]; + int fmtSize; + short fmtTag; + short nchan; + int fs; + int avgBps; + short nBlockAlign; + short bps; + char data[4]; + int datasize; +}; + + +void readWavHeader(wavHeader *wavhdr, FILE *fi) { + //===================================================== + // Reads the WAV file header considering the follow restrictions: + // - format tag needs to be 1=PCM (no encoding) + // - shoud be imidiately before the databytes + // (it should not contain chunks after 'data') + // Returns a pointer pointing to the begining of the data + + char *tag = (char *)wavhdr; + fread(wavhdr, 34, 1, fi); //starting tag should be "RIFF" + if (tag[0] != 'R' || tag[1] != 'I' || tag[2] != 'F' || tag[3] != 'F') { + fclose(fi); + perror("NO 'RIFF'."); + } + if (wavhdr->fmtTag != 1) { + fclose(fi); + perror("WAV file has encoded data or it is WAVEFORMATEXTENSIBLE."); + } + if (wavhdr->fmtSize == 14) { + wavhdr->bps = 16; + } + if (wavhdr->fmtSize >= 16) { + fread(&wavhdr->bps, 2, 1, fi); + } + if (wavhdr->fmtSize == 18) { + short lixo; + fread(&lixo, 2, 1, fi); + } + tag += 36; //aponta para wavhdr->data + fread(tag, 4, 1, fi); //data chunk deve estar aqui. + while (tag[0] != 'd' || tag[1] != 'a' || tag[2] != 't' || tag[3] != 'a') { + fread(tag, 4, 1, fi); + if (ftell(fi) >= long(wavhdr->RIFFsize)) { + fclose(fi); + perror("Bad WAV header !"); + } + } + fread(&wavhdr->datasize, 4, 1, fi); //data size + // Assuming that header ends here. + // From here until the end it is audio data +} + + + +int main(int argc, char * argv[]) { + std::string usage = + "C++ demo that shows how to use snowboy. In this examle user can read\n" + "the audio data from a file.\n" + "\n" + "Atention reading from a file: this software is for simulation/test\n" + "only. You need to take precautions when loading a file into the\n" + "memory.\n" + "\n" + "To run the example:\n" + " ./demo2 [filename.raw || filename.wav ]\n" + "\n" + "IMPORTANT NOTE: Raw file must be 16kHz sample, mono and 16bit\n"; + + // default + char * filename; + int fsize; + short * data_buffer = NULL; + bool isRaw = true; + FILE *f = NULL; + + if (argc > 2 or argc < 2) { + std::cout << usage << std::endl; + exit(1); + } else { + filename = argv[1]; + } + + std::string str = filename; + std::string type = ".wav"; + + if (str.find(type) != std::string::npos) { + isRaw = false; + } + + + if (filename != NULL) { + f = fopen(filename,"rb"); + } + + if (f == NULL) { + perror ("Error opening file"); + return(-1); + } + + if (!isRaw) { + wavHeader *wavhdr = new wavHeader(); + readWavHeader(wavhdr, f); + + data_buffer = (short *)malloc(wavhdr->datasize); + // Consume all the audio to the buffer + fread(data_buffer, wavhdr->datasize, 1, f); + fclose(f); + fsize = wavhdr->datasize; + } else { + fseek(f,0,SEEK_END); + fsize = ftell(f); + rewind(f); + + // Consume all the audio to the buffer + data_buffer = (short *)malloc(fsize); + int aa = fread(&data_buffer[0], 1 ,fsize, f); + std::cout << "Read bytes: " << aa << std::endl; + fclose(f); + + } + + // Initializes Snowboy detector. + snowboy::SnowboyDetect detector(resource_filename, model_filename); + detector.SetSensitivity(sensitivity_str); + detector.SetAudioGain(audio_gain); + detector.ApplyFrontend(apply_frontend); + + int result = detector.RunDetection(&data_buffer[0], fsize/sizeof(short)); + std::cout << ">>>>> Result: " << result << " <<<<<" << std::endl; + std::cout << "Legend: -2: noise | -1: error | 0: silence | 1: hotword" + << std::endl; + + return 0; +} diff --git a/examples/C++/install_portaudio.sh b/examples/C++/install_portaudio.sh index beaccd4f..ceecd2e0 100755 --- a/examples/C++/install_portaudio.sh +++ b/examples/C++/install_portaudio.sh @@ -2,22 +2,27 @@ # This script attempts to install PortAudio, which can grap a live audio stream # from the soundcard. +# +# On linux systems, we only build with ALSA, so make sure you install it using +# e.g.: +# sudo apt-get -y install libasound2-dev echo "Installing portaudio" -if [ ! -e pa_stable_v19_20140130.tgz ]; then +if [ ! -e pa_stable_v190600_20161030.tgz ]; then wget -T 10 -t 3 \ - http://www.portaudio.com/archives/pa_stable_v19_20140130.tgz || exit 1; + http://www.portaudio.com/archives/pa_stable_v190600_20161030.tgz || exit 1; fi -tar -xovzf pa_stable_v19_20140130.tgz || exit 1 +tar -xovzf pa_stable_v190600_20161030.tgz || exit 1 cd portaudio patch < ../patches/portaudio.patch MACOS=`uname 2>/dev/null | grep Darwin` if [ -z "$MACOS" ]; then - ./configure --prefix=`pwd`/install --with-pic + ./configure --without-jack --without-oss \ + --with-alsa --prefix=`pwd`/install --with-pic || exit 1; sed -i '40s:src/common/pa_ringbuffer.o::g' Makefile sed -i '40s:$: src/common/pa_ringbuffer.o:' Makefile else diff --git a/examples/C++/patches/portaudio.patch b/examples/C++/patches/portaudio.patch new file mode 100644 index 00000000..1c618329 --- /dev/null +++ b/examples/C++/patches/portaudio.patch @@ -0,0 +1,11 @@ +--- Makefile.in 2017-05-31 16:42:16.000000000 -0700 ++++ Makefile_new.in 2017-05-31 16:44:02.000000000 -0700 +@@ -193,6 +193,8 @@ + for include in $(INCLUDES); do \ + $(INSTALL_DATA) -m 644 $(top_srcdir)/include/$$include $(DESTDIR)$(includedir)/$$include; \ + done ++ $(INSTALL_DATA) -m 644 $(top_srcdir)/src/common/pa_ringbuffer.h $(DESTDIR)$(includedir)/$$include ++ $(INSTALL_DATA) -m 644 $(top_srcdir)/src/common/pa_util.h $(DESTDIR)$(includedir)/$$include + $(INSTALL) -d $(DESTDIR)$(libdir)/pkgconfig + $(INSTALL) -m 644 portaudio-2.0.pc $(DESTDIR)$(libdir)/pkgconfig/portaudio-2.0.pc + @echo "" diff --git a/examples/C/Makefile b/examples/C/Makefile new file mode 100644 index 00000000..a310eb9f --- /dev/null +++ b/examples/C/Makefile @@ -0,0 +1,20 @@ +include demo.mk + +BINFILE = demo + +OBJFILES = demo.o snowboy-detect-c-wrapper.o + +all: $(BINFILE) + +%.a: + $(MAKE) -C ${@D} ${@F} + +# We have to use the C++ compiler to link. +$(BINFILE): $(PORTAUDIOLIBS) $(SNOWBOYDETECTLIBFILE) $(OBJFILES) + $(CXX) $(OBJFILES) $(SNOWBOYDETECTLIBFILE) $(PORTAUDIOLIBS) $(LDLIBS) -o $(BINFILE) + +$(PORTAUDIOLIBS): + @-./install_portaudio.sh + +clean: + -rm -f *.o *.a $(BINFILE) $(OBJFILES) diff --git a/examples/C/demo.c b/examples/C/demo.c new file mode 100644 index 00000000..f72f36ba --- /dev/null +++ b/examples/C/demo.c @@ -0,0 +1,223 @@ +// example/C/demo.c + +// Copyright 2017 KITT.AI (author: Guoguo Chen) + +#include +#include +#include +#include +#include +#include +#include + +#include "snowboy-detect-c-wrapper.h" + +// Pointer to the ring buffer memory. +char* g_ringbuffer; +// Ring buffer wrapper used in PortAudio. +PaUtilRingBuffer g_pa_ringbuffer; +// Pointer to PortAudio stream. +PaStream* g_pa_stream; +// Number of lost samples at each LoadAudioData() due to ring buffer overflow. +int g_num_lost_samples; +// Wait for this number of samples in each LoadAudioData() call. +int g_min_read_samples; +// Pointer to the audio data. +int16_t* g_data; + +int PortAudioCallback(const void* input, + void* output, + unsigned long frame_count, + const PaStreamCallbackTimeInfo* time_info, + PaStreamCallbackFlags status_flags, + void* user_data) { + ring_buffer_size_t num_written_samples = + PaUtil_WriteRingBuffer(&g_pa_ringbuffer, input, frame_count); + g_num_lost_samples += frame_count - num_written_samples; + return paContinue; +} + +void StartAudioCapturing(int sample_rate, + int num_channels, int bits_per_sample) { + g_data = NULL; + g_num_lost_samples = 0; + g_min_read_samples = sample_rate * 0.1; + + // Allocates ring buffer memory. + int ringbuffer_size = 16384; + g_ringbuffer = (char*)( + PaUtil_AllocateMemory(bits_per_sample / 8 * ringbuffer_size)); + if (g_ringbuffer == NULL) { + fprintf(stderr, "Fail to allocate memory for ring buffer.\n"); + exit(1); + } + + // Initializes PortAudio ring buffer. + ring_buffer_size_t rb_init_ans = + PaUtil_InitializeRingBuffer(&g_pa_ringbuffer, bits_per_sample / 8, + ringbuffer_size, g_ringbuffer); + if (rb_init_ans == -1) { + fprintf(stderr, "Ring buffer size is not power of 2.\n"); + exit(1); + } + + // Initializes PortAudio. + PaError pa_init_ans = Pa_Initialize(); + if (pa_init_ans != paNoError) { + fprintf(stderr, "Fail to initialize PortAudio, error message is %s.\n", + Pa_GetErrorText(pa_init_ans)); + exit(1); + } + + PaError pa_open_ans; + if (bits_per_sample == 8) { + pa_open_ans = Pa_OpenDefaultStream( + &g_pa_stream, num_channels, 0, paUInt8, sample_rate, + paFramesPerBufferUnspecified, PortAudioCallback, NULL); + } else if (bits_per_sample == 16) { + pa_open_ans = Pa_OpenDefaultStream( + &g_pa_stream, num_channels, 0, paInt16, sample_rate, + paFramesPerBufferUnspecified, PortAudioCallback, NULL); + } else if (bits_per_sample == 32) { + pa_open_ans = Pa_OpenDefaultStream( + &g_pa_stream, num_channels, 0, paInt32, sample_rate, + paFramesPerBufferUnspecified, PortAudioCallback, NULL); + } else { + fprintf(stderr, "Unsupported BitsPerSample: %d.\n", bits_per_sample); + exit(1); + } + if (pa_open_ans != paNoError) { + fprintf(stderr, "Fail to open PortAudio stream, error message is %s.\n", + Pa_GetErrorText(pa_open_ans)); + exit(1); + } + + PaError pa_stream_start_ans = Pa_StartStream(g_pa_stream); + if (pa_stream_start_ans != paNoError) { + fprintf(stderr, "Fail to start PortAudio stream, error message is %s.\n", + Pa_GetErrorText(pa_stream_start_ans)); + exit(1); + } +} + +void StopAudioCapturing() { + if (g_data != NULL) { + free(g_data); + g_data = NULL; + } + Pa_StopStream(g_pa_stream); + Pa_CloseStream(g_pa_stream); + Pa_Terminate(); + PaUtil_FreeMemory(g_ringbuffer); +} + +int LoadAudioData() { + if (g_data != NULL) { + free(g_data); + g_data = NULL; + } + + // Checks ring buffer overflow. + if (g_num_lost_samples > 0) { + fprintf(stderr, "Lost %d samples due to ring buffer overflow.\n", + g_num_lost_samples); + g_num_lost_samples = 0; + } + + ring_buffer_size_t num_available_samples = 0; + while (true) { + num_available_samples = + PaUtil_GetRingBufferReadAvailable(&g_pa_ringbuffer); + if (num_available_samples >= g_min_read_samples) { + break; + } + Pa_Sleep(5); + } + + // Reads data. + num_available_samples = PaUtil_GetRingBufferReadAvailable(&g_pa_ringbuffer); + g_data = malloc(num_available_samples * sizeof(int16_t)); + ring_buffer_size_t num_read_samples = PaUtil_ReadRingBuffer( + &g_pa_ringbuffer, g_data, num_available_samples); + if (num_read_samples != num_available_samples) { + fprintf(stderr, "%d samples were available, but only %d samples were read" + ".\n", num_available_samples, num_read_samples); + } + return num_read_samples; +} + +void SignalHandler(int signal) { + fprintf(stderr, "Caught signal %d, terminating...\n", signal); + exit(0); +} + +int main(int argc, char* argv[]) { + const char usage[] = + "Example that shows how to use Snowboy in pure C. Snowboy was written\n" + "in C++, so we have to write a wrapper in order to use Snowboy in pure\n" + "C. See snowboy-detect-c-wrapper.h and snowboy-detect-c-wrapper.cc for\n" + "more details.\n" + "\n" + "Parameters are hard-coded in the parameter section for this example.\n" + "Please check the source code for more details.\n" + "\n" + "Audio is captured by PortAudio, feel free to replace PortAudio with\n" + "your own audio capturing tool.\n" + "\n" + "To run the example:\n" + " ./demo\n"; + + // Checks the command. + if (argc > 1) { + printf("%s", usage); + exit(1); + } + + // Configures signal handling. + struct sigaction sig_int_handler; + sig_int_handler.sa_handler = SignalHandler; + sigemptyset(&sig_int_handler.sa_mask); + sig_int_handler.sa_flags = 0; + sigaction(SIGINT, &sig_int_handler, NULL); + + // Parameter section. + // If you have multiple hotword models (e.g., 2), you should set + // and as follows: + // model_filename = + // "resources/models/snowboy.umdl,resources/models/smart_mirror.umdl"; + // sensitivity_str = "0.5,0.5"; + const char resource_filename[] = "resources/common.res"; + const char model_filename[] = "resources/models/snowboy.umdl"; + const char sensitivity_str[] = "0.5"; + float audio_gain = 1; + bool apply_frontend = false; + + // Initializes Snowboy detector. + SnowboyDetect* detector = SnowboyDetectConstructor(resource_filename, + model_filename); + SnowboyDetectSetSensitivity(detector, sensitivity_str); + SnowboyDetectSetAudioGain(detector, audio_gain); + SnowboyDetectApplyFrontend(detector, apply_frontend); + + // Initializes PortAudio. You may use other tools to capture the audio. + StartAudioCapturing(SnowboyDetectSampleRate(detector), + SnowboyDetectNumChannels(detector), + SnowboyDetectBitsPerSample(detector)); + + // Runs the detection. + printf("Listening... Press Ctrl+C to exit\n"); + while (true) { + int array_length = LoadAudioData(); + if (array_length != 0) { + int result = SnowboyDetectRunDetection(detector, + g_data, array_length, false); + if (result > 0) { + printf("Hotword %d detected!\n", result); + } + } + } + + StopAudioCapturing(); + SnowboyDetectDestructor(detector); + return 0; +} diff --git a/examples/C/demo.mk b/examples/C/demo.mk new file mode 100644 index 00000000..73d1140c --- /dev/null +++ b/examples/C/demo.mk @@ -0,0 +1,58 @@ +TOPDIR := ../../ +DYNAMIC := True +CC := +CXX := +LDFLAGS := +LDLIBS := +PORTAUDIOINC := portaudio/install/include +PORTAUDIOLIBS := portaudio/install/lib/libportaudio.a + +CFLAGS := +CXXFLAGS += -D_GLIBCXX_USE_CXX11_ABI=0 + +ifeq ($(DYNAMIC), True) + CFLAGS += -fPIC + CXXFLAGS += -fPIC +endif + +ifeq ($(shell uname -m | cut -c 1-3), x86) + CFLAGS += -msse -msse2 + CXXFLAGS += -msse -msse2 +endif + +ifeq ($(shell uname), Darwin) + # By default Mac uses clang++ as g++, but people may have changed their + # default configuration. + CC := clang + CXX := clang++ + CFLAGS += -I$(TOPDIR) -Wall -I$(PORTAUDIOINC) + CXXFLAGS += -I$(TOPDIR) -Wall -Wno-sign-compare -Winit-self \ + -DHAVE_POSIX_MEMALIGN -DHAVE_CLAPACK -I$(PORTAUDIOINC) + LDLIBS += -ldl -lm -framework Accelerate -framework CoreAudio \ + -framework AudioToolbox -framework AudioUnit -framework CoreServices \ + $(PORTAUDIOLIBS) + SNOWBOYDETECTLIBFILE := $(TOPDIR)/lib/osx/libsnowboy-detect.a +else ifeq ($(shell uname), Linux) + CC := gcc + CXX := g++ + CFLAGS += -I$(TOPDIR) -Wall -I$(PORTAUDIOINC) + CXXFLAGS += -I$(TOPDIR) -std=c++0x -Wall -Wno-sign-compare \ + -Wno-unused-local-typedefs -Winit-self -rdynamic \ + -DHAVE_POSIX_MEMALIGN -I$(PORTAUDIOINC) + LDLIBS += -ldl -lm -Wl,-Bstatic -Wl,-Bdynamic -lrt -lpthread $(PORTAUDIOLIBS)\ + -L/usr/lib/atlas-base -lf77blas -lcblas -llapack_atlas -latlas -lasound + SNOWBOYDETECTLIBFILE := $(TOPDIR)/lib/ubuntu64/libsnowboy-detect.a + ifneq (,$(findstring arm,$(shell uname -m))) + SNOWBOYDETECTLIBFILE := $(TOPDIR)/lib/rpi/libsnowboy-detect.a + endif +endif + +# Suppress clang warnings... +COMPILER = $(shell $(CXX) -v 2>&1 ) +ifeq ($(findstring clang,$(COMPILER)), clang) + CXXFLAGS += -Wno-mismatched-tags -Wno-c++11-extensions +endif + +# Set optimization level. +CFLAGS += -O3 +CXXFLAGS += -O3 diff --git a/examples/C/install_portaudio.sh b/examples/C/install_portaudio.sh new file mode 100755 index 00000000..ceecd2e0 --- /dev/null +++ b/examples/C/install_portaudio.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# This script attempts to install PortAudio, which can grap a live audio stream +# from the soundcard. +# +# On linux systems, we only build with ALSA, so make sure you install it using +# e.g.: +# sudo apt-get -y install libasound2-dev + +echo "Installing portaudio" + +if [ ! -e pa_stable_v190600_20161030.tgz ]; then + wget -T 10 -t 3 \ + http://www.portaudio.com/archives/pa_stable_v190600_20161030.tgz || exit 1; +fi + +tar -xovzf pa_stable_v190600_20161030.tgz || exit 1 + +cd portaudio +patch < ../patches/portaudio.patch + +MACOS=`uname 2>/dev/null | grep Darwin` +if [ -z "$MACOS" ]; then + ./configure --without-jack --without-oss \ + --with-alsa --prefix=`pwd`/install --with-pic || exit 1; + sed -i '40s:src/common/pa_ringbuffer.o::g' Makefile + sed -i '40s:$: src/common/pa_ringbuffer.o:' Makefile +else + # People may have changed OSX's default configuration -- we use clang++. + CC=clang CXX=clang++ ./configure --prefix=`pwd`/install --with-pic +fi + +make +make install + +cd .. diff --git a/examples/C/patches/portaudio.patch b/examples/C/patches/portaudio.patch new file mode 100644 index 00000000..1c618329 --- /dev/null +++ b/examples/C/patches/portaudio.patch @@ -0,0 +1,11 @@ +--- Makefile.in 2017-05-31 16:42:16.000000000 -0700 ++++ Makefile_new.in 2017-05-31 16:44:02.000000000 -0700 +@@ -193,6 +193,8 @@ + for include in $(INCLUDES); do \ + $(INSTALL_DATA) -m 644 $(top_srcdir)/include/$$include $(DESTDIR)$(includedir)/$$include; \ + done ++ $(INSTALL_DATA) -m 644 $(top_srcdir)/src/common/pa_ringbuffer.h $(DESTDIR)$(includedir)/$$include ++ $(INSTALL_DATA) -m 644 $(top_srcdir)/src/common/pa_util.h $(DESTDIR)$(includedir)/$$include + $(INSTALL) -d $(DESTDIR)$(libdir)/pkgconfig + $(INSTALL) -m 644 portaudio-2.0.pc $(DESTDIR)$(libdir)/pkgconfig/portaudio-2.0.pc + @echo "" diff --git a/examples/C/resources b/examples/C/resources new file mode 120000 index 00000000..bc764151 --- /dev/null +++ b/examples/C/resources @@ -0,0 +1 @@ +../../resources \ No newline at end of file diff --git a/examples/C/snowboy-detect-c-wrapper.cc b/examples/C/snowboy-detect-c-wrapper.cc new file mode 100644 index 00000000..9129271a --- /dev/null +++ b/examples/C/snowboy-detect-c-wrapper.cc @@ -0,0 +1,82 @@ +// snowboy-detect-c-wrapper.cc + +// Copyright 2017 KITT.AI (author: Guoguo Chen) + +#include + +#include "snowboy-detect-c-wrapper.h" +#include "include/snowboy-detect.h" + +extern "C" { + SnowboyDetect* SnowboyDetectConstructor(const char* const resource_filename, + const char* const model_str) { + return reinterpret_cast( + new snowboy::SnowboyDetect(resource_filename, model_str)); + } + + bool SnowboyDetectReset(SnowboyDetect* detector) { + assert(detector != NULL); + return reinterpret_cast(detector)->Reset(); + } + + int SnowboyDetectRunDetection(SnowboyDetect* detector, + const int16_t* const data, + const int array_length, bool is_end) { + assert(detector != NULL); + assert(data != NULL); + return reinterpret_cast( + detector)->RunDetection(data, array_length, is_end); + } + + void SnowboyDetectSetSensitivity(SnowboyDetect* detector, + const char* const sensitivity_str) { + assert(detector != NULL); + reinterpret_cast( + detector)->SetSensitivity(sensitivity_str); + } + + void SnowboyDetectSetAudioGain(SnowboyDetect* detector, + const float audio_gain) { + assert(detector != NULL); + reinterpret_cast( + detector)->SetAudioGain(audio_gain); + } + + void SnowboyDetectUpdateModel(SnowboyDetect* detector) { + assert(detector != NULL); + reinterpret_cast(detector)->UpdateModel(); + } + + void SnowboyDetectApplyFrontend(SnowboyDetect* detector, + const bool apply_frontend) { + assert(detector != NULL); + reinterpret_cast( + detector)->ApplyFrontend(apply_frontend); + } + + int SnowboyDetectNumHotwords(SnowboyDetect* detector) { + assert(detector != NULL); + return reinterpret_cast(detector)->NumHotwords(); + } + + int SnowboyDetectSampleRate(SnowboyDetect* detector) { + assert(detector != NULL); + return reinterpret_cast(detector)->SampleRate(); + } + + int SnowboyDetectNumChannels(SnowboyDetect* detector) { + assert(detector != NULL); + return reinterpret_cast(detector)->NumChannels(); + } + + int SnowboyDetectBitsPerSample(SnowboyDetect* detector) { + assert(detector != NULL); + return reinterpret_cast(detector)->BitsPerSample(); + } + + void SnowboyDetectDestructor(SnowboyDetect* detector) { + assert(detector != NULL); + delete reinterpret_cast(detector); + detector = NULL; + } +} diff --git a/examples/C/snowboy-detect-c-wrapper.h b/examples/C/snowboy-detect-c-wrapper.h new file mode 100644 index 00000000..99a4e02a --- /dev/null +++ b/examples/C/snowboy-detect-c-wrapper.h @@ -0,0 +1,51 @@ +// snowboy-detect-c-wrapper.h + +// Copyright 2017 KITT.AI (author: Guoguo Chen) + +#ifndef SNOWBOY_DETECT_C_WRAPPER_H_ +#define SNOWBOY_DETECT_C_WRAPPER_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct SnowboyDetect SnowboyDetect; + + SnowboyDetect* SnowboyDetectConstructor(const char* const resource_filename, + const char* const model_str); + + bool SnowboyDetectReset(SnowboyDetect* detector); + + int SnowboyDetectRunDetection(SnowboyDetect* detector, + const int16_t* const data, + const int array_length, bool is_end); + + void SnowboyDetectSetSensitivity(SnowboyDetect* detector, + const char* const sensitivity_str); + + void SnowboyDetectSetAudioGain(SnowboyDetect* detector, + const float audio_gain); + + void SnowboyDetectUpdateModel(SnowboyDetect* detector); + + void SnowboyDetectApplyFrontend(SnowboyDetect* detector, + const bool apply_frontend); + + int SnowboyDetectNumHotwords(SnowboyDetect* detector); + + int SnowboyDetectSampleRate(SnowboyDetect* detector); + + int SnowboyDetectNumChannels(SnowboyDetect* detector); + + int SnowboyDetectBitsPerSample(SnowboyDetect* detector); + + void SnowboyDetectDestructor(SnowboyDetect* detector); + +#ifdef __cplusplus +} +#endif + +#endif // SNOWBOY_DETECT_C_WRAPPER_H_ diff --git a/examples/Go/main.go b/examples/Go/detect/main.go similarity index 87% rename from examples/Go/main.go rename to examples/Go/detect/main.go index 4d476fd8..00ec04d6 100644 --- a/examples/Go/main.go +++ b/examples/Go/detect/main.go @@ -15,9 +15,10 @@ func main() { return } fmt.Printf("Snowboy detecting keyword in %s\n", os.Args[2]) - detector := snowboydetect.NewSnowboyDetect("../../resources/common.res", os.Args[1]) + detector := snowboydetect.NewSnowboyDetect("../../../resources/common.res", os.Args[1]) detector.SetSensitivity("0.5") detector.SetAudioGain(1) + detector.ApplyFrontend(false) defer snowboydetect.DeleteSnowboyDetect(detector) dat, err := ioutil.ReadFile(os.Args[2]) @@ -36,4 +37,4 @@ func main() { } else { fmt.Println("Snowboy detected keyword ", res) } -} \ No newline at end of file +} diff --git a/examples/Go/readme.md b/examples/Go/detect/readme.md similarity index 85% rename from examples/Go/readme.md rename to examples/Go/detect/readme.md index 7cf05b38..1b8ea908 100644 --- a/examples/Go/readme.md +++ b/examples/Go/detect/readme.md @@ -22,7 +22,7 @@ go build -o snowboy main.go ### Examples Cmd: -`./snowboy ../../resources/snowboy.umdl ../../resources/snowboy.wav` +`./snowboy ../../../resources/models/snowboy.umdl ../../../resources/snowboy.wav` Output: ``` @@ -37,4 +37,4 @@ Output: ``` Snowboy detecting keyword in ../../resources/snowboy.wav Snowboy detected nothing -``` \ No newline at end of file +``` diff --git a/examples/Go/listen/README.md b/examples/Go/listen/README.md new file mode 100644 index 00000000..cf8aab00 --- /dev/null +++ b/examples/Go/listen/README.md @@ -0,0 +1,36 @@ +## Dependencies + +### Swig +http://www.swig.org/ + +### Go Package alongside the more idiomatic wrapper `go-snowboy`, plus PortAudio +``` +github.com/brentnd/go-snowboy +github.com/gordonklaus/portaudio +``` + +## Building + +``` +go build -o listen main.go +``` + +## Running + +``` +./listen [path to snowboy resource file] [path to snowboy hotword file] +``` + +### Examples +Cmd: +`./listen ../../../resources/common.res ../../../resources/models/snowboy.umdl` + +Output: +``` +sample rate=16000, num channels=1, bit depth=16 +Silence detected. +Silence detected. +Silence detected. +You said the hotword! +Silence detected. +``` diff --git a/examples/Go/listen/main.go b/examples/Go/listen/main.go new file mode 100644 index 00000000..af772f72 --- /dev/null +++ b/examples/Go/listen/main.go @@ -0,0 +1,101 @@ +// This example streams the microphone thru Snowboy to listen for the hotword, +// by using the PortAudio interface. +// +// HOW TO USE: +// go run examples/Go/listen/main.go [path to snowboy resource file] [path to snowboy hotword file] +// +package main + +import ( + "bytes" + "encoding/binary" + "fmt" + "os" + "time" + + "github.com/brentnd/go-snowboy" + "github.com/gordonklaus/portaudio" +) + +// Sound represents a sound stream implementing the io.Reader interface +// that provides the microphone data. +type Sound struct { + stream *portaudio.Stream + data []int16 +} + +// Init initializes the Sound's PortAudio stream. +func (s *Sound) Init() { + inputChannels := 1 + outputChannels := 0 + sampleRate := 16000 + s.data = make([]int16, 1024) + + // initialize the audio recording interface + err := portaudio.Initialize() + if err != nil { + fmt.Errorf("Error initialize audio interface: %s", err) + return + } + + // open the sound input stream for the microphone + stream, err := portaudio.OpenDefaultStream(inputChannels, outputChannels, float64(sampleRate), len(s.data), s.data) + if err != nil { + fmt.Errorf("Error open default audio stream: %s", err) + return + } + + err = stream.Start() + if err != nil { + fmt.Errorf("Error on stream start: %s", err) + return + } + + s.stream = stream +} + +// Close closes down the Sound's PortAudio connection. +func (s *Sound) Close() { + s.stream.Close() + portaudio.Terminate() +} + +// Read is the Sound's implementation of the io.Reader interface. +func (s *Sound) Read(p []byte) (int, error) { + s.stream.Read() + + buf := &bytes.Buffer{} + for _, v := range s.data { + binary.Write(buf, binary.LittleEndian, v) + } + + copy(p, buf.Bytes()) + return len(p), nil +} + +func main() { + // open the mic + mic := &Sound{} + mic.Init() + defer mic.Close() + + // open the snowboy detector + d := snowboy.NewDetector(os.Args[1]) + defer d.Close() + + // set the handlers + d.HandleFunc(snowboy.NewHotword(os.Args[2], 0.5), func(string) { + fmt.Println("You said the hotword!") + }) + + d.HandleSilenceFunc(1*time.Second, func(string) { + fmt.Println("Silence detected.") + }) + + // display the detector's expected audio format + sr, nc, bd := d.AudioFormat() + fmt.Printf("sample rate=%d, num channels=%d, bit depth=%d\n", sr, nc, bd) + + // start detecting using the microphone + d.ReadAndDetect(mic) +} diff --git a/examples/Java/Demo.java b/examples/Java/Demo.java index 6360b8f5..d064007d 100644 --- a/examples/Java/Demo.java +++ b/examples/Java/Demo.java @@ -21,9 +21,10 @@ public static void main(String[] args) { // Sets up Snowboy. SnowboyDetect detector = new SnowboyDetect("resources/common.res", - "resources/snowboy.umdl"); + "resources/models/snowboy.umdl"); detector.SetSensitivity("0.5"); detector.SetAudioGain(1); + detector.ApplyFrontend(false); try { TargetDataLine targetLine = diff --git a/examples/Node/file.js b/examples/Node/file.js index 8e59f06e..c97bf540 100644 --- a/examples/Node/file.js +++ b/examples/Node/file.js @@ -6,7 +6,7 @@ const Models = require('../../').Models; const models = new Models(); models.add({ - file: 'resources/snowboy.umdl', + file: 'resources/models/snowboy.umdl', sensitivity: '0.5', hotwords : 'snowboy' }); @@ -14,14 +14,17 @@ models.add({ const detector = new Detector({ resource: "resources/common.res", models: models, - audioGain: 1.0 + audioGain: 1.0, + applyFrontend: false }); detector.on('silence', function () { console.log('silence'); }); -detector.on('sound', function () { +detector.on('sound', function (buffer) { + // contains the last chunk of the audio that triggers the "sound" + // event. It could be written to a wav stream. console.log('sound'); }); @@ -29,7 +32,11 @@ detector.on('error', function () { console.log('error'); }); -detector.on('hotword', function (index, hotword) { +detector.on('hotword', function (index, hotword, buffer) { + // contains the last chunk of the audio that triggers the "hotword" + // event. It could be written to a wav stream. You will have to use it + // together with the in the "sound" event if you want to get audio + // data after the hotword. console.log('hotword', index, hotword); }); diff --git a/examples/Node/microphone.js b/examples/Node/microphone.js index 40997bba..a0dfe2ee 100644 --- a/examples/Node/microphone.js +++ b/examples/Node/microphone.js @@ -5,7 +5,7 @@ const Models = require('../../').Models; const models = new Models(); models.add({ - file: 'resources/snowboy.umdl', + file: 'resources/models/snowboy.umdl', sensitivity: '0.5', hotwords : 'snowboy' }); @@ -13,14 +13,17 @@ models.add({ const detector = new Detector({ resource: "resources/common.res", models: models, - audioGain: 2.0 + audioGain: 2.0, + applyFrontend: true }); detector.on('silence', function () { console.log('silence'); }); -detector.on('sound', function () { +detector.on('sound', function (buffer) { + // contains the last chunk of the audio that triggers the "sound" + // event. It could be written to a wav stream. console.log('sound'); }); @@ -28,7 +31,12 @@ detector.on('error', function () { console.log('error'); }); -detector.on('hotword', function (index, hotword) { +detector.on('hotword', function (index, hotword, buffer) { + // contains the last chunk of the audio that triggers the "hotword" + // event. It could be written to a wav stream. You will have to use it + // together with the in the "sound" event if you want to get audio + // data after the hotword. + console.log(buffer); console.log('hotword', index, hotword); }); diff --git a/examples/Perl/snowboy_googlevoice.pl b/examples/Perl/snowboy_googlevoice.pl index 8c43af2d..97b5d055 100755 --- a/examples/Perl/snowboy_googlevoice.pl +++ b/examples/Perl/snowboy_googlevoice.pl @@ -3,9 +3,9 @@ # This script first uses Snowboy to wake up, then collects audio and sends to # Google Speech API for further recognition. It works with both personal and # universal models. By default, it uses the Snowboy universal model at -# resources/snowboy.umdl, you can change it to other universal models, or your -# own personal models. You also have to provide your Google API key in order to -# use it. +# resources/models/snowboy.umdl, you can change it to other universal models, or +# your own personal models. You also have to provide your Google API key in +# order to use it. use Snowboy; @@ -23,16 +23,16 @@ This script first uses Snowboy to wake up, then collects audio and sends to Google Speech API for further recognition. It works with both personal and universal models. By default, it uses the Snowboy universal model at -resources/snowboy.umdl, you can change it to other universal models, or your own -personal models. You also have to provide your Google API key in order to use -it. +resources/models/snowboy.umdl, you can change it to other universal models, or +your own personal models. You also have to provide your Google API key in order +to use it. Note: Google is now moving to Google Cloud Speech API, so we will have to update the API query later. Usage: ./snowboy_googlevoice.pl [Hotword_Model] e.g.: ./snowboy_googlevoice.pl \ - abcdefghijklmnopqrstuvwxyzABC0123456789 resources/snowboy.umdl + abcdefghijklmnopqrstuvwxyzABC0123456789 resources/models/snowboy.umdl Allowed options: --language : Language for speech recognizer. (string, default="en") @@ -48,9 +48,9 @@ # Gets parameters. my $api_key = shift @ARGV; -my $model = shift @ARGV || 'resources/snowboy.umdl'; +my $model = shift @ARGV || 'resources/models/snowboy.umdl'; -if ($model eq 'resources/snowboy.umdl') { +if ($model eq 'resources/models/snowboy.umdl') { $hotword = "Snowboy"; } else { $hotword = "your hotword"; @@ -133,6 +133,7 @@ $sb = new Snowboy::SnowboyDetect('resources/common.res', $model); $sb->SetSensitivity('0.5'); $sb->SetAudioGain(1.0); +$sb->ApplyFrontend(0); # Running the detection forever. print "\n"; diff --git a/examples/Perl/snowboy_unit_test.pl b/examples/Perl/snowboy_unit_test.pl index 45cae4e4..b54cc306 100755 --- a/examples/Perl/snowboy_unit_test.pl +++ b/examples/Perl/snowboy_unit_test.pl @@ -12,10 +12,11 @@ close WAV; $sb = new Snowboy::SnowboyDetect('resources/common.res', - 'resources/snowboy.umdl'); + 'resources/models/snowboy.umdl'); $sb->SetSensitivity ("0.5"); $sb->SetAudioGain (1); +$sb->ApplyFrontend (0); print "==== SnowBoy object properties ====\n"; print "Sample Rate : ", $sb->SampleRate(), "\n"; diff --git a/examples/Python/__init__.py b/examples/Python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/Python/demo3.py b/examples/Python/demo3.py index 67fd7307..29ecec63 100644 --- a/examples/Python/demo3.py +++ b/examples/Python/demo3.py @@ -4,11 +4,11 @@ # Demo code for detecting hotword in a .wav file # Example Usage: -# $ python demo3.py resources/snowboy.wav resources/snowboy.umdl +# $ python demo3.py resources/snowboy.wav resources/models/snowboy.umdl # Should print: # Hotword Detected! # -# $ python demo3.py resources/ding.wav resources/snowboy.umdl +# $ python demo3.py resources/ding.wav resources/models/snowboy.umdl # Should print: # Hotword Not Detected! diff --git a/examples/Python/demo4.py b/examples/Python/demo4.py new file mode 100644 index 00000000..a59eedaa --- /dev/null +++ b/examples/Python/demo4.py @@ -0,0 +1,76 @@ +import snowboydecoder +import sys +import signal +import speech_recognition as sr +import os + +""" +This demo file shows you how to use the new_message_callback to interact with +the recorded audio after a keyword is spoken. It uses the speech recognition +library in order to convert the recorded audio into text. + +Information on installing the speech recognition library can be found at: +https://pypi.python.org/pypi/SpeechRecognition/ +""" + + +interrupted = False + + +def audioRecorderCallback(fname): + print "converting audio to text" + r = sr.Recognizer() + with sr.AudioFile(fname) as source: + audio = r.record(source) # read the entire audio file + # recognize speech using Google Speech Recognition + try: + # for testing purposes, we're just using the default API key + # to use another API key, use `r.recognize_google(audio, key="GOOGLE_SPEECH_RECOGNITION_API_KEY")` + # instead of `r.recognize_google(audio)` + print(r.recognize_google(audio)) + except sr.UnknownValueError: + print "Google Speech Recognition could not understand audio" + except sr.RequestError as e: + print "Could not request results from Google Speech Recognition service; {0}".format(e) + + os.remove(fname) + + + +def detectedCallback(): + sys.stdout.write("recording audio...") + sys.stdout.flush() + +def signal_handler(signal, frame): + global interrupted + interrupted = True + + +def interrupt_callback(): + global interrupted + return interrupted + +if len(sys.argv) == 1: + print "Error: need to specify model name" + print "Usage: python demo.py your.model" + sys.exit(-1) + +model = sys.argv[1] + +# capture SIGINT signal, e.g., Ctrl+C +signal.signal(signal.SIGINT, signal_handler) + +detector = snowboydecoder.HotwordDetector(model, sensitivity=0.38) +print "Listening... Press Ctrl+C to exit" + +# main loop +detector.start(detected_callback=detectedCallback, + audio_recorder_callback=audioRecorderCallback, + interrupt_check=interrupt_callback, + sleep_time=0.01) + +detector.terminate() + + + + diff --git a/examples/Python/demo_arecord.py b/examples/Python/demo_arecord.py new file mode 100644 index 00000000..c14bf155 --- /dev/null +++ b/examples/Python/demo_arecord.py @@ -0,0 +1,35 @@ +import snowboydecoder_arecord +import sys +import signal + +interrupted = False + + +def signal_handler(signal, frame): + global interrupted + interrupted = True + + +def interrupt_callback(): + global interrupted + return interrupted + +if len(sys.argv) == 1: + print("Error: need to specify model name") + print("Usage: python demo.py your.model") + sys.exit(-1) + +model = sys.argv[1] + +# capture SIGINT signal, e.g., Ctrl+C +signal.signal(signal.SIGINT, signal_handler) + +detector = snowboydecoder_arecord.HotwordDetector(model, sensitivity=0.5) +print('Listening... Press Ctrl+C to exit') + +# main loop +detector.start(detected_callback=snowboydecoder_arecord.play_audio_file, + interrupt_check=interrupt_callback, + sleep_time=0.03) + +detector.terminate() diff --git a/examples/Python/demo_threaded.py b/examples/Python/demo_threaded.py new file mode 100644 index 00000000..ded146c7 --- /dev/null +++ b/examples/Python/demo_threaded.py @@ -0,0 +1,47 @@ +import snowboythreaded +import sys +import signal +import time + +stop_program = False + +# This a demo that shows running Snowboy in another thread + + +def signal_handler(signal, frame): + global stop_program + stop_program = True + + +if len(sys.argv) == 1: + print("Error: need to specify model name") + print("Usage: python demo4.py your.model") + sys.exit(-1) + +model = sys.argv[1] + +# capture SIGINT signal, e.g., Ctrl+C +signal.signal(signal.SIGINT, signal_handler) + +# Initialize ThreadedDetector object and start the detection thread +threaded_detector = snowboythreaded.ThreadedDetector(model, sensitivity=0.5) +threaded_detector.start() + +print('Listening... Press Ctrl+C to exit') + +# main loop +threaded_detector.start_recog(sleep_time=0.03) + +# Let audio initialization happen before requesting input +time.sleep(1) + +# Do a simple task separate from the detection - addition of numbers +while not stop_program: + try: + num1 = int(raw_input("Enter the first number to add: ")) + num2 = int(raw_input("Enter the second number to add: ")) + print "Sum of number: {}".format(num1 + num2) + except ValueError: + print "You did not enter a number." + +threaded_detector.terminate() diff --git a/examples/Python/snowboydecoder.py b/examples/Python/snowboydecoder.py index e13781dd..5da9aff0 100644 --- a/examples/Python/snowboydecoder.py +++ b/examples/Python/snowboydecoder.py @@ -7,6 +7,8 @@ import wave import os import logging +from ctypes import * +from contextlib import contextmanager logging.basicConfig() logger = logging.getLogger("snowboy") @@ -17,6 +19,23 @@ DETECT_DING = os.path.join(TOP_DIR, "resources/ding.wav") DETECT_DONG = os.path.join(TOP_DIR, "resources/dong.wav") +def py_error_handler(filename, line, function, err, fmt): + pass + +ERROR_HANDLER_FUNC = CFUNCTYPE(None, c_char_p, c_int, c_char_p, c_int, c_char_p) + +c_error_handler = ERROR_HANDLER_FUNC(py_error_handler) + +@contextmanager +def no_alsa_error(): + try: + asound = cdll.LoadLibrary('libasound.so') + asound.snd_lib_error_set_handler(c_error_handler) + yield + asound.snd_lib_error_set_handler(None) + except: + yield + pass class RingBuffer(object): """Ring buffer to hold audio from PortAudio""" @@ -43,7 +62,8 @@ def play_audio_file(fname=DETECT_DING): """ ding_wav = wave.open(fname, 'rb') ding_data = ding_wav.readframes(ding_wav.getnframes()) - audio = pyaudio.PyAudio() + with no_alsa_error(): + audio = pyaudio.PyAudio() stream_out = audio.open( format=audio.get_format_from_width(ding_wav.getsampwidth()), channels=ding_wav.getnchannels(), @@ -68,11 +88,13 @@ class HotwordDetector(object): decoder. If an empty list is provided, then the default sensitivity in the model will be used. :param audio_gain: multiply input volume by this factor. + :param apply_frontend: applies the frontend processing algorithm if True. """ def __init__(self, decoder_model, resource=RESOURCE_FILE, sensitivity=[], - audio_gain=1): + audio_gain=1, + apply_frontend=False): def audio_callback(in_data, frame_count, time_info, status): self.ring_buffer.extend(in_data) @@ -90,6 +112,7 @@ def audio_callback(in_data, frame_count, time_info, status): self.detector = snowboydetect.SnowboyDetect( resource_filename=resource.encode(), model_str=model_str.encode()) self.detector.SetAudioGain(audio_gain) + self.detector.ApplyFrontend(apply_frontend) self.num_hotwords = self.detector.NumHotwords() if len(decoder_model) > 1 and len(sensitivity) == 1: @@ -104,7 +127,8 @@ def audio_callback(in_data, frame_count, time_info, status): self.ring_buffer = RingBuffer( self.detector.NumChannels() * self.detector.SampleRate() * 5) - self.audio = pyaudio.PyAudio() + with no_alsa_error(): + self.audio = pyaudio.PyAudio() self.stream_in = self.audio.open( input=True, output=False, format=self.audio.get_format_from_width( @@ -117,7 +141,10 @@ def audio_callback(in_data, frame_count, time_info, status): def start(self, detected_callback=play_audio_file, interrupt_check=lambda: False, - sleep_time=0.03): + sleep_time=0.03, + audio_recorder_callback=None, + silent_count_threshold=15, + recording_timeout=100): """ Start the voice detector. For every `sleep_time` second it checks the audio buffer for triggering keywords. If detected, then call @@ -132,6 +159,16 @@ def start(self, detected_callback=play_audio_file, :param interrupt_check: a function that returns True if the main loop needs to stop. :param float sleep_time: how much time in second every loop waits. + :param audio_recorder_callback: if specified, this will be called after + a keyword has been spoken and after the + phrase immediately after the keyword has + been recorded. The function will be + passed the name of the file where the + phrase was recorded. + :param silent_count_threshold: indicates how long silence must be heard + to mark the end of a phrase that is + being recorded. + :param recording_timeout: limits the maximum length of a recording. :return: None """ if interrupt_check(): @@ -150,6 +187,7 @@ def start(self, detected_callback=play_audio_file, logger.debug("detecting...") + state = "PASSIVE" while True: if interrupt_check(): logger.debug("detect voice break") @@ -159,20 +197,71 @@ def start(self, detected_callback=play_audio_file, time.sleep(sleep_time) continue - ans = self.detector.RunDetection(data) - if ans == -1: + status = self.detector.RunDetection(data) + if status == -1: logger.warning("Error initializing streams or reading audio data") - elif ans > 0: - message = "Keyword " + str(ans) + " detected at time: " - message += time.strftime("%Y-%m-%d %H:%M:%S", + + #small state machine to handle recording of phrase after keyword + if state == "PASSIVE": + if status > 0: #key word found + self.recordedData = [] + self.recordedData.append(data) + silentCount = 0 + recordingCount = 0 + message = "Keyword " + str(status) + " detected at time: " + message += time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())) - logger.info(message) - callback = detected_callback[ans-1] - if callback is not None: - callback() + logger.info(message) + callback = detected_callback[status-1] + if callback is not None: + callback() + + if audio_recorder_callback is not None: + state = "ACTIVE" + continue + + elif state == "ACTIVE": + stopRecording = False + if recordingCount > recording_timeout: + stopRecording = True + elif status == -2: #silence found + if silentCount > silent_count_threshold: + stopRecording = True + else: + silentCount = silentCount + 1 + elif status == 0: #voice found + silentCount = 0 + + if stopRecording == True: + fname = self.saveMessage() + audio_recorder_callback(fname) + state = "PASSIVE" + continue + + recordingCount = recordingCount + 1 + self.recordedData.append(data) logger.debug("finished.") + def saveMessage(self): + """ + Save the message stored in self.recordedData to a timestamped file. + """ + filename = 'output' + str(int(time.time())) + '.wav' + data = b''.join(self.recordedData) + + #use wave to save data + wf = wave.open(filename, 'wb') + wf.setnchannels(1) + wf.setsampwidth(self.audio.get_sample_size( + self.audio.get_format_from_width( + self.detector.BitsPerSample() / 8))) + wf.setframerate(self.detector.SampleRate()) + wf.writeframes(data) + wf.close() + logger.debug("finished saving: " + filename) + return filename + def terminate(self): """ Terminate audio stream. Users cannot call start() again to detect. diff --git a/examples/Python/snowboydecoder_arecord.py b/examples/Python/snowboydecoder_arecord.py new file mode 100644 index 00000000..efa3be46 --- /dev/null +++ b/examples/Python/snowboydecoder_arecord.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python + +import collections +import snowboydetect +import time +import wave +import os +import logging +import subprocess +import threading + +logging.basicConfig() +logger = logging.getLogger("snowboy") +logger.setLevel(logging.INFO) +TOP_DIR = os.path.dirname(os.path.abspath(__file__)) + +RESOURCE_FILE = os.path.join(TOP_DIR, "resources/common.res") +DETECT_DING = os.path.join(TOP_DIR, "resources/ding.wav") +DETECT_DONG = os.path.join(TOP_DIR, "resources/dong.wav") + + +class RingBuffer(object): + """Ring buffer to hold audio from audio capturing tool""" + def __init__(self, size = 4096): + self._buf = collections.deque(maxlen=size) + + def extend(self, data): + """Adds data to the end of buffer""" + self._buf.extend(data) + + def get(self): + """Retrieves data from the beginning of buffer and clears it""" + tmp = bytes(bytearray(self._buf)) + self._buf.clear() + return tmp + + +def play_audio_file(fname=DETECT_DING): + """Simple callback function to play a wave file. By default it plays + a Ding sound. + + :param str fname: wave file name + :return: None + """ + os.system("aplay " + fname + " > /dev/null 2>&1") + + +class HotwordDetector(object): + """ + Snowboy decoder to detect whether a keyword specified by `decoder_model` + exists in a microphone input stream. + + :param decoder_model: decoder model file path, a string or a list of strings + :param resource: resource file path. + :param sensitivity: decoder sensitivity, a float of a list of floats. + The bigger the value, the more senstive the + decoder. If an empty list is provided, then the + default sensitivity in the model will be used. + :param audio_gain: multiply input volume by this factor. + """ + def __init__(self, decoder_model, + resource=RESOURCE_FILE, + sensitivity=[], + audio_gain=1): + + tm = type(decoder_model) + ts = type(sensitivity) + if tm is not list: + decoder_model = [decoder_model] + if ts is not list: + sensitivity = [sensitivity] + model_str = ",".join(decoder_model) + + self.detector = snowboydetect.SnowboyDetect( + resource_filename=resource.encode(), model_str=model_str.encode()) + self.detector.SetAudioGain(audio_gain) + self.num_hotwords = self.detector.NumHotwords() + + if len(decoder_model) > 1 and len(sensitivity) == 1: + sensitivity = sensitivity*self.num_hotwords + if len(sensitivity) != 0: + assert self.num_hotwords == len(sensitivity), \ + "number of hotwords in decoder_model (%d) and sensitivity " \ + "(%d) does not match" % (self.num_hotwords, len(sensitivity)) + sensitivity_str = ",".join([str(t) for t in sensitivity]) + if len(sensitivity) != 0: + self.detector.SetSensitivity(sensitivity_str.encode()) + + self.ring_buffer = RingBuffer( + self.detector.NumChannels() * self.detector.SampleRate() * 5) + + def record_proc(self): + CHUNK = 2048 + RECORD_RATE = 16000 + cmd = 'arecord -q -r %d -f S16_LE' % RECORD_RATE + process = subprocess.Popen(cmd.split(' '), + stdout = subprocess.PIPE, + stderr = subprocess.PIPE) + wav = wave.open(process.stdout, 'rb') + while self.recording: + data = wav.readframes(CHUNK) + self.ring_buffer.extend(data) + process.terminate() + + def init_recording(self): + """ + Start a thread for spawning arecord process and reading its stdout + """ + self.recording = True + self.record_thread = threading.Thread(target = self.record_proc) + self.record_thread.start() + + def start(self, detected_callback=play_audio_file, + interrupt_check=lambda: False, + sleep_time=0.03): + """ + Start the voice detector. For every `sleep_time` second it checks the + audio buffer for triggering keywords. If detected, then call + corresponding function in `detected_callback`, which can be a single + function (single model) or a list of callback functions (multiple + models). Every loop it also calls `interrupt_check` -- if it returns + True, then breaks from the loop and return. + + :param detected_callback: a function or list of functions. The number of + items must match the number of models in + `decoder_model`. + :param interrupt_check: a function that returns True if the main loop + needs to stop. + :param float sleep_time: how much time in second every loop waits. + :return: None + """ + + self.init_recording() + + if interrupt_check(): + logger.debug("detect voice return") + return + + tc = type(detected_callback) + if tc is not list: + detected_callback = [detected_callback] + if len(detected_callback) == 1 and self.num_hotwords > 1: + detected_callback *= self.num_hotwords + + assert self.num_hotwords == len(detected_callback), \ + "Error: hotwords in your models (%d) do not match the number of " \ + "callbacks (%d)" % (self.num_hotwords, len(detected_callback)) + + logger.debug("detecting...") + + while True: + if interrupt_check(): + logger.debug("detect voice break") + break + data = self.ring_buffer.get() + if len(data) == 0: + time.sleep(sleep_time) + continue + + ans = self.detector.RunDetection(data) + if ans == -1: + logger.warning("Error initializing streams or reading audio data") + elif ans > 0: + message = "Keyword " + str(ans) + " detected at time: " + message += time.strftime("%Y-%m-%d %H:%M:%S", + time.localtime(time.time())) + logger.info(message) + callback = detected_callback[ans-1] + if callback is not None: + callback() + + logger.debug("finished.") + + def terminate(self): + """ + Terminate audio stream. Users cannot call start() again to detect. + :return: None + """ + self.recording = False + self.record_thread.join() + diff --git a/examples/Python/snowboythreaded.py b/examples/Python/snowboythreaded.py new file mode 100644 index 00000000..7932d482 --- /dev/null +++ b/examples/Python/snowboythreaded.py @@ -0,0 +1,96 @@ +import snowboydecoder +import threading +import Queue + + +class ThreadedDetector(threading.Thread): + """ + Wrapper class around detectors to run them in a separate thread + and provide methods to pause, resume, and modify detection + """ + + def __init__(self, models, **kwargs): + """ + Initialize Detectors object. **kwargs is for any __init__ keyword + arguments to be passed into HotWordDetector __init__() method. + """ + threading.Thread.__init__(self) + self.models = models + self.init_kwargs = kwargs + self.interrupted = True + self.commands = Queue.Queue() + self.vars_are_changed = True + self.detectors = None # Initialize when thread is run in self.run() + self.run_kwargs = None # Initialize when detectors start in self.start_recog() + + def initialize_detectors(self): + """ + Returns initialized Snowboy HotwordDetector objects + """ + self.detectors = snowboydecoder.HotwordDetector(self.models, **self.init_kwargs) + + def run(self): + """ + Runs in separate thread - waits on command to either run detectors + or terminate thread from commands queue + """ + try: + while True: + command = self.commands.get(True) + if command == "Start": + self.interrupted = False + if self.vars_are_changed: + # If there is an existing detector object, terminate it + if self.detectors is not None: + self.detectors.terminate() + self.initialize_detectors() + self.vars_are_changed = False + # Start detectors - blocks until interrupted by self.interrupted variable + self.detectors.start(interrupt_check=lambda: self.interrupted, **self.run_kwargs) + elif command == "Terminate": + # Program ending - terminate thread + break + finally: + if self.detectors is not None: + self.detectors.terminate() + + def start_recog(self, **kwargs): + """ + Starts recognition in thread. Accepts kwargs to pass into the + HotWordDetector.start() method, but does not accept interrupt_callback, + as that is already set up. + """ + assert "interrupt_check" not in kwargs, \ + "Cannot set interrupt_check argument. To interrupt detectors, use Detectors.pause_recog() instead" + self.run_kwargs = kwargs + self.commands.put("Start") + + def pause_recog(self): + """ + Halts recognition in thread. + """ + self.interrupted = True + + def terminate(self): + """ + Terminates recognition thread - called when program terminates + """ + self.pause_recog() + self.commands.put("Terminate") + + def is_running(self): + return not self.interrupted + + def change_models(self, models): + if self.is_running(): + print("Models will be changed after restarting detectors.") + if self.models != models: + self.models = models + self.vars_are_changed = True + + def change_sensitivity(self, sensitivity): + if self.is_running(): + print("Sensitivity will be changed after restarting detectors.") + if self.init_kwargs['sensitivity'] != sensitivity: + self.init_kwargs['sensitivity'] = sensitivity + self.vars_are_changed = True diff --git a/examples/Python3/_snowboydetect.so b/examples/Python3/_snowboydetect.so new file mode 120000 index 00000000..447c5831 --- /dev/null +++ b/examples/Python3/_snowboydetect.so @@ -0,0 +1 @@ +../../swig/Python3/_snowboydetect.so \ No newline at end of file diff --git a/examples/Python3/demo.py b/examples/Python3/demo.py new file mode 100644 index 00000000..305b8b62 --- /dev/null +++ b/examples/Python3/demo.py @@ -0,0 +1,35 @@ +import snowboydecoder +import sys +import signal + +interrupted = False + + +def signal_handler(signal, frame): + global interrupted + interrupted = True + + +def interrupt_callback(): + global interrupted + return interrupted + +if len(sys.argv) == 1: + print("Error: need to specify model name") + print("Usage: python demo.py your.model") + sys.exit(-1) + +model = sys.argv[1] + +# capture SIGINT signal, e.g., Ctrl+C +signal.signal(signal.SIGINT, signal_handler) + +detector = snowboydecoder.HotwordDetector(model, sensitivity=0.5) +print('Listening... Press Ctrl+C to exit') + +# main loop +detector.start(detected_callback=snowboydecoder.play_audio_file, + interrupt_check=interrupt_callback, + sleep_time=0.03) + +detector.terminate() diff --git a/examples/Python3/demo2.py b/examples/Python3/demo2.py new file mode 100644 index 00000000..fab9b981 --- /dev/null +++ b/examples/Python3/demo2.py @@ -0,0 +1,41 @@ +import snowboydecoder +import sys +import signal + +# Demo code for listening to two hotwords at the same time + +interrupted = False + + +def signal_handler(signal, frame): + global interrupted + interrupted = True + + +def interrupt_callback(): + global interrupted + return interrupted + +if len(sys.argv) != 3: + print("Error: need to specify 2 model names") + print("Usage: python demo.py 1st.model 2nd.model") + sys.exit(-1) + +models = sys.argv[1:] + +# capture SIGINT signal, e.g., Ctrl+C +signal.signal(signal.SIGINT, signal_handler) + +sensitivity = [0.5]*len(models) +detector = snowboydecoder.HotwordDetector(models, sensitivity=sensitivity) +callbacks = [lambda: snowboydecoder.play_audio_file(snowboydecoder.DETECT_DING), + lambda: snowboydecoder.play_audio_file(snowboydecoder.DETECT_DONG)] +print('Listening... Press Ctrl+C to exit') + +# main loop +# make sure you have the same numbers of callbacks and models +detector.start(detected_callback=callbacks, + interrupt_check=interrupt_callback, + sleep_time=0.03) + +detector.terminate() diff --git a/examples/Python3/demo3.py b/examples/Python3/demo3.py new file mode 100644 index 00000000..29ecec63 --- /dev/null +++ b/examples/Python3/demo3.py @@ -0,0 +1,40 @@ +import snowboydecoder +import sys +import wave + +# Demo code for detecting hotword in a .wav file +# Example Usage: +# $ python demo3.py resources/snowboy.wav resources/models/snowboy.umdl +# Should print: +# Hotword Detected! +# +# $ python demo3.py resources/ding.wav resources/models/snowboy.umdl +# Should print: +# Hotword Not Detected! + + +if len(sys.argv) != 3: + print("Error: need to specify wave file name and model name") + print("Usage: python demo3.py wave_file model_file") + sys.exit(-1) + +wave_file = sys.argv[1] +model_file = sys.argv[2] + +f = wave.open(wave_file) +assert f.getnchannels() == 1, "Error: Snowboy only supports 1 channel of audio (mono, not stereo)" +assert f.getframerate() == 16000, "Error: Snowboy only supports 16K sampling rate" +assert f.getsampwidth() == 2, "Error: Snowboy only supports 16bit per sample" +data = f.readframes(f.getnframes()) +f.close() + +sensitivity = 0.5 +detection = snowboydecoder.HotwordDetector(model_file, sensitivity=sensitivity) + +ans = detection.detector.RunDetection(data) + +if ans == 1: + print('Hotword Detected!') +else: + print('Hotword Not Detected!') + diff --git a/examples/Python3/demo4.py b/examples/Python3/demo4.py new file mode 100644 index 00000000..dc52119e --- /dev/null +++ b/examples/Python3/demo4.py @@ -0,0 +1,75 @@ +import snowboydecoder +import sys +import signal +import speech_recognition as sr +import os + +""" +This demo file shows you how to use the new_message_callback to interact with +the recorded audio after a keyword is spoken. It uses the speech recognition +library in order to convert the recorded audio into text. + +Information on installing the speech recognition library can be found at: +https://pypi.python.org/pypi/SpeechRecognition/ +""" + + +interrupted = False + + +def audioRecorderCallback(fname): + print("converting audio to text") + r = sr.Recognizer() + with sr.AudioFile(fname) as source: + audio = r.record(source) # read the entire audio file + # recognize speech using Google Speech Recognition + try: + # for testing purposes, we're just using the default API key + # to use another API key, use `r.recognize_google(audio, key="GOOGLE_SPEECH_RECOGNITION_API_KEY")` + # instead of `r.recognize_google(audio)` + print(r.recognize_google(audio)) + except sr.UnknownValueError: + print("Google Speech Recognition could not understand audio") + except sr.RequestError as e: + print("Could not request results from Google Speech Recognition service; {0}".format(e)) + + os.remove(fname) + + + +def detectedCallback(): + print('recording audio...', end='', flush=True) + +def signal_handler(signal, frame): + global interrupted + interrupted = True + + +def interrupt_callback(): + global interrupted + return interrupted + +if len(sys.argv) == 1: + print("Error: need to specify model name") + print("Usage: python demo.py your.model") + sys.exit(-1) + +model = sys.argv[1] + +# capture SIGINT signal, e.g., Ctrl+C +signal.signal(signal.SIGINT, signal_handler) + +detector = snowboydecoder.HotwordDetector(model, sensitivity=0.38) +print('Listening... Press Ctrl+C to exit') + +# main loop +detector.start(detected_callback=detectedCallback, + audio_recorder_callback=audioRecorderCallback, + interrupt_check=interrupt_callback, + sleep_time=0.01) + +detector.terminate() + + + + diff --git a/examples/Python3/requirements.txt b/examples/Python3/requirements.txt new file mode 120000 index 00000000..d8614c7d --- /dev/null +++ b/examples/Python3/requirements.txt @@ -0,0 +1 @@ +../Python/requirements.txt \ No newline at end of file diff --git a/examples/Python3/resources b/examples/Python3/resources new file mode 120000 index 00000000..81bd1c59 --- /dev/null +++ b/examples/Python3/resources @@ -0,0 +1 @@ +../../resources/ \ No newline at end of file diff --git a/examples/Python3/snowboydecoder.py b/examples/Python3/snowboydecoder.py new file mode 100644 index 00000000..34ee26fb --- /dev/null +++ b/examples/Python3/snowboydecoder.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python + +import collections +import pyaudio +from . import snowboydetect +import time +import wave +import os +import logging +from ctypes import * +from contextlib import contextmanager + +logging.basicConfig() +logger = logging.getLogger("snowboy") +logger.setLevel(logging.INFO) +TOP_DIR = os.path.dirname(os.path.abspath(__file__)) + +RESOURCE_FILE = os.path.join(TOP_DIR, "resources/common.res") +DETECT_DING = os.path.join(TOP_DIR, "resources/ding.wav") +DETECT_DONG = os.path.join(TOP_DIR, "resources/dong.wav") + +def py_error_handler(filename, line, function, err, fmt): + pass + +ERROR_HANDLER_FUNC = CFUNCTYPE(None, c_char_p, c_int, c_char_p, c_int, c_char_p) + +c_error_handler = ERROR_HANDLER_FUNC(py_error_handler) + +@contextmanager +def no_alsa_error(): + try: + asound = cdll.LoadLibrary('libasound.so') + asound.snd_lib_error_set_handler(c_error_handler) + yield + asound.snd_lib_error_set_handler(None) + except: + yield + pass + +class RingBuffer(object): + """Ring buffer to hold audio from PortAudio""" + + def __init__(self, size=4096): + self._buf = collections.deque(maxlen=size) + + def extend(self, data): + """Adds data to the end of buffer""" + self._buf.extend(data) + + def get(self): + """Retrieves data from the beginning of buffer and clears it""" + tmp = bytes(bytearray(self._buf)) + self._buf.clear() + return tmp + + +def play_audio_file(fname=DETECT_DING): + """Simple callback function to play a wave file. By default it plays + a Ding sound. + + :param str fname: wave file name + :return: None + """ + ding_wav = wave.open(fname, 'rb') + ding_data = ding_wav.readframes(ding_wav.getnframes()) + with no_alsa_error(): + audio = pyaudio.PyAudio() + stream_out = audio.open( + format=audio.get_format_from_width(ding_wav.getsampwidth()), + channels=ding_wav.getnchannels(), + rate=ding_wav.getframerate(), input=False, output=True) + stream_out.start_stream() + stream_out.write(ding_data) + time.sleep(0.2) + stream_out.stop_stream() + stream_out.close() + audio.terminate() + + +class HotwordDetector(object): + """ + Snowboy decoder to detect whether a keyword specified by `decoder_model` + exists in a microphone input stream. + + :param decoder_model: decoder model file path, a string or a list of strings + :param resource: resource file path. + :param sensitivity: decoder sensitivity, a float of a list of floats. + The bigger the value, the more senstive the + decoder. If an empty list is provided, then the + default sensitivity in the model will be used. + :param audio_gain: multiply input volume by this factor. + :param apply_frontend: applies the frontend processing algorithm if True. + """ + + def __init__(self, decoder_model, + resource=RESOURCE_FILE, + sensitivity=[], + audio_gain=1, + apply_frontend=False): + + tm = type(decoder_model) + ts = type(sensitivity) + if tm is not list: + decoder_model = [decoder_model] + if ts is not list: + sensitivity = [sensitivity] + model_str = ",".join(decoder_model) + + self.detector = snowboydetect.SnowboyDetect( + resource_filename=resource.encode(), model_str=model_str.encode()) + self.detector.SetAudioGain(audio_gain) + self.detector.ApplyFrontend(apply_frontend) + self.num_hotwords = self.detector.NumHotwords() + + if len(decoder_model) > 1 and len(sensitivity) == 1: + sensitivity = sensitivity * self.num_hotwords + if len(sensitivity) != 0: + assert self.num_hotwords == len(sensitivity), \ + "number of hotwords in decoder_model (%d) and sensitivity " \ + "(%d) does not match" % (self.num_hotwords, len(sensitivity)) + sensitivity_str = ",".join([str(t) for t in sensitivity]) + if len(sensitivity) != 0: + self.detector.SetSensitivity(sensitivity_str.encode()) + + self.ring_buffer = RingBuffer( + self.detector.NumChannels() * self.detector.SampleRate() * 5) + + def start(self, detected_callback=play_audio_file, + interrupt_check=lambda: False, + sleep_time=0.03, + audio_recorder_callback=None, + silent_count_threshold=15, + recording_timeout=100): + """ + Start the voice detector. For every `sleep_time` second it checks the + audio buffer for triggering keywords. If detected, then call + corresponding function in `detected_callback`, which can be a single + function (single model) or a list of callback functions (multiple + models). Every loop it also calls `interrupt_check` -- if it returns + True, then breaks from the loop and return. + + :param detected_callback: a function or list of functions. The number of + items must match the number of models in + `decoder_model`. + :param interrupt_check: a function that returns True if the main loop + needs to stop. + :param float sleep_time: how much time in second every loop waits. + :param audio_recorder_callback: if specified, this will be called after + a keyword has been spoken and after the + phrase immediately after the keyword has + been recorded. The function will be + passed the name of the file where the + phrase was recorded. + :param silent_count_threshold: indicates how long silence must be heard + to mark the end of a phrase that is + being recorded. + :param recording_timeout: limits the maximum length of a recording. + :return: None + """ + self._running = True + + def audio_callback(in_data, frame_count, time_info, status): + self.ring_buffer.extend(in_data) + play_data = chr(0) * len(in_data) + return play_data, pyaudio.paContinue + + with no_alsa_error(): + self.audio = pyaudio.PyAudio() + self.stream_in = self.audio.open( + input=True, output=False, + format=self.audio.get_format_from_width( + self.detector.BitsPerSample() / 8), + channels=self.detector.NumChannels(), + rate=self.detector.SampleRate(), + frames_per_buffer=2048, + stream_callback=audio_callback) + + if interrupt_check(): + logger.debug("detect voice return") + return + + tc = type(detected_callback) + if tc is not list: + detected_callback = [detected_callback] + if len(detected_callback) == 1 and self.num_hotwords > 1: + detected_callback *= self.num_hotwords + + assert self.num_hotwords == len(detected_callback), \ + "Error: hotwords in your models (%d) do not match the number of " \ + "callbacks (%d)" % (self.num_hotwords, len(detected_callback)) + + logger.debug("detecting...") + + state = "PASSIVE" + while self._running is True: + if interrupt_check(): + logger.debug("detect voice break") + break + data = self.ring_buffer.get() + if len(data) == 0: + time.sleep(sleep_time) + continue + + status = self.detector.RunDetection(data) + if status == -1: + logger.warning("Error initializing streams or reading audio data") + + #small state machine to handle recording of phrase after keyword + if state == "PASSIVE": + if status > 0: #key word found + self.recordedData = [] + self.recordedData.append(data) + silentCount = 0 + recordingCount = 0 + message = "Keyword " + str(status) + " detected at time: " + message += time.strftime("%Y-%m-%d %H:%M:%S", + time.localtime(time.time())) + logger.info(message) + callback = detected_callback[status-1] + if callback is not None: + callback() + + if audio_recorder_callback is not None: + state = "ACTIVE" + continue + + elif state == "ACTIVE": + stopRecording = False + if recordingCount > recording_timeout: + stopRecording = True + elif status == -2: #silence found + if silentCount > silent_count_threshold: + stopRecording = True + else: + silentCount = silentCount + 1 + elif status == 0: #voice found + silentCount = 0 + + if stopRecording == True: + fname = self.saveMessage() + audio_recorder_callback(fname) + state = "PASSIVE" + continue + + recordingCount = recordingCount + 1 + self.recordedData.append(data) + + logger.debug("finished.") + + def saveMessage(self): + """ + Save the message stored in self.recordedData to a timestamped file. + """ + filename = 'output' + str(int(time.time())) + '.wav' + data = b''.join(self.recordedData) + + #use wave to save data + wf = wave.open(filename, 'wb') + wf.setnchannels(1) + wf.setsampwidth(self.audio.get_sample_size( + self.audio.get_format_from_width( + self.detector.BitsPerSample() / 8))) + wf.setframerate(self.detector.SampleRate()) + wf.writeframes(data) + wf.close() + logger.debug("finished saving: " + filename) + return filename + + def terminate(self): + """ + Terminate audio stream. Users can call start() again to detect. + :return: None + """ + self.stream_in.stop_stream() + self.stream_in.close() + self.audio.terminate() + self._running = False diff --git a/examples/Python3/snowboydetect.py b/examples/Python3/snowboydetect.py new file mode 120000 index 00000000..9b6bc3ec --- /dev/null +++ b/examples/Python3/snowboydetect.py @@ -0,0 +1 @@ +../../swig/Python3/snowboydetect.py \ No newline at end of file diff --git a/examples/iOS/Obj-C/SnowboyTest/ViewController.mm b/examples/iOS/Obj-C/SnowboyTest/ViewController.mm index 4497ec50..16467f6e 100644 --- a/examples/iOS/Obj-C/SnowboyTest/ViewController.mm +++ b/examples/iOS/Obj-C/SnowboyTest/ViewController.mm @@ -31,6 +31,7 @@ - (void)initSnowboy { std::string([[[NSBundle mainBundle]pathForResource:@"alexa" ofType:@"umdl"] UTF8String])); _snowboyDetect->SetSensitivity("0.5"); _snowboyDetect->SetAudioGain(1.0); + _snowboyDetect->ApplyFrontend(false); } - (void) initMic { diff --git a/examples/iOS/Swift3/SnowboyTest/SnowboyWrapper.mm b/examples/iOS/Swift3/SnowboyTest/SnowboyWrapper.mm index 53329aed..9ac53637 100644 --- a/examples/iOS/Swift3/SnowboyTest/SnowboyWrapper.mm +++ b/examples/iOS/Swift3/SnowboyTest/SnowboyWrapper.mm @@ -43,7 +43,9 @@ -(int)runDetection:(NSArray*)data length:(int)length dataArray[i] = [[data objectAtIndex:i] floatValue]; } - return snowboy->RunDetection(dataArray, length); + int detected = snowboy->RunDetection(dataArray, length); + free(dataArray); + return detected; } -(bool)reset diff --git a/include/snowboy-detect.h b/include/snowboy-detect.h index ab903e10..95e06c59 100644 --- a/include/snowboy-detect.h +++ b/include/snowboy-detect.h @@ -13,6 +13,7 @@ namespace snowboy { // Forward declaration. struct WaveHeader; class PipelineDetect; +class PipelineVad; //////////////////////////////////////////////////////////////////////////////// // @@ -97,6 +98,13 @@ class SnowboyDetect { // hotword. void SetSensitivity(const std::string& sensitivity_str); + // Similar to the sensitivity setting above. When set higher than the above + // sensitivity, the algorithm automatically chooses between the normal + // sensitivity set above and the higher sensitivity set here, to maximize the + // performance. By default, it is not set, which means the algorithm will + // stick with the sensitivity set above. + void SetHighSensitivity(const std::string& high_sensitivity_str); + // Returns the sensitivity string for the current hotwords. std::string GetSensitivity() const; @@ -116,7 +124,13 @@ class SnowboyDetect { int NumHotwords() const; // If is true, then apply frontend audio processing; - // otherwise turns the audio processing off. + // otherwise turns the audio processing off. Frontend audio processing + // includes algorithms such as automatic gain control (AGC), noise suppression + // (NS) and so on. Generally adding frontend audio processing helps the + // performance, but if the model is not trained with frontend audio + // processing, it may decrease the performance. The general rule of thumb is: + // 1. For personal models, set it to false. + // 2. For universal models, follow the instruction of each published model void ApplyFrontend(const bool apply_frontend); // Returns the required sampling rate, number of channels and bits per sample @@ -133,6 +147,80 @@ class SnowboyDetect { std::unique_ptr detect_pipeline_; }; +//////////////////////////////////////////////////////////////////////////////// +// +// SnowboyVad class interface. +// +//////////////////////////////////////////////////////////////////////////////// +class SnowboyVad { + public: + // Constructor that takes a resource file. It shares the same resource file + // with SnowboyDetect. + SnowboyVad(const std::string& resource_filename); + + // Resets the VAD. + bool Reset(); + + // Runs the VAD algorithm. Supported audio format is WAVE (with linear PCM, + // 8-bits unsigned integer, 16-bits signed integer or 32-bits signed integer). + // See SampleRate(), NumChannels() and BitsPerSample() for the required + // sampling rate, number of channels and bits per sample values. You are + // supposed to provide a small chunk of data (e.g., 0.1 second) each time you + // call RunDetection(). Larger chunk usually leads to longer delay, but less + // CPU usage. + // + // Definition of return values: + // -2: Silence. + // -1: Error. + // 0: Non-silence. + // + // @param [in] data Small chunk of data to be detected. See + // above for the supported data format. + // @param [in] is_end Set it to true if it is the end of a + // utterance or file. + int RunVad(const std::string& data, bool is_end = false); + + // Various versions of RunVad() that take different format of audio. If + // NumChannels() > 1, e.g., NumChannels() == 2, then the array is as follows: + // + // d1c1, d1c2, d2c1, d2c2, d3c1, d3c2, ..., dNc1, dNc2 + // + // where d1c1 means data point 1 of channel 1. + // + // @param [in] data Small chunk of data to be detected. See + // above for the supported data format. + // @param [in] array_length Length of the data array. + // @param [in] is_end Set it to true if it is the end of a + // utterance or file. + int RunVad(const float* const data, + const int array_length, bool is_end = false); + int RunVad(const int16_t* const data, + const int array_length, bool is_end = false); + int RunVad(const int32_t* const data, + const int array_length, bool is_end = false); + + // Applied a fixed gain to the input audio. In case you have a very weak + // microphone, you can use this function to boost input audio level. + void SetAudioGain(const float audio_gain); + + // If is true, then apply frontend audio processing; + // otherwise turns the audio processing off. + void ApplyFrontend(const bool apply_frontend); + + // Returns the required sampling rate, number of channels and bits per sample + // values for the audio data. You should use this information to set up your + // audio capturing interface. + int SampleRate() const; + int NumChannels() const; + int BitsPerSample() const; + + ~SnowboyVad(); + + private: + std::unique_ptr wave_header_; + std::unique_ptr vad_pipeline_; +}; + } // namespace snowboy #endif // SNOWBOY_INCLUDE_SNOWBOY_DETECT_H_ diff --git a/lib/aarch64-ubuntu1604/libsnowboy-detect.a b/lib/aarch64-ubuntu1604/libsnowboy-detect.a new file mode 100644 index 00000000..6b8ec973 Binary files /dev/null and b/lib/aarch64-ubuntu1604/libsnowboy-detect.a differ diff --git a/lib/android/armv7a/libsnowboy-detect.a b/lib/android/armv7a/libsnowboy-detect.a index ace75615..112bc12f 100644 Binary files a/lib/android/armv7a/libsnowboy-detect.a and b/lib/android/armv7a/libsnowboy-detect.a differ diff --git a/lib/android/armv8-aarch64/libsnowboy-detect.a b/lib/android/armv8-aarch64/libsnowboy-detect.a new file mode 100644 index 00000000..e8c4f197 Binary files /dev/null and b/lib/android/armv8-aarch64/libsnowboy-detect.a differ diff --git a/lib/edison/libsnowboy-detect.a b/lib/edison/libsnowboy-detect.a deleted file mode 100644 index 3f78ab14..00000000 Binary files a/lib/edison/libsnowboy-detect.a and /dev/null differ diff --git a/lib/fedora25-armv7/libsnowboy-detect.a b/lib/fedora25-armv7/libsnowboy-detect.a deleted file mode 100644 index 24f8ae4b..00000000 Binary files a/lib/fedora25-armv7/libsnowboy-detect.a and /dev/null differ diff --git a/lib/ios/libsnowboy-detect.a b/lib/ios/libsnowboy-detect.a index 6be483de..345832cd 100644 Binary files a/lib/ios/libsnowboy-detect.a and b/lib/ios/libsnowboy-detect.a differ diff --git a/lib/node/SnowboyDetectNative.d.ts b/lib/node/SnowboyDetectNative.d.ts index 97a2dd0a..d0714416 100644 --- a/lib/node/SnowboyDetectNative.d.ts +++ b/lib/node/SnowboyDetectNative.d.ts @@ -3,6 +3,7 @@ interface SnowboyDetectNativeInterface { Reset(): boolean; RunDetection(audioData: Buffer): number; SetSensitivity(sensitivity: string): void; + SetHighSensitivity(highSensitivity: string): void; GetSensitivity(): string; SetAudioGain(audioGain: number): void; UpdateModel(): void; @@ -10,4 +11,5 @@ interface SnowboyDetectNativeInterface { SampleRate(): number; NumChannels(): number; BitsPerSample(): number; + ApplyFrontend(applyFrontend: boolean): void; } diff --git a/lib/node/index.ts b/lib/node/index.ts index 7ba7e5b1..409e4690 100644 --- a/lib/node/index.ts +++ b/lib/node/index.ts @@ -17,7 +17,7 @@ enum ModelType { UMDL } -interface HotwordModel { +export interface HotwordModel { file: string; sensitivity?: string; hotwords: string | Array; @@ -29,16 +29,19 @@ interface HotwordModelsInterface { numHotwords(): number; } -interface DetectorOptions { +export interface DetectorOptions { resource: string; models: HotwordModels; audioGain?: number; + applyFrontend?: boolean; + highSensitivity?: string; } -interface SnowboyDetectInterface { +export interface SnowboyDetectInterface { reset(): boolean; runDetection(buffer: Buffer): number; setSensitivity(sensitivity: string): void; + setHighSensitivity(highSensitivity: string): void; getSensitivity(): string; setAudioGain(gain: number): void; updateModel(): void; @@ -116,6 +119,14 @@ export class SnowboyDetect extends stream.Writable implements SnowboyDetectInter if (options.audioGain) { this.nativeInstance.SetAudioGain(options.audioGain); } + + if (options.applyFrontend) { + this.nativeInstance.ApplyFrontend(options.applyFrontend); + } + + if (options.highSensitivity) { + this.nativeInstance.SetHighSensitivity(options.highSensitivity); + } } reset(): boolean { @@ -124,7 +135,7 @@ export class SnowboyDetect extends stream.Writable implements SnowboyDetectInter runDetection(buffer: Buffer): number { const index = this.nativeInstance.RunDetection(buffer); - this.processDetectionResult(index); + this.processDetectionResult(index, buffer); return index; } @@ -132,6 +143,10 @@ export class SnowboyDetect extends stream.Writable implements SnowboyDetectInter this.nativeInstance.SetSensitivity(sensitivity); } + setHighSensitivity(highSensitivity: string): void { + this.nativeInstance.SetHighSensitivity(highSensitivity); + } + getSensitivity(): string { return this.nativeInstance.GetSensitivity(); } @@ -163,11 +178,11 @@ export class SnowboyDetect extends stream.Writable implements SnowboyDetectInter // Stream implementation _write(chunk: Buffer, encoding: string, callback: Function) { const index = this.nativeInstance.RunDetection(chunk); - this.processDetectionResult(index); + this.processDetectionResult(index, chunk); return callback(); } - private processDetectionResult(index: number): void { + private processDetectionResult(index: number, buffer: Buffer): void { switch (index) { case DetectionResult.ERROR: this.emit('error'); @@ -178,12 +193,12 @@ export class SnowboyDetect extends stream.Writable implements SnowboyDetectInter break; case DetectionResult.SOUND: - this.emit('sound'); + this.emit('sound', buffer); break; default: const hotword = this.models.lookup(index); - this.emit('hotword', index, hotword); + this.emit('hotword', index, hotword, buffer); break; } } diff --git a/lib/osx/libsnowboy-detect.a b/lib/osx/libsnowboy-detect.a index eca070ec..b0736ada 100644 Binary files a/lib/osx/libsnowboy-detect.a and b/lib/osx/libsnowboy-detect.a differ diff --git a/lib/pine64/libsnowboy-detect.a b/lib/pine64/libsnowboy-detect.a deleted file mode 100755 index ef94636f..00000000 Binary files a/lib/pine64/libsnowboy-detect.a and /dev/null differ diff --git a/lib/rpi/libsnowboy-detect.a b/lib/rpi/libsnowboy-detect.a index ba6ab216..df04d29d 100644 Binary files a/lib/rpi/libsnowboy-detect.a and b/lib/rpi/libsnowboy-detect.a differ diff --git a/lib/ubuntu64/libsnowboy-detect.a b/lib/ubuntu64/libsnowboy-detect.a index bc5b9991..6aaad1a3 100644 Binary files a/lib/ubuntu64/libsnowboy-detect.a and b/lib/ubuntu64/libsnowboy-detect.a differ diff --git a/package.json b/package.json index fae4e763..42489798 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,9 @@ { "name": "snowboy", - "version": "1.2.0", + "version": "1.3.1", "description": "Snowboy is a customizable hotword detection engine", "main": "lib/node/index.js", + "typings": "lib/node/index.d.ts", "binary": { "module_name": "snowboy", "module_path": "./lib/node/binding/{configuration}/{node_abi}-{platform}-{arch}/", @@ -11,7 +12,6 @@ "host": "https://snowboy-release-node.s3-us-west-2.amazonaws.com" }, "scripts": { - "preinstall": "npm install node-pre-gyp", "install": "node-pre-gyp install --fallback-to-build", "test": "node index.js", "prepublish": "tsc --listFiles" diff --git a/resources/alexa.umdl b/resources/alexa.umdl deleted file mode 100644 index 0d9db6f2..00000000 Binary files a/resources/alexa.umdl and /dev/null differ diff --git a/resources/alexa/SnowboyAlexaDemo.apk b/resources/alexa/SnowboyAlexaDemo.apk index f3f61fa5..df55e427 100644 Binary files a/resources/alexa/SnowboyAlexaDemo.apk and b/resources/alexa/SnowboyAlexaDemo.apk differ diff --git a/resources/alexa/alexa-avs-sample-app/avs-kittai.patch b/resources/alexa/alexa-avs-sample-app/avs-kittai.patch new file mode 100644 index 00000000..d1c2b2b3 --- /dev/null +++ b/resources/alexa/alexa-avs-sample-app/avs-kittai.patch @@ -0,0 +1,72 @@ +diff -Naur avs-sensory/pi.sh avs-kitt/pi.sh +--- pi.sh 2019-05-03 18:09:18.063849909 -0700 ++++ pi.sh 2019-05-03 18:09:39.273744305 -0700 +@@ -20,18 +20,19 @@ + + SOUND_CONFIG="$HOME/.asoundrc" + START_SCRIPT="$INSTALL_BASE/startsample.sh" +-CMAKE_PLATFORM_SPECIFIC=(-DSENSORY_KEY_WORD_DETECTOR=ON \ ++CMAKE_PLATFORM_SPECIFIC=(-DKITTAI_KEY_WORD_DETECTOR=ON \ + -DGSTREAMER_MEDIA_PLAYER=ON -DPORTAUDIO=ON \ + -DPORTAUDIO_LIB_PATH="$THIRD_PARTY_PATH/portaudio/lib/.libs/libportaudio.$LIB_SUFFIX" \ + -DPORTAUDIO_INCLUDE_DIR="$THIRD_PARTY_PATH/portaudio/include" \ +- -DSENSORY_KEY_WORD_DETECTOR_LIB_PATH=$THIRD_PARTY_PATH/alexa-rpi/lib/libsnsr.a \ +- -DSENSORY_KEY_WORD_DETECTOR_INCLUDE_DIR=$THIRD_PARTY_PATH/alexa-rpi/include) ++ -DKITTAI_KEY_WORD_DETECTOR_LIB_PATH=$THIRD_PARTY_PATH/snowboy/lib/rpi/libsnowboy-detect.a \ ++ -DKITTAI_KEY_WORD_DETECTOR_INCLUDE_DIR=$THIRD_PARTY_PATH/snowboy/include \ ++ -DCMAKE_BUILD_TYPE=MINSIZEREL) + + GSTREAMER_AUDIO_SINK="alsasink" + + install_dependencies() { + sudo apt-get update +- sudo apt-get -y install git gcc cmake build-essential libsqlite3-dev libcurl4-openssl-dev libssl-dev libfaad-dev libsoup2.4-dev libgcrypt20-dev libgstreamer-plugins-bad1.0-dev gstreamer1.0-plugins-good libasound2-dev sox gedit vim python3-pip ++ sudo apt-get -y install git gcc cmake build-essential libsqlite3-dev libcurl4-openssl-dev libssl-dev libfaad-dev libsoup2.4-dev libgcrypt20-dev libgstreamer-plugins-bad1.0-dev gstreamer1.0-plugins-good libasound2-dev sox gedit vim python3-pip python-pip libatlas-base-dev + pip install flask commentjson + } + +@@ -64,18 +65,18 @@ + build_kwd_engine() { +- #get sensory and build ++ #get kittai and build + echo +- echo "==============> CLONING AND BUILDING SENSORY ==============" ++ echo "==============> CLONING AND BUILDING KITTAI ==============" + echo + + cd $THIRD_PARTY_PATH +- git clone git://github.com/Sensory/alexa-rpi.git +- bash ./alexa-rpi/bin/license.sh ++ git clone https://github.com/Kitt-AI/snowboy.git ++ cp snowboy/resources/alexa/alexa-avs-sample-app/alexa.umdl snowboy/resources/alexa.umdl + } + + generate_start_script() { + cat << EOF > "$START_SCRIPT" + cd "$BUILD_PATH/SampleApp/src" + +- ./SampleApp "$OUTPUT_CONFIG_FILE" "$THIRD_PARTY_PATH/alexa-rpi/models" DEBUG9 ++ ./SampleApp "$OUTPUT_CONFIG_FILE" "$THIRD_PARTY_PATH/snowboy/resources" DEBUG9 + EOF + } +diff -Naur avs-sensory/setup.sh avs-kitt/setup.sh +--- setup.sh 2019-05-03 18:09:24.383818365 -0700 ++++ setup.sh 2019-05-03 18:09:35.193764563 -0700 +@@ -50,6 +50,8 @@ + LIB_SUFFIX="a" + ANDROID_CONFIG_FILE="" + ++BUILDTYPE="MINSIZEREL" ++ + # Default device serial number if nothing is specified + DEVICE_SERIAL_NUMBER="123456" + +@@ -242,7 +244,7 @@ + cmake "$SOURCE_PATH/avs-device-sdk" \ + -DCMAKE_BUILD_TYPE=DEBUG \ + "${CMAKE_PLATFORM_SPECIFIC[@]}" +- ++ sed -E -i "s:CXX_PLATFORM_DEPENDENT_FLAGS_"$BUILDTYPE"\s+\"(.*)\":CXX_PLATFORM_DEPENDENT_FLAGS_"$BUILDTYPE" \"\1 -D_GLIBCXX_USE_CXX11_ABI=0 -pg\":" ../avs-device-sdk/build/cmake/BuildOptions.cmake + cd $BUILD_PATH + make SampleApp -j1 + diff --git a/resources/common.res b/resources/common.res index 0e267f5e..fe6fa951 100644 Binary files a/resources/common.res and b/resources/common.res differ diff --git a/resources/models/computer.umdl b/resources/models/computer.umdl new file mode 100644 index 00000000..37606881 Binary files /dev/null and b/resources/models/computer.umdl differ diff --git a/resources/models/hey_extreme.umdl b/resources/models/hey_extreme.umdl new file mode 100644 index 00000000..4af46fdb Binary files /dev/null and b/resources/models/hey_extreme.umdl differ diff --git a/resources/models/jarvis.umdl b/resources/models/jarvis.umdl new file mode 100644 index 00000000..793d2539 Binary files /dev/null and b/resources/models/jarvis.umdl differ diff --git a/resources/models/neoya.umdl b/resources/models/neoya.umdl new file mode 100644 index 00000000..90dba542 Binary files /dev/null and b/resources/models/neoya.umdl differ diff --git a/resources/models/smart_mirror.umdl b/resources/models/smart_mirror.umdl new file mode 100644 index 00000000..497dc5b9 Binary files /dev/null and b/resources/models/smart_mirror.umdl differ diff --git a/resources/snowboy.umdl b/resources/models/snowboy.umdl similarity index 100% rename from resources/snowboy.umdl rename to resources/models/snowboy.umdl diff --git a/resources/models/subex.umdl b/resources/models/subex.umdl new file mode 100644 index 00000000..e9c261ee Binary files /dev/null and b/resources/models/subex.umdl differ diff --git a/resources/models/view_glass.umdl b/resources/models/view_glass.umdl new file mode 100644 index 00000000..e367dfba Binary files /dev/null and b/resources/models/view_glass.umdl differ diff --git a/resources/snowboy.raw b/resources/snowboy.raw new file mode 100644 index 00000000..93149a70 Binary files /dev/null and b/resources/snowboy.raw differ diff --git a/resources/snowboy.wav b/resources/snowboy.wav index 8ce41003..6cf6fa0a 100644 Binary files a/resources/snowboy.wav and b/resources/snowboy.wav differ diff --git a/scripts/install_swig.sh b/scripts/install_swig.sh new file mode 100755 index 00000000..46208db1 --- /dev/null +++ b/scripts/install_swig.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# SWIG is a tool to compile c++ code into Python. + +echo "Installing SWIG" + +if [ ! -e swig-3.0.10.tar.gz ]; then + cp exteral_tools/swig-3.0.10.tar.gz ./ || \ + wget -T 10 -t 3 \ + http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz || exit 1; +fi + +tar -xovzf swig-3.0.10.tar.gz || exit 1 +ln -s swig-3.0.10 swig + +cd swig + +# We first have to install PCRE. +if [ ! -e pcre-8.37.tar.gz ]; then + cp ../exteral_tools/pcre-8.37.tar.gz ./ || \ + wget -T 10 -t 3 \ + https://sourceforge.net/projects/pcre/files/pcre/8.37/pcre-8.37.tar.gz || exit 1; +fi +Tools/pcre-build.sh + +./configure --prefix=`pwd` --with-pic +make +make install + +cd .. diff --git a/scripts/publish-node.sh b/scripts/publish-node.sh index 142ee6b4..8b7304a3 100755 --- a/scripts/publish-node.sh +++ b/scripts/publish-node.sh @@ -1,7 +1,7 @@ #!/bin/bash -NODE_VERSIONS=( "4.0.0" "5.0.0" "6.0.0" "7.0.0" ) +NODE_VERSIONS=( "4.0.0" "5.0.0" "6.0.0" "7.0.0" "8.0.0" "9.0.0") # Makes sure nvm is installed. if ! which nvm > /dev/null; then diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..3ce2c9d8 --- /dev/null +++ b/setup.py @@ -0,0 +1,61 @@ +import os +import sys +from setuptools import setup, find_packages +from distutils.command.build import build +from distutils.dir_util import copy_tree +from subprocess import call + + +py_dir = 'Python' if sys.version_info[0] < 3 else 'Python3' + +class SnowboyBuild(build): + + def run(self): + + cmd = ['make'] + swig_dir = os.path.join('swig', py_dir) + def compile(): + call(cmd, cwd=swig_dir) + + self.execute(compile, [], 'Compiling snowboy...') + + # copy generated .so to build folder + self.mkpath(self.build_lib) + snowboy_build_lib = os.path.join(self.build_lib, 'snowboy') + self.mkpath(snowboy_build_lib) + target_file = os.path.join(swig_dir, '_snowboydetect.so') + if not self.dry_run: + self.copy_file(target_file, + snowboy_build_lib) + + # copy resources too since it is a symlink + resources_dir = 'resources' + resources_dir_on_build = os.path.join(snowboy_build_lib, + 'resources') + copy_tree(resources_dir, resources_dir_on_build) + + build.run(self) + + +setup( + name='snowboy', + version='1.3.0', + description='Snowboy is a customizable hotword detection engine', + maintainer='KITT.AI', + maintainer_email='snowboy@kitt.ai', + license='Apache-2.0', + url='https://snowboy.kitt.ai', + packages=find_packages(os.path.join('examples', py_dir)), + package_dir={'snowboy': os.path.join('examples', py_dir)}, + py_modules=['snowboy.snowboydecoder', 'snowboy.snowboydetect'], + package_data={'snowboy': ['resources/*']}, + zip_safe=False, + long_description="", + classifiers=[], + install_requires=[ + 'PyAudio', + ], + cmdclass={ + 'build': SnowboyBuild + } +) diff --git a/swig/Android/Makefile b/swig/Android/Makefile index 48d8c432..35f5486e 100644 --- a/swig/Android/Makefile +++ b/swig/Android/Makefile @@ -1,15 +1,38 @@ # Example Makefile that wrappers snowboy c++ library (snowboy-detect.a) through # JNI interface, using swig. -# This Makefile is optimized for armv7-a architecture. Also, please make sure +# This Makefile is optimized for armv7-a/armv8 architecture. Also, please make sure # "unzip" is installed. +# Usage: +# make # for 32bit ARM +# make BIT=64 # for 64bit ARM # Please use swig-3.0.10 or up. SWIG := swig -# Please specify your NDK root directory here. -NDK_VERSION="r14b" -NDKINSTALLEDROOT := $(PWD)/ndk_install +SWIG_VERSION := $(shell expr `$(SWIG) -version | grep -i Version | \ + sed "s/^.* //g" | sed -e "s/\.\([0-9][0-9]\)/\1/g" -e "s/\.\([0-9]\)/0\1/g" \ + -e "s/^[0-9]\{3,4\}$$/&00/"` \>= 30010) + +ifeq ($(SWIG_VERSION), 0) +checkversion: + $(info You need at least Swig 3.0.10 to run) + $(info Your current version is $(shell $(SWIG) -version | grep -i Version)) + @exit -1 +endif + + +NDK_VERSION=r14b +BIT = 32 +ifeq ($(BIT), 64) + NDKINSTALLEDROOT := $(PWD)/ndk_install_64bit + OPENBLASTARGET := ARMV8 + OPENBLASDIR := OpenBLAS-Android-ARM64 +else + NDKINSTALLEDROOT := $(PWD)/ndk_install_32bit + OPENBLASTARGET := ARMV7 + OPENBLASDIR := OpenBLAS-Android-ARM32 +endif NDKROOT := $(PWD)/android-ndk-${NDK_VERSION} SNOWBOYDETECTSWIGITF = snowboy-detect-swig.i @@ -18,7 +41,7 @@ SNOWBOYDETECTSWIGCC = snowboy-detect-swig.cc SNOWBOYDETECTJAVAPKG = ai.kitt.snowboy SNOWBOYDETECTJAVAPKGDIR = java/ai/kitt/snowboy/ SNOWBOYDETECTSWIGLIBFILE = libsnowboy-detect-android.so -OPENBLASLIBFILE = OpenBLAS-Android/install/lib/libopenblas.a +OPENBLASLIBFILE = $(OPENBLASDIR)/install/lib/libopenblas.a ARCH := arm TOPDIR := ../../ @@ -28,21 +51,37 @@ CXXFLAGS := -O3 --sysroot=$(NDKINSTALLEDROOT)/sysroot LDLIBS := -L$(NDKINSTALLEDROOT)/sysroot/usr/lib ifeq ($(ARCH), arm) - AR := $(NDKINSTALLEDROOT)/bin/arm-linux-androideabi-ar - CC := $(NDKINSTALLEDROOT)/bin/arm-linux-androideabi-gcc - CXX := $(NDKINSTALLEDROOT)/bin/arm-linux-androideabi-g++ - STRIP := $(NDKINSTALLEDROOT)/bin/arm-linux-androideabi-strip - OPENBLASTARGET := ARMV7 - SNOWBOYDETECTLIBFILE = $(TOPDIR)/lib/android/armv7a/libsnowboy-detect.a - CXXFLAGS += -std=c++0x -rdynamic -I$(TOPDIR) -Werror -Wall \ - -fsigned-char -fpic -fPIC -mfloat-abi=softfp -march=armv7-a -mfpu=neon \ - -DNDEBUG -ffast-math -fomit-frame-pointer -O3 -pie -fPIE -DHAVE_NEON=1 \ - -fno-strict-aliasing -Wno-unused-function -shared - LDLIBS += \ - -L$(NDKROOT)/sources/cxx-stl/gnu-libstdc++/4.9/libs/armeabi-v7a \ - -lgnustl_static -lsupc++ -lgcc -ldl -lc -lm -llog -pthread - SNOWBOYDETECTSWIGLIBFILE := jniLibs/armeabi-v7a/$(SNOWBOYDETECTSWIGLIBFILE) - SNOWBOYDETECTSWIGLIBNAME := $(shell basename $(SNOWBOYDETECTSWIGLIBFILE)) + ifeq ($(BIT), 64) + AR := $(NDKINSTALLEDROOT)/bin/aarch64-linux-android-ar + CC := $(NDKINSTALLEDROOT)/bin/aarch64-linux-android-gcc + CXX := $(NDKINSTALLEDROOT)/bin/aarch64-linux-android-g++ + STRIP := $(NDKINSTALLEDROOT)/bin/aarch64-linux-android-strip + SNOWBOYDETECTLIBFILE = $(TOPDIR)/lib/android/armv8-aarch64/libsnowboy-detect.a + CXXFLAGS += -std=c++0x -rdynamic -I$(TOPDIR) -Werror -Wall \ + -fsigned-char -fpic -fPIC -march=armv8-a \ + -DNDEBUG -ffast-math -fomit-frame-pointer -O3 -pie -fPIE -DHAVE_NEON=1 \ + -fno-strict-aliasing -Wno-unused-function -shared + LDLIBS += \ + -L$(NDKROOT)/sources/cxx-stl/gnu-libstdc++/4.9/libs/arm64-v8a \ + -lgnustl_static -lsupc++ -lgcc -ldl -lc -lm -llog -pthread + SNOWBOYDETECTSWIGLIBFILE := jniLibs/arm64-v8a/$(SNOWBOYDETECTSWIGLIBFILE) + SNOWBOYDETECTSWIGLIBNAME := $(shell basename $(SNOWBOYDETECTSWIGLIBFILE)) + else + AR := $(NDKINSTALLEDROOT)/bin/arm-linux-androideabi-ar + CC := $(NDKINSTALLEDROOT)/bin/arm-linux-androideabi-gcc + CXX := $(NDKINSTALLEDROOT)/bin/arm-linux-androideabi-g++ + STRIP := $(NDKINSTALLEDROOT)/bin/arm-linux-androideabi-strip + SNOWBOYDETECTLIBFILE = $(TOPDIR)/lib/android/armv7a/libsnowboy-detect.a + CXXFLAGS += -std=c++0x -rdynamic -I$(TOPDIR) -Werror -Wall \ + -fsigned-char -fpic -fPIC -mfloat-abi=softfp -march=armv7-a -mfpu=neon \ + -DNDEBUG -ffast-math -fomit-frame-pointer -O3 -pie -fPIE -DHAVE_NEON=1 \ + -fno-strict-aliasing -Wno-unused-function -shared + LDLIBS += \ + -L$(NDKROOT)/sources/cxx-stl/gnu-libstdc++/4.9/libs/armeabi-v7a \ + -lgnustl_static -lsupc++ -lgcc -ldl -lc -lm -llog -pthread + SNOWBOYDETECTSWIGLIBFILE := jniLibs/armeabi-v7a/$(SNOWBOYDETECTSWIGLIBFILE) + SNOWBOYDETECTSWIGLIBNAME := $(shell basename $(SNOWBOYDETECTSWIGLIBFILE)) + endif endif all: $(SNOWBOYSWIGLIBFILE) $(SNOWBOYDETECTSWIGLIBFILE) @@ -51,10 +90,10 @@ all: $(SNOWBOYSWIGLIBFILE) $(SNOWBOYDETECTSWIGLIBFILE) $(MAKE) -C ${@D} ${@F} $(NDKINSTALLEDROOT): - @-./install_ndk.sh ${NDK_VERSION} + @-./install_ndk.sh ${NDK_VERSION} ${BIT} $(OPENBLASLIBFILE): $(NDKINSTALLEDROOT) - @-./install_openblas.sh $(CC) $(AR) $(OPENBLASTARGET) + @-./install_openblas.sh $(CC) $(AR) $(OPENBLASTARGET) $(OPENBLASDIR) $(SNOWBOYDETECTSWIGCC): $(SNOWBOYDETECTSWIGITF) @-mkdir -p $(SNOWBOYDETECTJAVAPKGDIR) diff --git a/swig/Android/install_ndk.sh b/swig/Android/install_ndk.sh index 8e928997..266b4973 100755 --- a/swig/Android/install_ndk.sh +++ b/swig/Android/install_ndk.sh @@ -6,22 +6,28 @@ UNAME_INFO=`uname -a` NDK_REPOSITORY_URL="https://dl.google.com/android/repository/" NDK_VERSION=$1 +BIT=$2 if [[ $UNAME_INFO == *"Darwin"* ]]; then if [ ! -d "android-ndk-${NDK_VERSION}" ]; then wget -T 10 -t 3 ${NDK_REPOSITORY_URL}/android-ndk-${NDK_VERSION}-darwin-x86_64.zip \ -O android-ndk-${NDK_VERSION}-darwin-x86_64.zip || exit 1; + unzip android-ndk-${NDK_VERSION}-darwin-x86_64.zip 1>/dev/null || exit 1; fi - unzip android-ndk-${NDK_VERSION}-darwin-x86_64.zip 1>/dev/null || exit 1; elif [[ $UNAME_INFO == *"Linux"* ]]; then if [ ! -d "android-ndk-${NDK_VERSION}" ]; then wget -T 10 -t 3 ${NDK_REPOSITORY_URL}/android-ndk-${NDK_VERSION}-linux-x86_64.zip \ -O android-ndk-${NDK_VERSION}-linux-x86_64.zip || exit 1; + unzip android-ndk-${NDK_VERSION}-linux-x86_64.zip 1>/dev/null || exit 1; fi - unzip android-ndk-${NDK_VERSION}-linux-x86_64.zip 1>/dev/null || exit 1; else echo "Your platform is not supported yet." || exit 1; fi -./android-ndk-${NDK_VERSION}/build/tools/make-standalone-toolchain.sh \ - --arch=arm --platform=android-14 --install-dir=`pwd`/ndk_install || exit 1; +if [[ $BIT == *"64"* ]]; then + ./android-ndk-${NDK_VERSION}/build/tools/make-standalone-toolchain.sh --verbose \ + --arch=arm64 --platform=android-21 --install-dir=`pwd`/ndk_install_64bit || exit 1; +else + ./android-ndk-${NDK_VERSION}/build/tools/make-standalone-toolchain.sh --verbose \ + --arch=arm --platform=android-14 --install-dir=`pwd`/ndk_install_32bit || exit 1; +fi diff --git a/swig/Android/install_openblas.sh b/swig/Android/install_openblas.sh index 57fe3c19..d914558d 100755 --- a/swig/Android/install_openblas.sh +++ b/swig/Android/install_openblas.sh @@ -7,16 +7,17 @@ CC=$1 AR=$2 TARGET=$3 +DIR=$4 -if [ ! -d "OpenBLAS-Android" ]; then - git clone https://github.com/xianyi/OpenBLAS.git OpenBLAS-Android - cd OpenBLAS-Android +if [ ! -d $DIR ]; then + git clone https://github.com/xianyi/OpenBLAS.git $DIR + cd $DIR git checkout arm_soft_fp_abi || exit 1; git reset --hard b5c96fcfcdc82945502a2303116a64d89985daf5 || exit 1; cd .. fi -cd OpenBLAS-Android +cd $DIR make USE_THREAD=0 TARGET=${TARGET} HOSTCC=gcc CC=${CC} AR=${AR} \ NOFORTRAN=1 ARM_SOFTFP_ABI=1 libs || exit 1; make PREFIX=`pwd`/install install || exit 1; diff --git a/swig/Go/snowboy.go b/swig/Go/snowboy.go index 92d90b3b..b03774a9 100644 --- a/swig/Go/snowboy.go +++ b/swig/Go/snowboy.go @@ -2,8 +2,8 @@ package snowboydetect /* #cgo CXXFLAGS: -std=c++11 -D_GLIBCXX_USE_CXX11_ABI=0 -#cgo linux,amd64 LDFLAGS: -lcblas -L${SRCDIR}/../../lib/ubuntu64 -lsnowboy-detect -#cgo linux,arm LDFLAGS: -lcblas -L${SRCDIR}/../../lib/rpi -lsnowboy-detect -#cgo darwin LDFLAGS: -lcblas -L${SRCDIR}/../../lib/osx -lsnowboy-detect +#cgo linux,amd64 LDFLAGS: -L${SRCDIR}/../../lib/ubuntu64 -lsnowboy-detect -lcblas +#cgo linux,arm LDFLAGS: -L${SRCDIR}/../../lib/rpi -lsnowboy-detect -lcblas +#cgo darwin LDFLAGS: -L${SRCDIR}/../../lib/osx -lsnowboy-detect -lcblas */ import "C" diff --git a/swig/Java/Makefile b/swig/Java/Makefile index 317c82b7..7e58bb20 100644 --- a/swig/Java/Makefile +++ b/swig/Java/Makefile @@ -6,6 +6,18 @@ # Please use swig-3.0.10 or up. SWIG := swig +SWIG_VERSION := $(shell expr `$(SWIG) -version | grep -i Version | \ + sed "s/^.* //g" | sed -e "s/\.\([0-9][0-9]\)/\1/g" -e "s/\.\([0-9]\)/0\1/g" \ + -e "s/^[0-9]\{3,4\}$$/&00/"` \>= 30010) + +ifeq ($(SWIG_VERSION), 0) +checkversion: + $(info You need at least Swig 3.0.10 to run) + $(info Your current version is $(shell $(SWIG) -version | grep -i Version)) + @exit -1 +endif + + SNOWBOYDETECTSWIGITF = snowboy-detect-swig.i SNOWBOYDETECTSWIGOBJ = snowboy-detect-swig.o SNOWBOYDETECTSWIGCC = snowboy-detect-swig.cc diff --git a/swig/Node/snowboy.cc b/swig/Node/snowboy.cc index ea020fba..e0aa5432 100644 --- a/swig/Node/snowboy.cc +++ b/swig/Node/snowboy.cc @@ -22,6 +22,7 @@ class SnowboyDetect : public Nan::ObjectWrap { static NAN_METHOD(SampleRate); static NAN_METHOD(NumChannels); static NAN_METHOD(BitsPerSample); + static NAN_METHOD(ApplyFrontend); static Nan::Persistent constructor; @@ -59,6 +60,7 @@ NAN_MODULE_INIT(SnowboyDetect::Init) { SetPrototypeMethod(tpl, "SampleRate", SampleRate); SetPrototypeMethod(tpl, "NumChannels", NumChannels); SetPrototypeMethod(tpl, "BitsPerSample", BitsPerSample); + SetPrototypeMethod(tpl, "ApplyFrontend", ApplyFrontend); constructor.Reset(Nan::GetFunction(tpl).ToLocalChecked()); Nan::Set(target, Nan::New("SnowboyDetect").ToLocalChecked(), @@ -123,6 +125,14 @@ NAN_METHOD(SnowboyDetect::SetSensitivity) { ptr->detector->SetSensitivity(*sensitivityString); } +NAN_METHOD(SnowboyDetect::ApplyFrontend) { + Nan::Maybe applyFrontend= Nan::To(info[0]); + bool applyFrontendBool=applyFrontend.FromJust(); + + SnowboyDetect* ptr = Nan::ObjectWrap::Unwrap(info.Holder()); + ptr->detector->ApplyFrontend(applyFrontendBool); +} + NAN_METHOD(SnowboyDetect::GetSensitivity) { SnowboyDetect* ptr = Nan::ObjectWrap::Unwrap(info.Holder()); std::string sensitivity = ptr->detector->GetSensitivity(); diff --git a/swig/Perl/Makefile b/swig/Perl/Makefile index 96909aee..9b2b17b1 100644 --- a/swig/Perl/Makefile +++ b/swig/Perl/Makefile @@ -4,6 +4,18 @@ # Please use swig-3.0.10 or up. SWIG := swig +SWIG_VERSION := $(shell expr `$(SWIG) -version | grep -i Version | \ + sed "s/^.* //g" | sed -e "s/\.\([0-9][0-9]\)/\1/g" -e "s/\.\([0-9]\)/0\1/g" \ + -e "s/^[0-9]\{3,4\}$$/&00/"` \>= 30010) + +ifeq ($(SWIG_VERSION), 0) +checkversion: + $(info You need at least Swig 3.0.10 to run) + $(info Your current version is $(shell $(SWIG) -version | grep -i Version)) + @exit -1 +endif + + SNOWBOYDETECTSWIGITF = snowboy-detect.i SNOWBOYDETECTSWIGOBJ = snowboy-detect-swig.o SNOWBOYDETECTSWIGCC = snowboy-detect-swig.cc diff --git a/swig/Python/Makefile b/swig/Python/Makefile index 956de089..c4aa5cbc 100644 --- a/swig/Python/Makefile +++ b/swig/Python/Makefile @@ -4,6 +4,17 @@ # Please use swig-3.0.10 or up. SWIG := swig +SWIG_VERSION := $(shell expr `$(SWIG) -version | grep -i Version | \ + sed "s/^.* //g" | sed -e "s/\.\([0-9][0-9]\)/\1/g" -e "s/\.\([0-9]\)/0\1/g" \ + -e "s/^[0-9]\{3,4\}$$/&00/"` \>= 30010) + +ifeq ($(SWIG_VERSION), 0) +checkversion: + $(info You need at least Swig 3.0.10 to run) + $(info Your current version is $(shell $(SWIG) -version | grep -i Version)) + @exit -1 +endif + SNOWBOYDETECTSWIGITF = snowboy-detect-swig.i SNOWBOYDETECTSWIGOBJ = snowboy-detect-swig.o SNOWBOYDETECTSWIGCC = snowboy-detect-swig.cc @@ -28,7 +39,11 @@ else CXXFLAGS += -std=c++0x # Make sure you have Atlas installed. You can statically link Atlas if you # would like to be able to move the library to a machine without Atlas. - LDLIBS := -lm -ldl -lf77blas -lcblas -llapack_atlas -latlas + ifneq ("$(ldconfig -p | grep lapack_atlas)","") + LDLIBS := -lm -ldl -lf77blas -lcblas -llapack_atlas -latlas + else + LDLIBS := -lm -ldl -lf77blas -lcblas -llapack -latlas + endif SNOWBOYDETECTLIBFILE = $(TOPDIR)/lib/ubuntu64/libsnowboy-detect.a ifneq (,$(findstring arm,$(shell uname -m))) SNOWBOYDETECTLIBFILE = $(TOPDIR)/lib/rpi/libsnowboy-detect.a diff --git a/swig/Python3/Makefile b/swig/Python3/Makefile new file mode 100644 index 00000000..5bf8b15d --- /dev/null +++ b/swig/Python3/Makefile @@ -0,0 +1,77 @@ +# Example Makefile that converts snowboy c++ library (snowboy-detect.a) to +# python3 library (_snowboydetect.so, snowboydetect.py), using swig. + +# Please use swig-3.0.10 or up. +SWIG := swig + +SWIG_VERSION := $(shell expr `$(SWIG) -version | grep -i Version | \ + sed "s/^.* //g" | sed -e "s/\.\([0-9][0-9]\)/\1/g" -e "s/\.\([0-9]\)/0\1/g" \ + -e "s/^[0-9]\{3,4\}$$/&00/"` \>= 30010) + +ifeq ($(SWIG_VERSION), 0) +checkversion: + $(info You need at least Swig 3.0.10 to run) + $(info Your current version is $(shell $(SWIG) -version | grep -i Version)) + @exit -1 +endif + + +SNOWBOYDETECTSWIGITF = snowboy-detect-swig.i +SNOWBOYDETECTSWIGOBJ = snowboy-detect-swig.o +SNOWBOYDETECTSWIGCC = snowboy-detect-swig.cc +SNOWBOYDETECTSWIGLIBFILE = _snowboydetect.so + +TOPDIR := ../../ +CXXFLAGS := -I$(TOPDIR) -O3 -fPIC -D_GLIBCXX_USE_CXX11_ABI=0 +LDFLAGS := + +ifeq ($(shell uname), Darwin) + CXX := clang++ + PYINC := $(shell python3-config --includes) + # If you use Anaconda, the command `python3-config` will not return full path. + # In this case, please manually specify the full path like the following: + # PYLIBS := -L/Users/YOURNAME/anaconda3/lib/python3.6/config-3.6m-darwin -lpython3.6m -ldl -framework CoreFoundation + PYLIBS := $(shell python3-config --ldflags) + SWIGFLAGS := -bundle -flat_namespace -undefined suppress + LDLIBS := -lm -ldl -framework Accelerate + SNOWBOYDETECTLIBFILE = $(TOPDIR)/lib/osx/libsnowboy-detect.a +else + CXX := g++ + PYINC := $(shell python3-config --cflags) + PYLIBS := $(shell python3-config --ldflags) + SWIGFLAGS := -shared + CXXFLAGS += -std=c++0x + # Make sure you have Atlas installed. You can statically link Atlas if you + # would like to be able to move the library to a machine without Atlas. + ifneq ("$(ldconfig -p | grep lapack_atlas)","") + LDLIBS := -lm -ldl -lf77blas -lcblas -llapack_atlas -latlas + else + LDLIBS := -lm -ldl -lf77blas -lcblas -llapack -latlas + endif + SNOWBOYDETECTLIBFILE = $(TOPDIR)/lib/ubuntu64/libsnowboy-detect.a + ifneq (,$(findstring arm,$(shell uname -m))) + SNOWBOYDETECTLIBFILE = $(TOPDIR)/lib/rpi/libsnowboy-detect.a + ifeq ($(findstring fc,$(shell uname -r)), fc) + SNOWBOYDETECTLIBFILE = $(TOPDIR)/lib/fedora25-armv7/libsnowboy-detect.a + LDLIBS := -L/usr/lib/atlas -lm -ldl -lsatlas + endif + endif +endif + +all: $(SNOWBOYSWIGLIBFILE) $(SNOWBOYDETECTSWIGLIBFILE) + +%.a: + $(MAKE) -C ${@D} ${@F} + +$(SNOWBOYDETECTSWIGCC): $(SNOWBOYDETECTSWIGITF) + $(SWIG) -I$(TOPDIR) -c++ -python -o $(SNOWBOYDETECTSWIGCC) $(SNOWBOYDETECTSWIGITF) + +$(SNOWBOYDETECTSWIGOBJ): $(SNOWBOYDETECTSWIGCC) + $(CXX) $(PYINC) $(CXXFLAGS) -c $(SNOWBOYDETECTSWIGCC) + +$(SNOWBOYDETECTSWIGLIBFILE): $(SNOWBOYDETECTSWIGOBJ) $(SNOWBOYDETECTLIBFILE) + $(CXX) $(CXXFLAGS) $(LDFLAGS) $(SWIGFLAGS) $(SNOWBOYDETECTSWIGOBJ) \ + $(SNOWBOYDETECTLIBFILE) $(PYLIBS) $(LDLIBS) -o $(SNOWBOYDETECTSWIGLIBFILE) + +clean: + -rm -f *.o *.a *.so snowboydetect.py *.pyc $(SNOWBOYDETECTSWIGCC) diff --git a/swig/Python3/snowboy-detect-swig.i b/swig/Python3/snowboy-detect-swig.i new file mode 100644 index 00000000..d383fed7 --- /dev/null +++ b/swig/Python3/snowboy-detect-swig.i @@ -0,0 +1,24 @@ +// swig/Python/snowboy-detect-swig.i + +// Copyright 2016 KITT.AI (author: Guoguo Chen) + +%module snowboydetect + +// Suppress SWIG warnings. +#pragma SWIG nowarn=SWIGWARN_PARSE_NESTED_CLASS +%include "std_string.i" + +%{ +#include "include/snowboy-detect.h" +%} + +%include "include/snowboy-detect.h" + +// below is Python 3 support, however, +// adding it will generate wrong .so file +// for Fedora 25 on ARMv7. So be sure to +// comment them when you compile for +// Fedora 25 on ARMv7. +%begin %{ +#define SWIG_PYTHON_STRICT_BYTE_CHAR +%} diff --git a/tsconfig.json b/tsconfig.json index e74e67e8..2914c2f6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,7 +7,7 @@ "jsx": "react", "experimentalDecorators": false, "emitDecoratorMetadata": false, - "declaration": false, + "declaration": true, "noImplicitAny": true, "noImplicitUseStrict": false, "noFallthroughCasesInSwitch": true,